scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 74

feat(router): reject with 429 when a deployment's max_parallel_requests slots are all in use

Browse source 
Replace the per-deployment asyncio.Semaphore with MaxParallelRequestsLimit, which admits a call synchronously or raises the router's RateLimitError (429) right away. Nothing waits for a slot any more, so the max_parallel_requests_queue_size and default_max_parallel_requests_queue_size settings from the earlier commits are dropped along with their proxy validation, dashboard control and generated schema entries. The rpm/tpm derivation of the cap is unchanged. Every router endpoint family now enters the slot through one _deployment_slot context, and the provider coroutine is only created once the slot is held

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-17 18:14:09 +00:00
parent 45ceb56110
commit 8d972eefc7
17 changed files with 160 additions and 500 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/constants.py
View file

@ -30,10 +30,8 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"enable_tag_filtering",
"tag_routing_prefix",
"optional_pre_call_checks",
"default_max_parallel_requests_queue_size",
}
)
NULLABLE_RUNTIME_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset({"default_max_parallel_requests_queue_size"})
ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset(
{
"model_list",

1
litellm/llms/anthropic/prompt_cache_prediction.py
View file

@ -50,7 +50,6 @@ _DEPLOYMENT_OPTIONS: Final = frozenset(
"max_retries",
"num_retries",
"max_parallel_requests",
"max_parallel_requests_queue_size",
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",

28
litellm/proxy/proxy_server.py
View file

@ -70,7 +70,6 @@ from litellm.constants import (
LITELLM_SETTINGS_SAFE_DB_OVERRIDES,
LITELLM_UI_ALLOW_HEADERS,
LITELLM_UI_SESSION_DURATION,
NULLABLE_RUNTIME_ROUTER_SETTINGS,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
)
from litellm.litellm_core_utils.asyncify import asyncify
@ -776,7 +775,6 @@ from litellm.types.router import (
RoutingPlugin,
SearchToolTypedDict,
updateDeployment,
validate_max_parallel_requests_queue_size,
)
from litellm.types.router import ModelInfo as RouterModelInfo
from litellm.types.scheduler import DefaultPriorities
@ -6902,20 +6900,13 @@ class ProxyConfig:
):
from litellm.utils import _update_dictionary
db_settings: Final = db_router_settings.param_value
db_overlay_deferring_empty_lists_to_config: Final = {
k: v
for k, v in db_settings.items()
for k, v in db_router_settings.param_value.items()
if not (k in config_router_settings and isinstance(v, list) and len(v) == 0)
}
cleared_nullable_settings: Final = MappingProxyType(
{k: None for k in NULLABLE_RUNTIME_ROUTER_SETTINGS if k in db_settings and db_settings[k] is None}
)
combined_router_settings = MappingProxyType(
{
**_update_dictionary(config_router_settings, db_overlay_deferring_empty_lists_to_config),
**cleared_nullable_settings,
}
combined_router_settings = _update_dictionary(
config_router_settings, db_overlay_deferring_empty_lists_to_config
)
elif config_router_settings is not None and isinstance(config_router_settings, dict):
combined_router_settings = config_router_settings
@ -16937,17 +16928,6 @@ async def update_config(
)
},
)
raw_queue_size: Final = raw_router_settings.get("default_max_parallel_requests_queue_size")
try:
validate_max_parallel_requests_queue_size(raw_queue_size)
except ValueError as invalid_queue_size:
raise HTTPException(
status_code=400,
detail=(
f"default_max_parallel_requests_queue_size={raw_queue_size!r} is not valid, "
"it must be a non-negative integer or null"
),
) from invalid_queue_size
if prisma_client is None:
raise Exception("No DB Connected")
@ -17059,7 +17039,7 @@ async def update_config(
raw_router_settings_without_none: Final = {
key: value
for key, value in raw_router_settings.items()
if key not in typed_router_settings and (value is not None or key in NULLABLE_RUNTIME_ROUTER_SETTINGS)
if key not in typed_router_settings and value is not None
}
router_settings_updates: Final = {**typed_router_settings, **raw_router_settings_without_none}
new_router_settings: Final = {**existing, **router_settings_updates}

31
litellm/router.py
View file

@ -148,7 +148,7 @@ from litellm.router_utils.batch_utils import (
replace_model_in_jsonl,
should_replace_model_in_jsonl,
)
from litellm.router_utils.client_initalization_utils import DeploymentSemaphore, InitalizeCachedClient
from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit
from litellm.router_utils.clientside_credential_handler import (
get_dynamic_litellm_params,
is_clientside_credential,
@ -263,7 +263,6 @@ from litellm.types.router import (
RoutingStrategy,
SearchToolTypedDict,
TaggedPreRoutingStrategy,
validate_max_parallel_requests_queue_size,
)
from litellm.types.services import ServiceTypes
from litellm.types.utils import (
@ -739,7 +738,6 @@ class Router:
stream_timeout: float | None = None,
default_litellm_params: dict | None = None, # default params for Router.chat.completion.create
default_max_parallel_requests: int | None = None,
default_max_parallel_requests_queue_size: int | None = None,
set_verbose: bool = False,
debug_level: Literal["DEBUG", "INFO"] = "INFO",
default_fallbacks: list[str] | None = None, # generic fallbacks, works across all deployments
@ -937,9 +935,6 @@ class Router:
None # use this to track the users default deployment, when they want to use model = *
)
self.default_max_parallel_requests = default_max_parallel_requests
self._default_max_parallel_requests_queue_size = validate_max_parallel_requests_queue_size(
default_max_parallel_requests_queue_size
)
self.provider_default_deployment_ids: list[str] = []
self.pattern_router = PatternMatchRouter()
self.team_pattern_routers: dict[str, PatternMatchRouter] = {} # {"TEAM_ID": PatternMatchRouter}
@ -3637,14 +3632,14 @@ class Router:
logging_obj: Final[LiteLLMLogging | None] = kwargs.get("litellm_logging_obj", None)
rpm_semaphore: Final = self._get_client(
max_parallel_requests_limit: Final = self._get_client(
deployment=deployment,
kwargs=kwargs,
client_type="max_parallel_requests",
)
async with contextlib.AsyncExitStack() as deployment_slot:
if isinstance(rpm_semaphore, DeploymentSemaphore):
await deployment_slot.enter_async_context(rpm_semaphore)
if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit):
deployment_slot.enter_context(max_parallel_requests_limit)
await self.async_routing_strategy_pre_call_checks(
deployment=deployment,
logging_obj=logging_obj,
@ -8509,14 +8504,14 @@ class Router:
) -> AsyncGenerator[None, None]:
"""Holds the deployment's max_parallel_requests slot, if it has one, around the provider call. Routing
strategy pre-call checks run inside the slot so their rpm accounting stays concurrency-safe."""
rpm_semaphore: Final = self._get_client(
max_parallel_requests_limit: Final = self._get_client(
deployment=deployment,
kwargs=kwargs,
client_type="max_parallel_requests",
)
async with contextlib.AsyncExitStack() as slot:
if isinstance(rpm_semaphore, DeploymentSemaphore):
await slot.enter_async_context(rpm_semaphore)
if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit):
slot.enter_context(max_parallel_requests_limit)
await self.async_routing_strategy_pre_call_checks(deployment=deployment, parent_otel_span=parent_otel_span)
yield
@ -11846,20 +11841,8 @@ class Router:
_settings_to_return[var] = self.lowestlatency_logger.routing_args.json()
_settings_to_return["routing_groups"] = [group.model_dump() for group in self._routing_groups.values()]
_settings_to_return["default_max_parallel_requests_queue_size"] = self.default_max_parallel_requests_queue_size
return _settings_to_return
@property
def default_max_parallel_requests_queue_size(self) -> int | None:
return self._default_max_parallel_requests_queue_size
@default_max_parallel_requests_queue_size.setter
def default_max_parallel_requests_queue_size(self, queue_size: int | None) -> None:
self._default_max_parallel_requests_queue_size = validate_max_parallel_requests_queue_size(queue_size)
InitalizeCachedClient.apply_default_max_parallel_requests_queue_size(
litellm_router_instance=self, queue_size=self._default_max_parallel_requests_queue_size
)
def update_settings(self, **kwargs):
"""
Update the router settings.

96
litellm/router_utils/client_initalization_utils.py
View file

@ -1,11 +1,8 @@
import asyncio
import time
from types import TracebackType
from typing import TYPE_CHECKING, Any, Final
from litellm._logging import verbose_router_logger
from litellm.exceptions import RateLimitError, RateLimitErrorCategory, RateLimitType
from litellm.types.router import RouterErrors, validate_max_parallel_requests_queue_size
from litellm.types.router import RouterErrors
from litellm.utils import calculate_max_parallel_requests
if TYPE_CHECKING:
@ -16,71 +13,41 @@ else:
LitellmRouter = Any
class DeploymentSemaphore:
"""A deployment's max_parallel_requests slots. ``queue_size=None`` parks callers without bound, like a plain
``asyncio.Semaphore``; otherwise a caller arriving while all slots are busy and ``queue_size`` callers already
wait gets a 429 instead of being parked."""
class MaxParallelRequestsLimit:
"""A deployment's max_parallel_requests slots. A caller arriving while every slot is in use gets a 429 instead
of waiting for one to free up."""
def __init__(self, max_parallel_requests: int, model_id: str, model_group: str, queue_size: int | None) -> None:
self._slots: Final = asyncio.Semaphore(max_parallel_requests)
def __init__(self, max_parallel_requests: int, model_id: str, model_group: str) -> None:
self.max_parallel_requests: Final = max_parallel_requests
self.model_id: Final = model_id
self.model_group: Final = model_group
self.queue_size = validate_max_parallel_requests_queue_size(queue_size)
self.waiting = 0
self.in_flight = 0
def locked(self) -> bool:
return self._slots.locked()
def __enter__(self) -> None:
self.acquire()
def release(self) -> None:
self._slots.release()
async def __aenter__(self) -> None:
await self.acquire()
async def __aexit__(
def __exit__(
self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None
) -> None:
self._slots.release()
self.release()
async def acquire(self) -> bool:
if not self._slots.locked():
return await self._slots.acquire()
if self.queue_size is not None and self.waiting >= self.queue_size:
def acquire(self) -> None:
if self.in_flight >= self.max_parallel_requests:
raise RateLimitError(
message=(
f"{RouterErrors.max_parallel_requests_queue_full.value} Deployment model_group={self.model_group}, "
f"id={self.model_id} has all max_parallel_requests={self.max_parallel_requests} slots in use and "
f"{self.waiting} requests already waiting, which is its max_parallel_requests_queue_size="
f"{self.queue_size}. Raise max_parallel_requests or max_parallel_requests_queue_size for this "
"deployment, or unset max_parallel_requests_queue_size to queue without a bound"
f"{RouterErrors.max_parallel_requests_exceeded.value} Deployment model_group={self.model_group}, "
f"id={self.model_id} already has max_parallel_requests={self.max_parallel_requests} requests in "
"flight. Raise max_parallel_requests (or the rpm/tpm it is derived from) for this deployment"
),
llm_provider="",
model=self.model_group,
category=RateLimitErrorCategory.LITELLM_RATE_LIMIT,
rate_limit_type=RateLimitType.CONCURRENT_REQUESTS,
)
self.waiting += 1
queued_at: Final = time.perf_counter()
verbose_router_logger.debug(
"Deployment model_group=%s, id=%s has all max_parallel_requests=%s slots in use, request queued "
"(waiting=%s, max_parallel_requests_queue_size=%s)",
self.model_group,
self.model_id,
self.max_parallel_requests,
self.waiting,
self.queue_size,
)
try:
return await self._slots.acquire()
finally:
self.waiting -= 1
verbose_router_logger.debug(
"Deployment model_group=%s, id=%s request left the max_parallel_requests queue after %.1f ms",
self.model_group,
self.model_id,
(time.perf_counter() - queued_at) * 1000,
)
self.in_flight += 1
def release(self) -> None:
self.in_flight -= 1
class InitalizeCachedClient:
@ -98,35 +65,14 @@ class InitalizeCachedClient:
default_max_parallel_requests=litellm_router_instance.default_max_parallel_requests,
)
if calculated_max_parallel_requests:
deployment_queue_size: Final = litellm_params.get("max_parallel_requests_queue_size", None)
semaphore: Final = DeploymentSemaphore(
limit: Final = MaxParallelRequestsLimit(
max_parallel_requests=calculated_max_parallel_requests,
model_id=model_id,
model_group=model.get("model_name", ""),
queue_size=(
deployment_queue_size
if deployment_queue_size is not None
else litellm_router_instance.default_max_parallel_requests_queue_size
),
)
cache_key: Final = f"{model_id}_max_parallel_requests_client"
litellm_router_instance.cache.set_cache(
key=cache_key,
value=semaphore,
value=limit,
local_only=True,
)
@staticmethod
def apply_default_max_parallel_requests_queue_size(
litellm_router_instance: LitellmRouter, queue_size: int | None
) -> None:
inheriting_semaphores: Final = (
litellm_router_instance.cache.get_cache(
key=f"{model['model_info']['id']}_max_parallel_requests_client", local_only=True
)
for model in litellm_router_instance.model_list
if model["litellm_params"].get("max_parallel_requests_queue_size") is None
)
for semaphore in inheriting_semaphores:
if isinstance(semaphore, DeploymentSemaphore):
semaphore.queue_size = queue_size

11
litellm/types/management_endpoints/router_settings_endpoints.py
View file

@ -244,17 +244,6 @@ ROUTER_SETTINGS_FIELDS: Final[list[RouterSettingsField]] = [
field_default=None,
ui_field_name="Max Parallel Requests",
),
RouterSettingsField(
field_name="default_max_parallel_requests_queue_size",
field_type="Integer",
field_value=None,
field_description=(
"Default cap on how many requests may wait for a deployment's max_parallel_requests slot before "
"further requests get a 429. Unset queues without a bound"
),
field_default=None,
ui_field_name="Max Parallel Requests Queue Size",
),
RouterSettingsField(
field_name="enable_tag_filtering",
field_type="Boolean",

16
litellm/types/router.py
View file

@ -6,10 +6,10 @@ import datetime
import enum
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints
from typing import TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints
import httpx
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator, model_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
from litellm._logging import verbose_logger
@ -314,14 +314,6 @@ class CredentialLiteLLMParams(BaseModel):
_RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"})
MaxParallelRequestsQueueSize = Annotated[int, Field(strict=True, ge=0)]
_MAX_PARALLEL_REQUESTS_QUEUE_SIZE_ADAPTER: Final = TypeAdapter(MaxParallelRequestsQueueSize | None)
def validate_max_parallel_requests_queue_size(value: object) -> int | None:
return _MAX_PARALLEL_REQUESTS_QUEUE_SIZE_ADAPTER.validate_python(value)
class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
"""
LiteLLM Params without 'model' arg (used across completion / assistants api)
@ -332,7 +324,6 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
rpm: int | None = None
itpm: int | None = None
otpm: int | None = None
max_parallel_requests_queue_size: MaxParallelRequestsQueueSize | None = None
timeout: float | str | httpx.Timeout | None = None # if str, pass in as os.environ/
stream_timeout: float | str | None = None # timeout when making stream=True calls, if str, pass in as os.environ/
max_retries: int | None = None
@ -506,7 +497,6 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
order: int | None
weight: int | None
max_parallel_requests: int | None
max_parallel_requests_queue_size: ReadOnly[MaxParallelRequestsQueueSize | None]
api_key: str | None
api_base: str | None
api_version: str | None
@ -657,7 +647,7 @@ class RouterErrors(enum.Enum):
"""
user_defined_ratelimit_error = "Deployment over user-defined ratelimit."
max_parallel_requests_queue_full = "Deployment max_parallel_requests queue is full."
max_parallel_requests_exceeded = "Deployment has all max_parallel_requests slots in use."
no_deployments_available = "No deployments available for selected model"
all_deployments_in_cooldown = "All deployments for selected model are in cooldown"
no_deployments_with_tag_routing = "Not allowed to access model due to tags configuration"

1
litellm/types/utils.py
View file

@ -3841,7 +3841,6 @@ all_litellm_params = (
"itpm",
"otpm",
"max_parallel_requests",
"max_parallel_requests_queue_size",
"input_cost_per_token",
"output_cost_per_token",
"input_cost_per_second",

1
tests/code_coverage_tests/router_code_coverage.py
View file

@ -88,7 +88,6 @@ ignored_function_names = [
"_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py
"_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py
"_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name)
"default_max_parallel_requests_queue_size",
]

13
tests/local_testing/test_router_max_parallel_requests.py
View file

@ -11,6 +11,7 @@ import pytest
from typing import Optional
import litellm
from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
from litellm.utils import calculate_max_parallel_requests
"""
@ -93,26 +94,26 @@ def test_setting_mpr_limits_per_model(
default_max_parallel_requests=default_max_parallel_requests,
)
mpr_client: Optional[asyncio.Semaphore] = router._get_client(
mpr_client: Optional[MaxParallelRequestsLimit] = router._get_client(
deployment=deployment,
kwargs={},
client_type="max_parallel_requests",
)
if max_parallel_requests is not None:
assert max_parallel_requests == mpr_client._value
assert max_parallel_requests == mpr_client.max_parallel_requests
elif rpm is not None:
assert rpm == mpr_client._value
assert rpm == mpr_client.max_parallel_requests
elif tpm is not None:
calculated_rpm = int(tpm / 1000 * 6)
if calculated_rpm == 0:
calculated_rpm = 1
print(
f"test calculated_rpm: {calculated_rpm}, calculated_max_parallel_requests={mpr_client._value}"
f"test calculated_rpm: {calculated_rpm}, calculated_max_parallel_requests={mpr_client.max_parallel_requests}"
)
assert calculated_rpm == mpr_client._value
assert calculated_rpm == mpr_client.max_parallel_requests
elif default_max_parallel_requests is not None:
assert mpr_client._value == default_max_parallel_requests
assert mpr_client.max_parallel_requests == default_max_parallel_requests
else:
assert mpr_client is None

9
tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py
View file

@ -181,15 +181,6 @@ async def test_environment_credential_matches_native_count_and_observed_scope(
assert observed.scope == cache_scope(_CALLER, _DEPLOYMENT, target.api_key, target.model)
def test_deployment_concurrency_knobs_keep_native_prediction_supported() -> None:
target: Final = resolve_prediction_target(LiteLLM_Params(
model=f"anthropic/{_MODEL}", api_key=_KEY, api_base="https://api.anthropic.com",
max_parallel_requests=1, max_parallel_requests_queue_size=0,
))
assert isinstance(target, NativePredictionTarget)
assert (target.model, target.api_key) == (_MODEL, _KEY)
@pytest.mark.parametrize("inline_key", [None, _KEY])
@pytest.mark.asyncio
async def test_named_credential_is_explicitly_unsupported_before_count(

78
tests/test_litellm/proxy/test_proxy_server.py
View file

@ -5051,39 +5051,6 @@ async def test_add_router_settings_from_db_config_empty_db_list_still_clears_unc
assert combined_settings["num_retries"] == 1
@pytest.mark.asyncio
async def test_add_router_settings_from_db_config_null_queue_size_reaches_router():
"""A cleared Admin UI field is stored as null. The reload must hand that None to the
router so a config.yaml bound is lifted, while an unrelated null still falls back to
the config value."""
from unittest.mock import AsyncMock, MagicMock
from litellm.proxy.proxy_server import ProxyConfig
proxy_config = ProxyConfig()
mock_router = MagicMock()
mock_router.update_settings = MagicMock()
config_data = {"router_settings": {"default_max_parallel_requests_queue_size": 2, "num_retries": 1}}
mock_db_config = MagicMock()
mock_db_config.param_value = {"default_max_parallel_requests_queue_size": None, "num_retries": None}
mock_prisma_client = MagicMock()
mock_prisma_client.db.litellm_config.find_first = AsyncMock(return_value=mock_db_config)
await proxy_config._add_router_settings_from_db_config(
config_data=config_data,
llm_router=mock_router,
prisma_client=mock_prisma_client,
)
combined_settings = mock_router.update_settings.call_args.kwargs
assert "default_max_parallel_requests_queue_size" in combined_settings
assert combined_settings["default_max_parallel_requests_queue_size"] is None
assert combined_settings["num_retries"] == 1
@pytest.mark.asyncio
async def test_add_router_settings_from_db_config_edge_cases():
"""
@ -9367,51 +9334,6 @@ def test_update_config_litellm_settings_request_wins_for_non_callback_keys(
restore()
def test_update_config_router_settings_null_clears_max_parallel_requests_queue_size(
_update_config_setup,
):
"""Clearing the Admin UI field sends null. The stored row must hold null so the
reload hands None to the router and queueing becomes unbounded again, while an
unrelated null is still dropped rather than persisted."""
client, prisma, restore = _update_config_setup(
initial_rows={
"router_settings": {"default_max_parallel_requests_queue_size": 3, "num_retries": 2},
}
)
try:
resp = client.post(
"/config/update",
json={"router_settings": {"default_max_parallel_requests_queue_size": None, "timeout": None}},
)
assert resp.status_code == 200
stored = prisma.db.litellm_config.rows["router_settings"]
assert "default_max_parallel_requests_queue_size" in stored
assert stored["default_max_parallel_requests_queue_size"] is None
assert stored["num_retries"] == 2
assert "timeout" not in stored
finally:
restore()
@pytest.mark.parametrize("invalid_queue_size", [-1, 2.5, "3"])
def test_update_config_rejects_invalid_max_parallel_requests_queue_size_before_persisting(
_update_config_setup, invalid_queue_size
):
client, prisma, restore = _update_config_setup(
initial_rows={"router_settings": {"default_max_parallel_requests_queue_size": 3}},
)
try:
resp = client.post(
"/config/update",
json={"router_settings": {"default_max_parallel_requests_queue_size": invalid_queue_size}},
)
assert resp.status_code == 400
assert "default_max_parallel_requests_queue_size" in resp.json()["error"]["message"]
assert prisma.db.litellm_config.rows["router_settings"] == {"default_max_parallel_requests_queue_size": 3}
finally:
restore()
def test_update_config_success_callback_normalizes_existing_mixed_case(
_update_config_setup,
):

236
tests/test_litellm/router_utils/test_client_initalization_utils.py
View file

@ -2,220 +2,124 @@ import asyncio
from typing import Final
import pytest
from pydantic import ValidationError
import litellm
from litellm import Router
from litellm.router_utils.client_initalization_utils import DeploymentSemaphore
from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
def _semaphore(queue_size: int | None, max_parallel_requests: int = 1) -> DeploymentSemaphore:
return DeploymentSemaphore(
max_parallel_requests=max_parallel_requests,
model_id="deployment-1",
model_group="gpt-5.6",
queue_size=queue_size,
def _limit(max_parallel_requests: int = 1) -> MaxParallelRequestsLimit:
return MaxParallelRequestsLimit(
max_parallel_requests=max_parallel_requests, model_id="deployment-1", model_group="gpt-5.6"
)
async def _hold(semaphore: DeploymentSemaphore, release: asyncio.Event) -> str:
async with semaphore:
async def _hold(limit: MaxParallelRequestsLimit, release: asyncio.Event) -> str:
with limit:
await release.wait()
return "ok"
async def _expect_rejection(semaphore: DeploymentSemaphore) -> litellm.RateLimitError:
def _expect_rejection(limit: MaxParallelRequestsLimit) -> litellm.RateLimitError:
with pytest.raises(litellm.RateLimitError) as excinfo:
await asyncio.wait_for(semaphore.acquire(), timeout=1)
limit.acquire()
return excinfo.value
@pytest.mark.asyncio
async def test_queue_full_rejects_new_caller_while_queued_callers_still_complete():
semaphore: Final = _semaphore(queue_size=2)
async def test_request_arriving_while_every_slot_is_in_use_gets_429_without_waiting():
limit: Final = _limit(max_parallel_requests=2)
release: Final = asyncio.Event()
holder: Final = asyncio.create_task(_hold(semaphore, release))
holders: Final = [asyncio.create_task(_hold(limit, release)) for _ in range(2)]
await asyncio.sleep(0)
queued: Final = [asyncio.create_task(_hold(semaphore, release)) for _ in range(2)]
await asyncio.sleep(0)
assert semaphore.locked() and semaphore.waiting == 2
assert limit.in_flight == 2
rejection: Final = await _expect_rejection(semaphore)
rejection: Final = _expect_rejection(limit)
assert rejection.status_code == 429
assert "deployment-1" in rejection.message
assert "gpt-5.6" in rejection.message
assert "max_parallel_requests=1" in rejection.message
assert "max_parallel_requests_queue_size=2" in rejection.message
assert semaphore.waiting == 2
release.set()
assert await asyncio.wait_for(asyncio.gather(holder, *queued), timeout=2) == ["ok", "ok", "ok"]
assert semaphore.waiting == 0
assert not semaphore.locked()
@pytest.mark.asyncio
async def test_zero_queue_size_rejects_as_soon_as_every_slot_is_busy():
semaphore: Final = _semaphore(queue_size=0, max_parallel_requests=2)
release: Final = asyncio.Event()
holders: Final = [asyncio.create_task(_hold(semaphore, release)) for _ in range(2)]
await asyncio.sleep(0)
await _expect_rejection(semaphore)
assert semaphore.waiting == 0
assert "max_parallel_requests=2" in rejection.message
assert limit.in_flight == 2
release.set()
assert await asyncio.wait_for(asyncio.gather(*holders), timeout=2) == ["ok", "ok"]
assert limit.in_flight == 0
with limit:
assert limit.in_flight == 1
assert limit.in_flight == 0
@pytest.mark.asyncio
async def test_unset_queue_size_parks_every_caller_until_a_slot_frees():
semaphore: Final = _semaphore(queue_size=None)
async def test_burst_over_the_cap_admits_exactly_max_parallel_requests_and_rejects_the_rest():
limit: Final = _limit(max_parallel_requests=3)
release: Final = asyncio.Event()
callers: Final = [asyncio.create_task(_hold(semaphore, release)) for _ in range(50)]
await asyncio.sleep(0)
assert semaphore.waiting == 49
async def attempt() -> str:
try:
return await _hold(limit, release)
except litellm.RateLimitError as e:
return f"rejected:{e.status_code}"
callers: Final = [asyncio.create_task(attempt()) for _ in range(10)]
await asyncio.sleep(0)
assert limit.in_flight == 3
release.set()
assert await asyncio.wait_for(asyncio.gather(*callers), timeout=2) == ["ok"] * 50
assert semaphore.waiting == 0
outcomes: Final = await asyncio.wait_for(asyncio.gather(*callers), timeout=2)
assert outcomes.count("ok") == 3
assert outcomes.count("rejected:429") == 7
assert limit.in_flight == 0
@pytest.mark.asyncio
async def test_cancelled_waiter_gives_its_queue_slot_back():
semaphore: Final = _semaphore(queue_size=1)
release: Final = asyncio.Event()
holder: Final = asyncio.create_task(_hold(semaphore, release))
await asyncio.sleep(0)
cancelled: Final = asyncio.create_task(_hold(semaphore, release))
await asyncio.sleep(0)
assert semaphore.waiting == 1
cancelled.cancel()
with pytest.raises(asyncio.CancelledError):
await cancelled
assert semaphore.waiting == 0
replacement: Final = asyncio.create_task(_hold(semaphore, release))
await asyncio.sleep(0)
assert semaphore.waiting == 1
release.set()
assert await asyncio.wait_for(asyncio.gather(holder, replacement), timeout=2) == ["ok", "ok"]
def test_slot_is_released_when_the_held_call_raises():
limit: Final = _limit()
with pytest.raises(RuntimeError):
with limit:
raise RuntimeError("provider blew up")
assert limit.in_flight == 0
with limit:
assert limit.in_flight == 1
def _router_semaphore(router: Router, model_name: str) -> DeploymentSemaphore:
def _router_limit(router: Router, model_name: str) -> MaxParallelRequestsLimit:
deployment: Final = router.get_deployment_by_model_group_name(model_group_name=model_name)
assert deployment is not None
client: Final = router._get_client(deployment=deployment.model_dump(), kwargs={}, client_type="max_parallel_requests")
assert isinstance(client, DeploymentSemaphore)
client: Final = router._get_client(
deployment=deployment.model_dump(), kwargs={}, client_type="max_parallel_requests"
)
assert isinstance(client, MaxParallelRequestsLimit)
return client
@pytest.mark.parametrize("invalid_queue_size", [-1, 2.5, True, "3"])
def test_invalid_queue_sizes_are_rejected_instead_of_coerced(invalid_queue_size: object):
"""A negative bound would reject every busy request and a fraction would be truncated, so
neither may reach a semaphore, the router default, or a live update of that default."""
with pytest.raises(ValidationError):
_semaphore(queue_size=invalid_queue_size)
model_list: Final = [{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6", "rpm": 1}}]
with pytest.raises(ValidationError):
Router(model_list=model_list, default_max_parallel_requests_queue_size=invalid_queue_size)
with pytest.raises(ValidationError):
Router(
model_list=[
{
"model_name": "gpt-5.6",
"litellm_params": {
"model": "openai/gpt-5.6",
"rpm": 1,
"max_parallel_requests_queue_size": invalid_queue_size,
},
}
]
)
router: Final = Router(model_list=model_list, default_max_parallel_requests_queue_size=4)
semaphore: Final = _router_semaphore(router, "gpt-5.6")
with pytest.raises(ValidationError):
router.update_settings(default_max_parallel_requests_queue_size=invalid_queue_size)
assert router.default_max_parallel_requests_queue_size == 4
assert semaphore.queue_size == 4
@pytest.mark.parametrize(
("litellm_params", "expected_cap"),
[
({"max_parallel_requests": 2, "rpm": 7, "tpm": 100_000}, 2),
({"rpm": 7, "tpm": 100_000}, 7),
({"tpm": 100_000}, 600),
({"tpm": 100}, 1),
],
)
@pytest.mark.asyncio
async def test_deployment_queue_size_overrides_router_default_and_zero_is_honored():
async def test_router_deployment_rejects_past_its_derived_cap(litellm_params: dict[str, int], expected_cap: int):
router: Final = Router(
model_list=[
{"model_name": "inherits-default", "litellm_params": {"model": "openai/gpt-5.6", "rpm": 1}},
{
"model_name": "no-queue",
"litellm_params": {"model": "openai/gpt-5.6", "tpm": 100, "max_parallel_requests_queue_size": 0},
},
],
default_max_parallel_requests_queue_size=1,
model_list=[{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6", **litellm_params}}]
)
limit: Final = _router_limit(router, "gpt-5.6")
assert limit.max_parallel_requests == expected_cap
release: Final = asyncio.Event()
inherits: Final = _router_semaphore(router, "inherits-default")
inherits_holder: Final = asyncio.create_task(_hold(inherits, release))
holders: Final = [asyncio.create_task(_hold(limit, release)) for _ in range(expected_cap)]
await asyncio.sleep(0)
inherits_waiter: Final = asyncio.create_task(_hold(inherits, release))
await asyncio.sleep(0)
assert "max_parallel_requests_queue_size=1" in (await _expect_rejection(inherits)).message
no_queue: Final = _router_semaphore(router, "no-queue")
no_queue_holder: Final = asyncio.create_task(_hold(no_queue, release))
await asyncio.sleep(0)
assert "max_parallel_requests_queue_size=0" in (await _expect_rejection(no_queue)).message
assert limit.in_flight == expected_cap
assert f"max_parallel_requests={expected_cap}" in _expect_rejection(limit).message
release.set()
await asyncio.wait_for(asyncio.gather(inherits_holder, inherits_waiter, no_queue_holder), timeout=2)
assert await asyncio.wait_for(asyncio.gather(*holders), timeout=2) == ["ok"] * expected_cap
@pytest.mark.asyncio
async def test_router_without_queue_size_keeps_unbounded_queueing():
router: Final = Router(
model_list=[{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6", "max_parallel_requests": 1}}]
def test_router_without_any_concurrency_setting_has_no_limit():
router: Final = Router(model_list=[{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6"}}])
deployment: Final = router.get_deployment_by_model_group_name(model_group_name="gpt-5.6")
assert deployment is not None
assert (
router._get_client(deployment=deployment.model_dump(), kwargs={}, client_type="max_parallel_requests") is None
)
semaphore: Final = _router_semaphore(router, "gpt-5.6")
release: Final = asyncio.Event()
callers: Final = [asyncio.create_task(_hold(semaphore, release)) for _ in range(20)]
await asyncio.sleep(0)
assert semaphore.waiting == 19
release.set()
assert await asyncio.wait_for(asyncio.gather(*callers), timeout=2) == ["ok"] * 20
@pytest.mark.asyncio
async def test_update_settings_applies_default_queue_size_to_live_semaphores_without_an_override():
router: Final = Router(
model_list=[
{"model_name": "inherits-default", "litellm_params": {"model": "openai/gpt-5.6", "rpm": 1}},
{
"model_name": "pinned",
"litellm_params": {"model": "openai/gpt-5.6", "rpm": 1, "max_parallel_requests_queue_size": 5},
},
],
)
inherits: Final = _router_semaphore(router, "inherits-default")
pinned: Final = _router_semaphore(router, "pinned")
assert router.get_settings()["default_max_parallel_requests_queue_size"] is None
router.update_settings(default_max_parallel_requests_queue_size=0)
assert router.get_settings()["default_max_parallel_requests_queue_size"] == 0
assert (inherits.queue_size, pinned.queue_size) == (0, 5)
release: Final = asyncio.Event()
holder: Final = asyncio.create_task(_hold(inherits, release))
await asyncio.sleep(0)
assert "max_parallel_requests_queue_size=0" in (await _expect_rejection(inherits)).message
router.update_settings(default_max_parallel_requests_queue_size=None)
assert (inherits.queue_size, pinned.queue_size) == (None, 5)
waiter: Final = asyncio.create_task(_hold(inherits, release))
await asyncio.sleep(0)
assert inherits.waiting == 1
release.set()
assert await asyncio.wait_for(asyncio.gather(holder, waiter), timeout=2) == ["ok", "ok"]

87
tests/test_litellm/test_router.py
View file

@ -47,7 +47,7 @@ from litellm.router import (
_is_retriable_anthropic_status,
)
from litellm.router_strategy import simple_shuffle
from litellm.router_utils.client_initalization_utils import DeploymentSemaphore
from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit
from litellm.router_utils.cooldown_handlers import _async_get_cooldown_deployments
from litellm.types.llms.openai import ChatCompletionRequest
from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo, PreRoutingHookResponse, RetryPolicy
@ -1521,8 +1521,8 @@ async def test_router_ageneric_api_call_with_fallbacks_helper():
},
}
mock_semaphore = DeploymentSemaphore(
max_parallel_requests=1, model_id="deployment-1", model_group="gpt-3.5-turbo", queue_size=None
mock_semaphore = MaxParallelRequestsLimit(
max_parallel_requests=1, model_id="deployment-1", model_group="gpt-3.5-turbo"
)
with patch.object(
@ -15968,7 +15968,7 @@ def _max_parallel_router(max_parallel_requests: int) -> Router:
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_router_max_parallel_requests_bounds_in_flight_upstream_calls(
async def test_router_max_parallel_requests_admits_the_cap_and_rejects_the_rest_with_429(
monkeypatch: pytest.MonkeyPatch, stream: bool
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
@ -15994,24 +15994,33 @@ async def test_router_max_parallel_requests_bounds_in_flight_upstream_calls(
},
)
async def one_call() -> None:
response = await router.acompletion(
model="gpt-5.6", messages=[{"role": "user", "content": "hi"}], stream=stream
)
async def one_call() -> str:
try:
response = await router.acompletion(
model="gpt-5.6", messages=[{"role": "user", "content": "hi"}], stream=stream
)
except litellm.RateLimitError as e:
return f"rejected:{e.status_code}"
if stream:
async for _ in response:
pass
return "ok"
with respx.mock(assert_all_called=True) as respx_mock:
respx_mock.post("https://max-parallel.local/v1/chat/completions").mock(side_effect=upstream)
await asyncio.wait_for(asyncio.gather(*(one_call() for _ in range(10))), timeout=10)
route: Final = respx_mock.post("https://max-parallel.local/v1/chat/completions").mock(side_effect=upstream)
outcomes: Final = await asyncio.wait_for(asyncio.gather(*(one_call() for _ in range(10))), timeout=10)
assert tracker.peak <= 2
assert outcomes.count("ok") == 2
assert outcomes.count("rejected:429") == 8
assert route.call_count == 2
assert tracker.peak == 2
assert tracker.current == 0
@pytest.mark.asyncio
async def test_router_max_parallel_requests_slot_released_when_stream_closed_early(monkeypatch: pytest.MonkeyPatch):
async def test_router_max_parallel_requests_slot_held_until_stream_closed_then_released(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
tracker: Final = _InFlightTracker()
router: Final = _max_parallel_router(max_parallel_requests=1)
@ -16034,18 +16043,21 @@ async def test_router_max_parallel_requests_slot_released_when_stream_closed_ear
async for _ in second:
pass
second_task: Final = asyncio.create_task(second_call())
await asyncio.sleep(0.05)
assert tracker.current == 1
with pytest.raises(litellm.RateLimitError) as while_streaming:
await second_call()
assert while_streaming.value.status_code == 429
await first.aclose()
await asyncio.wait_for(second_task, timeout=2)
await asyncio.wait_for(second_call(), timeout=2)
assert tracker.peak == 1
assert tracker.current == 0
@pytest.mark.asyncio
async def test_router_max_parallel_requests_queue_size_turns_overflow_into_429(monkeypatch: pytest.MonkeyPatch):
async def test_router_max_parallel_requests_overflow_is_429_without_cooldown_or_provider_call(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router: Final = Router(
model_list=[
@ -16056,9 +16068,8 @@ async def test_router_max_parallel_requests_queue_size_turns_overflow_into_429(m
"api_key": "sk-fake",
"api_base": "https://max-parallel.local/v1",
"max_parallel_requests": 1,
"max_parallel_requests_queue_size": 1,
},
"model_info": {"id": "queue-bounded-deployment"},
"model_info": {"id": "capped-deployment"},
},
{
"model_name": "gpt-5.6",
@ -16067,7 +16078,7 @@ async def test_router_max_parallel_requests_queue_size_turns_overflow_into_429(m
"api_key": "sk-fake",
"api_base": "https://max-parallel-sibling.local/v1",
},
"model_info": {"id": "queue-sibling-deployment"},
"model_info": {"id": "sibling-deployment"},
},
],
num_retries=0,
@ -16094,7 +16105,7 @@ async def test_router_max_parallel_requests_queue_size_turns_overflow_into_429(m
results: Final = await asyncio.wait_for(
asyncio.gather(
*(
router.acompletion(model="queue-bounded-deployment", messages=[{"role": "user", "content": "hi"}])
router.acompletion(model="capped-deployment", messages=[{"role": "user", "content": "hi"}])
for _ in range(3)
),
return_exceptions=True,
@ -16103,19 +16114,18 @@ async def test_router_max_parallel_requests_queue_size_turns_overflow_into_429(m
)
rejected: Final = [r for r in results if isinstance(r, BaseException)]
assert len(rejected) == 1 and len(results) == 3
assert isinstance(rejected[0], litellm.RateLimitError)
assert rejected[0].status_code == 429
assert "queue-bounded-deployment" in rejected[0].message
assert "max_parallel_requests_queue_size=1" in rejected[0].message
assert route.call_count == 2
assert len(rejected) == 2 and len(results) == 3
assert all(isinstance(r, litellm.RateLimitError) and r.status_code == 429 for r in rejected)
assert all("capped-deployment" in r.message and "max_parallel_requests=1" in r.message for r in rejected)
assert route.call_count == 1
assert sibling_route.call_count == 0
assert all("max_parallel_requests_queue_size" not in call.request.content.decode() for call in route.calls)
assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == []
@pytest.mark.asyncio
async def test_router_embedding_path_honors_max_parallel_requests_queue_size(monkeypatch: pytest.MonkeyPatch):
async def test_router_embedding_path_rejects_past_max_parallel_requests_without_orphan_coroutines(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router: Final = Router(
model_list=[
@ -16127,10 +16137,9 @@ async def test_router_embedding_path_honors_max_parallel_requests_queue_size(mon
"api_base": "https://max-parallel-embed.local/v1",
"max_parallel_requests": 1,
},
"model_info": {"id": "embed-bounded-deployment"},
"model_info": {"id": "embed-capped-deployment"},
}
],
default_max_parallel_requests_queue_size=1,
num_retries=0,
)
@ -16159,15 +16168,15 @@ async def test_router_embedding_path_honors_max_parallel_requests_queue_size(mon
gc.collect()
rejected: Final = [r for r in results if isinstance(r, BaseException)]
assert len(rejected) == 1 and len(results) == 3
assert isinstance(rejected[0], litellm.RateLimitError) and rejected[0].status_code == 429
assert "embed-bounded-deployment" in rejected[0].message
assert route.call_count == 2
assert len(rejected) == 2 and len(results) == 3
assert all(isinstance(r, litellm.RateLimitError) and r.status_code == 429 for r in rejected)
assert all("embed-capped-deployment" in r.message for r in rejected)
assert route.call_count == 1
assert [str(w.message) for w in caught if "never awaited" in str(w.message)] == []
@pytest.mark.asyncio
async def test_router_max_parallel_requests_queue_overflow_takes_the_ordinary_429_fallback_path(
async def test_router_max_parallel_requests_overflow_takes_the_ordinary_429_fallback_path(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
@ -16180,9 +16189,8 @@ async def test_router_max_parallel_requests_queue_overflow_takes_the_ordinary_42
"api_key": "sk-fake",
"api_base": "https://max-parallel-primary.local/v1",
"max_parallel_requests": 1,
"max_parallel_requests_queue_size": 0,
},
"model_info": {"id": "queue-primary-deployment"},
"model_info": {"id": "capped-primary-deployment"},
},
{
"model_name": "gpt-5.6-fallback",
@ -16191,7 +16199,7 @@ async def test_router_max_parallel_requests_queue_overflow_takes_the_ordinary_42
"api_key": "sk-fake",
"api_base": "https://max-parallel-fallback.local/v1",
},
"model_info": {"id": "queue-fallback-deployment"},
"model_info": {"id": "fallback-deployment"},
},
],
fallbacks=[{"gpt-5.6": ["gpt-5.6-fallback"]}],
@ -16232,7 +16240,7 @@ async def test_router_max_parallel_requests_queue_overflow_takes_the_ordinary_42
@pytest.mark.asyncio
async def test_router_deployment_slot_rejects_once_queue_is_full_and_frees_slot_on_exit():
async def test_router_deployment_slot_rejects_while_held_and_frees_slot_on_exit():
router: Final = Router(
model_list=[
{
@ -16241,7 +16249,6 @@ async def test_router_deployment_slot_rejects_once_queue_is_full_and_frees_slot_
"model": "openai/gpt-5.6",
"api_key": "sk-fake",
"max_parallel_requests": 1,
"max_parallel_requests_queue_size": 0,
},
"model_info": {"id": "slot-deployment"},
}

35
ui/litellm-dashboard/src/components/router_settings/index.test.tsx
View file

@ -137,41 +137,6 @@ describe("RouterSettings", () => {
);
});
it("should save default_max_parallel_requests_queue_size as a number and an empty field as null", async () => {
vi.mocked(getCallbacksCall).mockResolvedValue({
router_settings: { ...mockCallbacksResponse.router_settings, default_max_parallel_requests_queue_size: null },
});
const user = userEvent.setup();
renderWithProviders(<RouterSettings {...defaultProps} />);
await findStrategySelect();
const queueSize = await screen.findByRole("textbox", { name: /default_max_parallel_requests_queue_size/i });
fireEvent.change(queueSize, { target: { value: "4" } });
await user.click(screen.getByRole("button", { name: /save changes/i }));
await waitFor(() =>
expect(setCallbacksCall).toHaveBeenLastCalledWith(
"test-token",
expect.objectContaining({
router_settings: expect.objectContaining({ default_max_parallel_requests_queue_size: 4 }),
}),
),
);
fireEvent.change(queueSize, { target: { value: "" } });
await user.click(screen.getByRole("button", { name: /save changes/i }));
await waitFor(() =>
expect(setCallbacksCall).toHaveBeenLastCalledWith(
"test-token",
expect.objectContaining({
router_settings: expect.objectContaining({ default_max_parallel_requests_queue_size: null }),
}),
),
);
});
it("should show a success notification after saving", async () => {
const user = userEvent.setup();
renderWithProviders(<RouterSettings {...defaultProps} />);

11
ui/litellm-dashboard/src/components/router_settings/index.tsx
View file

@ -86,15 +86,7 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
const router_settings = formValue.routerSettings;
const numberKeys = new Set([
"allowed_fails",
"cooldown_time",
"num_retries",
"timeout",
"retry_after",
"default_max_parallel_requests_queue_size",
]);
const unsettableNumberKeys = new Set(["default_max_parallel_requests_queue_size"]);
const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]);
const jsonKeys = new Set(["model_group_alias"]);
// retry_policy and model_group_retry_policy are owned by the Model Retry Settings tab;
// routing_groups is owned by the Routing Groups tab. This page must not read or write them.
@ -108,7 +100,6 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
if (v.toLowerCase() === "null") return null;
if (numberKeys.has(key)) {
if (v === "" && unsettableNumberKeys.has(key)) return null;
const n = Number(v);
return Number.isNaN(n) ? fallback : n;
}

4
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -30441,8 +30441,6 @@ export interface components {
max_budget?: number | null;
/** Max File Size Mb */
max_file_size_mb?: number | null;
/** Max Parallel Requests Queue Size */
max_parallel_requests_queue_size?: number | null;
/** Max Retries */
max_retries?: number | null;
/**
@ -40895,8 +40893,6 @@ export interface components {
max_budget?: number | null;
/** Max File Size Mb */
max_file_size_mb?: number | null;
/** Max Parallel Requests Queue Size */
max_parallel_requests_queue_size?: number | null;
/** Max Retries */
max_retries?: number | null;
/**