diff --git a/litellm/constants.py b/litellm/constants.py index 0cd59706015..8409a161800 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -30,10 +30,8 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "enable_tag_filtering", "tag_routing_prefix", "optional_pre_call_checks", - "default_max_parallel_requests_queue_size", } ) -NULLABLE_RUNTIME_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset({"default_max_parallel_requests_queue_size"}) ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset( { "model_list", diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index a0ce5bf0360..e69a02bd93a 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -50,7 +50,6 @@ _DEPLOYMENT_OPTIONS: Final = frozenset( "max_retries", "num_retries", "max_parallel_requests", - "max_parallel_requests_queue_size", "input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 216a146143d..7bc36e175c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -70,7 +70,6 @@ from litellm.constants import ( LITELLM_SETTINGS_SAFE_DB_OVERRIDES, LITELLM_UI_ALLOW_HEADERS, LITELLM_UI_SESSION_DURATION, - NULLABLE_RUNTIME_ROUTER_SETTINGS, RUNTIME_UPDATABLE_ROUTER_SETTINGS, ) from litellm.litellm_core_utils.asyncify import asyncify @@ -776,7 +775,6 @@ from litellm.types.router import ( RoutingPlugin, SearchToolTypedDict, updateDeployment, - validate_max_parallel_requests_queue_size, ) from litellm.types.router import ModelInfo as RouterModelInfo from litellm.types.scheduler import DefaultPriorities @@ -6902,20 +6900,13 @@ class ProxyConfig: ): from litellm.utils import _update_dictionary - db_settings: Final = db_router_settings.param_value db_overlay_deferring_empty_lists_to_config: Final = { k: v - for k, v in db_settings.items() + for k, v in db_router_settings.param_value.items() if not (k in config_router_settings and isinstance(v, list) and len(v) == 0) } - cleared_nullable_settings: Final = MappingProxyType( - {k: None for k in NULLABLE_RUNTIME_ROUTER_SETTINGS if k in db_settings and db_settings[k] is None} - ) - combined_router_settings = MappingProxyType( - { - **_update_dictionary(config_router_settings, db_overlay_deferring_empty_lists_to_config), - **cleared_nullable_settings, - } + combined_router_settings = _update_dictionary( + config_router_settings, db_overlay_deferring_empty_lists_to_config ) elif config_router_settings is not None and isinstance(config_router_settings, dict): combined_router_settings = config_router_settings @@ -16937,17 +16928,6 @@ async def update_config( ) }, ) - raw_queue_size: Final = raw_router_settings.get("default_max_parallel_requests_queue_size") - try: - validate_max_parallel_requests_queue_size(raw_queue_size) - except ValueError as invalid_queue_size: - raise HTTPException( - status_code=400, - detail=( - f"default_max_parallel_requests_queue_size={raw_queue_size!r} is not valid, " - "it must be a non-negative integer or null" - ), - ) from invalid_queue_size if prisma_client is None: raise Exception("No DB Connected") @@ -17059,7 +17039,7 @@ async def update_config( raw_router_settings_without_none: Final = { key: value for key, value in raw_router_settings.items() - if key not in typed_router_settings and (value is not None or key in NULLABLE_RUNTIME_ROUTER_SETTINGS) + if key not in typed_router_settings and value is not None } router_settings_updates: Final = {**typed_router_settings, **raw_router_settings_without_none} new_router_settings: Final = {**existing, **router_settings_updates} diff --git a/litellm/router.py b/litellm/router.py index 97325e6c450..d645fe0fab8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -148,7 +148,7 @@ from litellm.router_utils.batch_utils import ( replace_model_in_jsonl, should_replace_model_in_jsonl, ) -from litellm.router_utils.client_initalization_utils import DeploymentSemaphore, InitalizeCachedClient +from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit from litellm.router_utils.clientside_credential_handler import ( get_dynamic_litellm_params, is_clientside_credential, @@ -263,7 +263,6 @@ from litellm.types.router import ( RoutingStrategy, SearchToolTypedDict, TaggedPreRoutingStrategy, - validate_max_parallel_requests_queue_size, ) from litellm.types.services import ServiceTypes from litellm.types.utils import ( @@ -739,7 +738,6 @@ class Router: stream_timeout: float | None = None, default_litellm_params: dict | None = None, # default params for Router.chat.completion.create default_max_parallel_requests: int | None = None, - default_max_parallel_requests_queue_size: int | None = None, set_verbose: bool = False, debug_level: Literal["DEBUG", "INFO"] = "INFO", default_fallbacks: list[str] | None = None, # generic fallbacks, works across all deployments @@ -937,9 +935,6 @@ class Router: None # use this to track the users default deployment, when they want to use model = * ) self.default_max_parallel_requests = default_max_parallel_requests - self._default_max_parallel_requests_queue_size = validate_max_parallel_requests_queue_size( - default_max_parallel_requests_queue_size - ) self.provider_default_deployment_ids: list[str] = [] self.pattern_router = PatternMatchRouter() self.team_pattern_routers: dict[str, PatternMatchRouter] = {} # {"TEAM_ID": PatternMatchRouter} @@ -3637,14 +3632,14 @@ class Router: logging_obj: Final[LiteLLMLogging | None] = kwargs.get("litellm_logging_obj", None) - rpm_semaphore: Final = self._get_client( + max_parallel_requests_limit: Final = self._get_client( deployment=deployment, kwargs=kwargs, client_type="max_parallel_requests", ) async with contextlib.AsyncExitStack() as deployment_slot: - if isinstance(rpm_semaphore, DeploymentSemaphore): - await deployment_slot.enter_async_context(rpm_semaphore) + if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit): + deployment_slot.enter_context(max_parallel_requests_limit) await self.async_routing_strategy_pre_call_checks( deployment=deployment, logging_obj=logging_obj, @@ -8509,14 +8504,14 @@ class Router: ) -> AsyncGenerator[None, None]: """Holds the deployment's max_parallel_requests slot, if it has one, around the provider call. Routing strategy pre-call checks run inside the slot so their rpm accounting stays concurrency-safe.""" - rpm_semaphore: Final = self._get_client( + max_parallel_requests_limit: Final = self._get_client( deployment=deployment, kwargs=kwargs, client_type="max_parallel_requests", ) async with contextlib.AsyncExitStack() as slot: - if isinstance(rpm_semaphore, DeploymentSemaphore): - await slot.enter_async_context(rpm_semaphore) + if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit): + slot.enter_context(max_parallel_requests_limit) await self.async_routing_strategy_pre_call_checks(deployment=deployment, parent_otel_span=parent_otel_span) yield @@ -11846,20 +11841,8 @@ class Router: _settings_to_return[var] = self.lowestlatency_logger.routing_args.json() _settings_to_return["routing_groups"] = [group.model_dump() for group in self._routing_groups.values()] - _settings_to_return["default_max_parallel_requests_queue_size"] = self.default_max_parallel_requests_queue_size return _settings_to_return - @property - def default_max_parallel_requests_queue_size(self) -> int | None: - return self._default_max_parallel_requests_queue_size - - @default_max_parallel_requests_queue_size.setter - def default_max_parallel_requests_queue_size(self, queue_size: int | None) -> None: - self._default_max_parallel_requests_queue_size = validate_max_parallel_requests_queue_size(queue_size) - InitalizeCachedClient.apply_default_max_parallel_requests_queue_size( - litellm_router_instance=self, queue_size=self._default_max_parallel_requests_queue_size - ) - def update_settings(self, **kwargs): """ Update the router settings. diff --git a/litellm/router_utils/client_initalization_utils.py b/litellm/router_utils/client_initalization_utils.py index be5f71a4e70..55b4c071cb0 100644 --- a/litellm/router_utils/client_initalization_utils.py +++ b/litellm/router_utils/client_initalization_utils.py @@ -1,11 +1,8 @@ -import asyncio -import time from types import TracebackType from typing import TYPE_CHECKING, Any, Final -from litellm._logging import verbose_router_logger from litellm.exceptions import RateLimitError, RateLimitErrorCategory, RateLimitType -from litellm.types.router import RouterErrors, validate_max_parallel_requests_queue_size +from litellm.types.router import RouterErrors from litellm.utils import calculate_max_parallel_requests if TYPE_CHECKING: @@ -16,71 +13,41 @@ else: LitellmRouter = Any -class DeploymentSemaphore: - """A deployment's max_parallel_requests slots. ``queue_size=None`` parks callers without bound, like a plain - ``asyncio.Semaphore``; otherwise a caller arriving while all slots are busy and ``queue_size`` callers already - wait gets a 429 instead of being parked.""" +class MaxParallelRequestsLimit: + """A deployment's max_parallel_requests slots. A caller arriving while every slot is in use gets a 429 instead + of waiting for one to free up.""" - def __init__(self, max_parallel_requests: int, model_id: str, model_group: str, queue_size: int | None) -> None: - self._slots: Final = asyncio.Semaphore(max_parallel_requests) + def __init__(self, max_parallel_requests: int, model_id: str, model_group: str) -> None: self.max_parallel_requests: Final = max_parallel_requests self.model_id: Final = model_id self.model_group: Final = model_group - self.queue_size = validate_max_parallel_requests_queue_size(queue_size) - self.waiting = 0 + self.in_flight = 0 - def locked(self) -> bool: - return self._slots.locked() + def __enter__(self) -> None: + self.acquire() - def release(self) -> None: - self._slots.release() - - async def __aenter__(self) -> None: - await self.acquire() - - async def __aexit__( + def __exit__( self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None ) -> None: - self._slots.release() + self.release() - async def acquire(self) -> bool: - if not self._slots.locked(): - return await self._slots.acquire() - if self.queue_size is not None and self.waiting >= self.queue_size: + def acquire(self) -> None: + if self.in_flight >= self.max_parallel_requests: raise RateLimitError( message=( - f"{RouterErrors.max_parallel_requests_queue_full.value} Deployment model_group={self.model_group}, " - f"id={self.model_id} has all max_parallel_requests={self.max_parallel_requests} slots in use and " - f"{self.waiting} requests already waiting, which is its max_parallel_requests_queue_size=" - f"{self.queue_size}. Raise max_parallel_requests or max_parallel_requests_queue_size for this " - "deployment, or unset max_parallel_requests_queue_size to queue without a bound" + f"{RouterErrors.max_parallel_requests_exceeded.value} Deployment model_group={self.model_group}, " + f"id={self.model_id} already has max_parallel_requests={self.max_parallel_requests} requests in " + "flight. Raise max_parallel_requests (or the rpm/tpm it is derived from) for this deployment" ), llm_provider="", model=self.model_group, category=RateLimitErrorCategory.LITELLM_RATE_LIMIT, rate_limit_type=RateLimitType.CONCURRENT_REQUESTS, ) - self.waiting += 1 - queued_at: Final = time.perf_counter() - verbose_router_logger.debug( - "Deployment model_group=%s, id=%s has all max_parallel_requests=%s slots in use, request queued " - "(waiting=%s, max_parallel_requests_queue_size=%s)", - self.model_group, - self.model_id, - self.max_parallel_requests, - self.waiting, - self.queue_size, - ) - try: - return await self._slots.acquire() - finally: - self.waiting -= 1 - verbose_router_logger.debug( - "Deployment model_group=%s, id=%s request left the max_parallel_requests queue after %.1f ms", - self.model_group, - self.model_id, - (time.perf_counter() - queued_at) * 1000, - ) + self.in_flight += 1 + + def release(self) -> None: + self.in_flight -= 1 class InitalizeCachedClient: @@ -98,35 +65,14 @@ class InitalizeCachedClient: default_max_parallel_requests=litellm_router_instance.default_max_parallel_requests, ) if calculated_max_parallel_requests: - deployment_queue_size: Final = litellm_params.get("max_parallel_requests_queue_size", None) - semaphore: Final = DeploymentSemaphore( + limit: Final = MaxParallelRequestsLimit( max_parallel_requests=calculated_max_parallel_requests, model_id=model_id, model_group=model.get("model_name", ""), - queue_size=( - deployment_queue_size - if deployment_queue_size is not None - else litellm_router_instance.default_max_parallel_requests_queue_size - ), ) cache_key: Final = f"{model_id}_max_parallel_requests_client" litellm_router_instance.cache.set_cache( key=cache_key, - value=semaphore, + value=limit, local_only=True, ) - - @staticmethod - def apply_default_max_parallel_requests_queue_size( - litellm_router_instance: LitellmRouter, queue_size: int | None - ) -> None: - inheriting_semaphores: Final = ( - litellm_router_instance.cache.get_cache( - key=f"{model['model_info']['id']}_max_parallel_requests_client", local_only=True - ) - for model in litellm_router_instance.model_list - if model["litellm_params"].get("max_parallel_requests_queue_size") is None - ) - for semaphore in inheriting_semaphores: - if isinstance(semaphore, DeploymentSemaphore): - semaphore.queue_size = queue_size diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py index fe715e45b2f..cef180b202a 100644 --- a/litellm/types/management_endpoints/router_settings_endpoints.py +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -244,17 +244,6 @@ ROUTER_SETTINGS_FIELDS: Final[list[RouterSettingsField]] = [ field_default=None, ui_field_name="Max Parallel Requests", ), - RouterSettingsField( - field_name="default_max_parallel_requests_queue_size", - field_type="Integer", - field_value=None, - field_description=( - "Default cap on how many requests may wait for a deployment's max_parallel_requests slot before " - "further requests get a 429. Unset queues without a bound" - ), - field_default=None, - ui_field_name="Max Parallel Requests Queue Size", - ), RouterSettingsField( field_name="enable_tag_filtering", field_type="Boolean", diff --git a/litellm/types/router.py b/litellm/types/router.py index 848dd28aaac..29f3c3681e0 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -6,10 +6,10 @@ import datetime import enum from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints +from typing import TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints import httpx -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger @@ -314,14 +314,6 @@ class CredentialLiteLLMParams(BaseModel): _RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"}) -MaxParallelRequestsQueueSize = Annotated[int, Field(strict=True, ge=0)] -_MAX_PARALLEL_REQUESTS_QUEUE_SIZE_ADAPTER: Final = TypeAdapter(MaxParallelRequestsQueueSize | None) - - -def validate_max_parallel_requests_queue_size(value: object) -> int | None: - return _MAX_PARALLEL_REQUESTS_QUEUE_SIZE_ADAPTER.validate_python(value) - - class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): """ LiteLLM Params without 'model' arg (used across completion / assistants api) @@ -332,7 +324,6 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): rpm: int | None = None itpm: int | None = None otpm: int | None = None - max_parallel_requests_queue_size: MaxParallelRequestsQueueSize | None = None timeout: float | str | httpx.Timeout | None = None # if str, pass in as os.environ/ stream_timeout: float | str | None = None # timeout when making stream=True calls, if str, pass in as os.environ/ max_retries: int | None = None @@ -506,7 +497,6 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): order: int | None weight: int | None max_parallel_requests: int | None - max_parallel_requests_queue_size: ReadOnly[MaxParallelRequestsQueueSize | None] api_key: str | None api_base: str | None api_version: str | None @@ -657,7 +647,7 @@ class RouterErrors(enum.Enum): """ user_defined_ratelimit_error = "Deployment over user-defined ratelimit." - max_parallel_requests_queue_full = "Deployment max_parallel_requests queue is full." + max_parallel_requests_exceeded = "Deployment has all max_parallel_requests slots in use." no_deployments_available = "No deployments available for selected model" all_deployments_in_cooldown = "All deployments for selected model are in cooldown" no_deployments_with_tag_routing = "Not allowed to access model due to tags configuration" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8f902f34548..aaa16fd2d44 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3841,7 +3841,6 @@ all_litellm_params = ( "itpm", "otpm", "max_parallel_requests", - "max_parallel_requests_queue_size", "input_cost_per_token", "output_cost_per_token", "input_cost_per_second", diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 582977d613b..a11f015743b 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -88,7 +88,6 @@ ignored_function_names = [ "_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py "_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py "_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name) - "default_max_parallel_requests_queue_size", ] diff --git a/tests/local_testing/test_router_max_parallel_requests.py b/tests/local_testing/test_router_max_parallel_requests.py index 65602c968bc..051c69c9322 100644 --- a/tests/local_testing/test_router_max_parallel_requests.py +++ b/tests/local_testing/test_router_max_parallel_requests.py @@ -11,6 +11,7 @@ import pytest from typing import Optional import litellm +from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit from litellm.utils import calculate_max_parallel_requests """ @@ -93,26 +94,26 @@ def test_setting_mpr_limits_per_model( default_max_parallel_requests=default_max_parallel_requests, ) - mpr_client: Optional[asyncio.Semaphore] = router._get_client( + mpr_client: Optional[MaxParallelRequestsLimit] = router._get_client( deployment=deployment, kwargs={}, client_type="max_parallel_requests", ) if max_parallel_requests is not None: - assert max_parallel_requests == mpr_client._value + assert max_parallel_requests == mpr_client.max_parallel_requests elif rpm is not None: - assert rpm == mpr_client._value + assert rpm == mpr_client.max_parallel_requests elif tpm is not None: calculated_rpm = int(tpm / 1000 * 6) if calculated_rpm == 0: calculated_rpm = 1 print( - f"test calculated_rpm: {calculated_rpm}, calculated_max_parallel_requests={mpr_client._value}" + f"test calculated_rpm: {calculated_rpm}, calculated_max_parallel_requests={mpr_client.max_parallel_requests}" ) - assert calculated_rpm == mpr_client._value + assert calculated_rpm == mpr_client.max_parallel_requests elif default_max_parallel_requests is not None: - assert mpr_client._value == default_max_parallel_requests + assert mpr_client.max_parallel_requests == default_max_parallel_requests else: assert mpr_client is None diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py index c13217a0d46..2b36866a1a0 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -181,15 +181,6 @@ async def test_environment_credential_matches_native_count_and_observed_scope( assert observed.scope == cache_scope(_CALLER, _DEPLOYMENT, target.api_key, target.model) -def test_deployment_concurrency_knobs_keep_native_prediction_supported() -> None: - target: Final = resolve_prediction_target(LiteLLM_Params( - model=f"anthropic/{_MODEL}", api_key=_KEY, api_base="https://api.anthropic.com", - max_parallel_requests=1, max_parallel_requests_queue_size=0, - )) - assert isinstance(target, NativePredictionTarget) - assert (target.model, target.api_key) == (_MODEL, _KEY) - - @pytest.mark.parametrize("inline_key", [None, _KEY]) @pytest.mark.asyncio async def test_named_credential_is_explicitly_unsupported_before_count( diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 5a2e76039e8..41c4956dba6 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -5051,39 +5051,6 @@ async def test_add_router_settings_from_db_config_empty_db_list_still_clears_unc assert combined_settings["num_retries"] == 1 -@pytest.mark.asyncio -async def test_add_router_settings_from_db_config_null_queue_size_reaches_router(): - """A cleared Admin UI field is stored as null. The reload must hand that None to the - router so a config.yaml bound is lifted, while an unrelated null still falls back to - the config value.""" - from unittest.mock import AsyncMock, MagicMock - - from litellm.proxy.proxy_server import ProxyConfig - - proxy_config = ProxyConfig() - mock_router = MagicMock() - mock_router.update_settings = MagicMock() - - config_data = {"router_settings": {"default_max_parallel_requests_queue_size": 2, "num_retries": 1}} - - mock_db_config = MagicMock() - mock_db_config.param_value = {"default_max_parallel_requests_queue_size": None, "num_retries": None} - - mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_config.find_first = AsyncMock(return_value=mock_db_config) - - await proxy_config._add_router_settings_from_db_config( - config_data=config_data, - llm_router=mock_router, - prisma_client=mock_prisma_client, - ) - - combined_settings = mock_router.update_settings.call_args.kwargs - assert "default_max_parallel_requests_queue_size" in combined_settings - assert combined_settings["default_max_parallel_requests_queue_size"] is None - assert combined_settings["num_retries"] == 1 - - @pytest.mark.asyncio async def test_add_router_settings_from_db_config_edge_cases(): """ @@ -9367,51 +9334,6 @@ def test_update_config_litellm_settings_request_wins_for_non_callback_keys( restore() -def test_update_config_router_settings_null_clears_max_parallel_requests_queue_size( - _update_config_setup, -): - """Clearing the Admin UI field sends null. The stored row must hold null so the - reload hands None to the router and queueing becomes unbounded again, while an - unrelated null is still dropped rather than persisted.""" - client, prisma, restore = _update_config_setup( - initial_rows={ - "router_settings": {"default_max_parallel_requests_queue_size": 3, "num_retries": 2}, - } - ) - try: - resp = client.post( - "/config/update", - json={"router_settings": {"default_max_parallel_requests_queue_size": None, "timeout": None}}, - ) - assert resp.status_code == 200 - stored = prisma.db.litellm_config.rows["router_settings"] - assert "default_max_parallel_requests_queue_size" in stored - assert stored["default_max_parallel_requests_queue_size"] is None - assert stored["num_retries"] == 2 - assert "timeout" not in stored - finally: - restore() - - -@pytest.mark.parametrize("invalid_queue_size", [-1, 2.5, "3"]) -def test_update_config_rejects_invalid_max_parallel_requests_queue_size_before_persisting( - _update_config_setup, invalid_queue_size -): - client, prisma, restore = _update_config_setup( - initial_rows={"router_settings": {"default_max_parallel_requests_queue_size": 3}}, - ) - try: - resp = client.post( - "/config/update", - json={"router_settings": {"default_max_parallel_requests_queue_size": invalid_queue_size}}, - ) - assert resp.status_code == 400 - assert "default_max_parallel_requests_queue_size" in resp.json()["error"]["message"] - assert prisma.db.litellm_config.rows["router_settings"] == {"default_max_parallel_requests_queue_size": 3} - finally: - restore() - - def test_update_config_success_callback_normalizes_existing_mixed_case( _update_config_setup, ): diff --git a/tests/test_litellm/router_utils/test_client_initalization_utils.py b/tests/test_litellm/router_utils/test_client_initalization_utils.py index a6626d2f975..6f9a7b730ac 100644 --- a/tests/test_litellm/router_utils/test_client_initalization_utils.py +++ b/tests/test_litellm/router_utils/test_client_initalization_utils.py @@ -2,220 +2,124 @@ import asyncio from typing import Final import pytest -from pydantic import ValidationError import litellm from litellm import Router -from litellm.router_utils.client_initalization_utils import DeploymentSemaphore +from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit -def _semaphore(queue_size: int | None, max_parallel_requests: int = 1) -> DeploymentSemaphore: - return DeploymentSemaphore( - max_parallel_requests=max_parallel_requests, - model_id="deployment-1", - model_group="gpt-5.6", - queue_size=queue_size, +def _limit(max_parallel_requests: int = 1) -> MaxParallelRequestsLimit: + return MaxParallelRequestsLimit( + max_parallel_requests=max_parallel_requests, model_id="deployment-1", model_group="gpt-5.6" ) -async def _hold(semaphore: DeploymentSemaphore, release: asyncio.Event) -> str: - async with semaphore: +async def _hold(limit: MaxParallelRequestsLimit, release: asyncio.Event) -> str: + with limit: await release.wait() return "ok" -async def _expect_rejection(semaphore: DeploymentSemaphore) -> litellm.RateLimitError: +def _expect_rejection(limit: MaxParallelRequestsLimit) -> litellm.RateLimitError: with pytest.raises(litellm.RateLimitError) as excinfo: - await asyncio.wait_for(semaphore.acquire(), timeout=1) + limit.acquire() return excinfo.value @pytest.mark.asyncio -async def test_queue_full_rejects_new_caller_while_queued_callers_still_complete(): - semaphore: Final = _semaphore(queue_size=2) +async def test_request_arriving_while_every_slot_is_in_use_gets_429_without_waiting(): + limit: Final = _limit(max_parallel_requests=2) release: Final = asyncio.Event() - holder: Final = asyncio.create_task(_hold(semaphore, release)) + holders: Final = [asyncio.create_task(_hold(limit, release)) for _ in range(2)] await asyncio.sleep(0) - queued: Final = [asyncio.create_task(_hold(semaphore, release)) for _ in range(2)] - await asyncio.sleep(0) - assert semaphore.locked() and semaphore.waiting == 2 + assert limit.in_flight == 2 - rejection: Final = await _expect_rejection(semaphore) + rejection: Final = _expect_rejection(limit) assert rejection.status_code == 429 assert "deployment-1" in rejection.message assert "gpt-5.6" in rejection.message - assert "max_parallel_requests=1" in rejection.message - assert "max_parallel_requests_queue_size=2" in rejection.message - assert semaphore.waiting == 2 - - release.set() - assert await asyncio.wait_for(asyncio.gather(holder, *queued), timeout=2) == ["ok", "ok", "ok"] - assert semaphore.waiting == 0 - assert not semaphore.locked() - - -@pytest.mark.asyncio -async def test_zero_queue_size_rejects_as_soon_as_every_slot_is_busy(): - semaphore: Final = _semaphore(queue_size=0, max_parallel_requests=2) - release: Final = asyncio.Event() - holders: Final = [asyncio.create_task(_hold(semaphore, release)) for _ in range(2)] - await asyncio.sleep(0) - - await _expect_rejection(semaphore) - assert semaphore.waiting == 0 + assert "max_parallel_requests=2" in rejection.message + assert limit.in_flight == 2 release.set() assert await asyncio.wait_for(asyncio.gather(*holders), timeout=2) == ["ok", "ok"] + assert limit.in_flight == 0 + with limit: + assert limit.in_flight == 1 + assert limit.in_flight == 0 @pytest.mark.asyncio -async def test_unset_queue_size_parks_every_caller_until_a_slot_frees(): - semaphore: Final = _semaphore(queue_size=None) +async def test_burst_over_the_cap_admits_exactly_max_parallel_requests_and_rejects_the_rest(): + limit: Final = _limit(max_parallel_requests=3) release: Final = asyncio.Event() - callers: Final = [asyncio.create_task(_hold(semaphore, release)) for _ in range(50)] - await asyncio.sleep(0) - assert semaphore.waiting == 49 + async def attempt() -> str: + try: + return await _hold(limit, release) + except litellm.RateLimitError as e: + return f"rejected:{e.status_code}" + + callers: Final = [asyncio.create_task(attempt()) for _ in range(10)] + await asyncio.sleep(0) + assert limit.in_flight == 3 release.set() - assert await asyncio.wait_for(asyncio.gather(*callers), timeout=2) == ["ok"] * 50 - assert semaphore.waiting == 0 + outcomes: Final = await asyncio.wait_for(asyncio.gather(*callers), timeout=2) + assert outcomes.count("ok") == 3 + assert outcomes.count("rejected:429") == 7 + assert limit.in_flight == 0 -@pytest.mark.asyncio -async def test_cancelled_waiter_gives_its_queue_slot_back(): - semaphore: Final = _semaphore(queue_size=1) - release: Final = asyncio.Event() - holder: Final = asyncio.create_task(_hold(semaphore, release)) - await asyncio.sleep(0) - cancelled: Final = asyncio.create_task(_hold(semaphore, release)) - await asyncio.sleep(0) - assert semaphore.waiting == 1 - - cancelled.cancel() - with pytest.raises(asyncio.CancelledError): - await cancelled - assert semaphore.waiting == 0 - - replacement: Final = asyncio.create_task(_hold(semaphore, release)) - await asyncio.sleep(0) - assert semaphore.waiting == 1 - release.set() - assert await asyncio.wait_for(asyncio.gather(holder, replacement), timeout=2) == ["ok", "ok"] +def test_slot_is_released_when_the_held_call_raises(): + limit: Final = _limit() + with pytest.raises(RuntimeError): + with limit: + raise RuntimeError("provider blew up") + assert limit.in_flight == 0 + with limit: + assert limit.in_flight == 1 -def _router_semaphore(router: Router, model_name: str) -> DeploymentSemaphore: +def _router_limit(router: Router, model_name: str) -> MaxParallelRequestsLimit: deployment: Final = router.get_deployment_by_model_group_name(model_group_name=model_name) assert deployment is not None - client: Final = router._get_client(deployment=deployment.model_dump(), kwargs={}, client_type="max_parallel_requests") - assert isinstance(client, DeploymentSemaphore) + client: Final = router._get_client( + deployment=deployment.model_dump(), kwargs={}, client_type="max_parallel_requests" + ) + assert isinstance(client, MaxParallelRequestsLimit) return client -@pytest.mark.parametrize("invalid_queue_size", [-1, 2.5, True, "3"]) -def test_invalid_queue_sizes_are_rejected_instead_of_coerced(invalid_queue_size: object): - """A negative bound would reject every busy request and a fraction would be truncated, so - neither may reach a semaphore, the router default, or a live update of that default.""" - with pytest.raises(ValidationError): - _semaphore(queue_size=invalid_queue_size) - model_list: Final = [{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6", "rpm": 1}}] - with pytest.raises(ValidationError): - Router(model_list=model_list, default_max_parallel_requests_queue_size=invalid_queue_size) - with pytest.raises(ValidationError): - Router( - model_list=[ - { - "model_name": "gpt-5.6", - "litellm_params": { - "model": "openai/gpt-5.6", - "rpm": 1, - "max_parallel_requests_queue_size": invalid_queue_size, - }, - } - ] - ) - - router: Final = Router(model_list=model_list, default_max_parallel_requests_queue_size=4) - semaphore: Final = _router_semaphore(router, "gpt-5.6") - with pytest.raises(ValidationError): - router.update_settings(default_max_parallel_requests_queue_size=invalid_queue_size) - assert router.default_max_parallel_requests_queue_size == 4 - assert semaphore.queue_size == 4 - - +@pytest.mark.parametrize( + ("litellm_params", "expected_cap"), + [ + ({"max_parallel_requests": 2, "rpm": 7, "tpm": 100_000}, 2), + ({"rpm": 7, "tpm": 100_000}, 7), + ({"tpm": 100_000}, 600), + ({"tpm": 100}, 1), + ], +) @pytest.mark.asyncio -async def test_deployment_queue_size_overrides_router_default_and_zero_is_honored(): +async def test_router_deployment_rejects_past_its_derived_cap(litellm_params: dict[str, int], expected_cap: int): router: Final = Router( - model_list=[ - {"model_name": "inherits-default", "litellm_params": {"model": "openai/gpt-5.6", "rpm": 1}}, - { - "model_name": "no-queue", - "litellm_params": {"model": "openai/gpt-5.6", "tpm": 100, "max_parallel_requests_queue_size": 0}, - }, - ], - default_max_parallel_requests_queue_size=1, + model_list=[{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6", **litellm_params}}] ) + limit: Final = _router_limit(router, "gpt-5.6") + assert limit.max_parallel_requests == expected_cap release: Final = asyncio.Event() - - inherits: Final = _router_semaphore(router, "inherits-default") - inherits_holder: Final = asyncio.create_task(_hold(inherits, release)) + holders: Final = [asyncio.create_task(_hold(limit, release)) for _ in range(expected_cap)] await asyncio.sleep(0) - inherits_waiter: Final = asyncio.create_task(_hold(inherits, release)) - await asyncio.sleep(0) - assert "max_parallel_requests_queue_size=1" in (await _expect_rejection(inherits)).message - - no_queue: Final = _router_semaphore(router, "no-queue") - no_queue_holder: Final = asyncio.create_task(_hold(no_queue, release)) - await asyncio.sleep(0) - assert "max_parallel_requests_queue_size=0" in (await _expect_rejection(no_queue)).message - + assert limit.in_flight == expected_cap + assert f"max_parallel_requests={expected_cap}" in _expect_rejection(limit).message release.set() - await asyncio.wait_for(asyncio.gather(inherits_holder, inherits_waiter, no_queue_holder), timeout=2) + assert await asyncio.wait_for(asyncio.gather(*holders), timeout=2) == ["ok"] * expected_cap -@pytest.mark.asyncio -async def test_router_without_queue_size_keeps_unbounded_queueing(): - router: Final = Router( - model_list=[{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6", "max_parallel_requests": 1}}] +def test_router_without_any_concurrency_setting_has_no_limit(): + router: Final = Router(model_list=[{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6"}}]) + deployment: Final = router.get_deployment_by_model_group_name(model_group_name="gpt-5.6") + assert deployment is not None + assert ( + router._get_client(deployment=deployment.model_dump(), kwargs={}, client_type="max_parallel_requests") is None ) - semaphore: Final = _router_semaphore(router, "gpt-5.6") - release: Final = asyncio.Event() - callers: Final = [asyncio.create_task(_hold(semaphore, release)) for _ in range(20)] - await asyncio.sleep(0) - assert semaphore.waiting == 19 - release.set() - assert await asyncio.wait_for(asyncio.gather(*callers), timeout=2) == ["ok"] * 20 - - -@pytest.mark.asyncio -async def test_update_settings_applies_default_queue_size_to_live_semaphores_without_an_override(): - router: Final = Router( - model_list=[ - {"model_name": "inherits-default", "litellm_params": {"model": "openai/gpt-5.6", "rpm": 1}}, - { - "model_name": "pinned", - "litellm_params": {"model": "openai/gpt-5.6", "rpm": 1, "max_parallel_requests_queue_size": 5}, - }, - ], - ) - inherits: Final = _router_semaphore(router, "inherits-default") - pinned: Final = _router_semaphore(router, "pinned") - assert router.get_settings()["default_max_parallel_requests_queue_size"] is None - - router.update_settings(default_max_parallel_requests_queue_size=0) - assert router.get_settings()["default_max_parallel_requests_queue_size"] == 0 - assert (inherits.queue_size, pinned.queue_size) == (0, 5) - - release: Final = asyncio.Event() - holder: Final = asyncio.create_task(_hold(inherits, release)) - await asyncio.sleep(0) - assert "max_parallel_requests_queue_size=0" in (await _expect_rejection(inherits)).message - - router.update_settings(default_max_parallel_requests_queue_size=None) - assert (inherits.queue_size, pinned.queue_size) == (None, 5) - waiter: Final = asyncio.create_task(_hold(inherits, release)) - await asyncio.sleep(0) - assert inherits.waiting == 1 - - release.set() - assert await asyncio.wait_for(asyncio.gather(holder, waiter), timeout=2) == ["ok", "ok"] diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index a674f767dde..26f9803a022 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -47,7 +47,7 @@ from litellm.router import ( _is_retriable_anthropic_status, ) from litellm.router_strategy import simple_shuffle -from litellm.router_utils.client_initalization_utils import DeploymentSemaphore +from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit from litellm.router_utils.cooldown_handlers import _async_get_cooldown_deployments from litellm.types.llms.openai import ChatCompletionRequest from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo, PreRoutingHookResponse, RetryPolicy @@ -1521,8 +1521,8 @@ async def test_router_ageneric_api_call_with_fallbacks_helper(): }, } - mock_semaphore = DeploymentSemaphore( - max_parallel_requests=1, model_id="deployment-1", model_group="gpt-3.5-turbo", queue_size=None + mock_semaphore = MaxParallelRequestsLimit( + max_parallel_requests=1, model_id="deployment-1", model_group="gpt-3.5-turbo" ) with patch.object( @@ -15968,7 +15968,7 @@ def _max_parallel_router(max_parallel_requests: int) -> Router: @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) -async def test_router_max_parallel_requests_bounds_in_flight_upstream_calls( +async def test_router_max_parallel_requests_admits_the_cap_and_rejects_the_rest_with_429( monkeypatch: pytest.MonkeyPatch, stream: bool ): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -15994,24 +15994,33 @@ async def test_router_max_parallel_requests_bounds_in_flight_upstream_calls( }, ) - async def one_call() -> None: - response = await router.acompletion( - model="gpt-5.6", messages=[{"role": "user", "content": "hi"}], stream=stream - ) + async def one_call() -> str: + try: + response = await router.acompletion( + model="gpt-5.6", messages=[{"role": "user", "content": "hi"}], stream=stream + ) + except litellm.RateLimitError as e: + return f"rejected:{e.status_code}" if stream: async for _ in response: pass + return "ok" with respx.mock(assert_all_called=True) as respx_mock: - respx_mock.post("https://max-parallel.local/v1/chat/completions").mock(side_effect=upstream) - await asyncio.wait_for(asyncio.gather(*(one_call() for _ in range(10))), timeout=10) + route: Final = respx_mock.post("https://max-parallel.local/v1/chat/completions").mock(side_effect=upstream) + outcomes: Final = await asyncio.wait_for(asyncio.gather(*(one_call() for _ in range(10))), timeout=10) - assert tracker.peak <= 2 + assert outcomes.count("ok") == 2 + assert outcomes.count("rejected:429") == 8 + assert route.call_count == 2 + assert tracker.peak == 2 assert tracker.current == 0 @pytest.mark.asyncio -async def test_router_max_parallel_requests_slot_released_when_stream_closed_early(monkeypatch: pytest.MonkeyPatch): +async def test_router_max_parallel_requests_slot_held_until_stream_closed_then_released( + monkeypatch: pytest.MonkeyPatch, +): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) tracker: Final = _InFlightTracker() router: Final = _max_parallel_router(max_parallel_requests=1) @@ -16034,18 +16043,21 @@ async def test_router_max_parallel_requests_slot_released_when_stream_closed_ear async for _ in second: pass - second_task: Final = asyncio.create_task(second_call()) - await asyncio.sleep(0.05) assert tracker.current == 1 + with pytest.raises(litellm.RateLimitError) as while_streaming: + await second_call() + assert while_streaming.value.status_code == 429 await first.aclose() - await asyncio.wait_for(second_task, timeout=2) + await asyncio.wait_for(second_call(), timeout=2) assert tracker.peak == 1 assert tracker.current == 0 @pytest.mark.asyncio -async def test_router_max_parallel_requests_queue_size_turns_overflow_into_429(monkeypatch: pytest.MonkeyPatch): +async def test_router_max_parallel_requests_overflow_is_429_without_cooldown_or_provider_call( + monkeypatch: pytest.MonkeyPatch, +): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) router: Final = Router( model_list=[ @@ -16056,9 +16068,8 @@ async def test_router_max_parallel_requests_queue_size_turns_overflow_into_429(m "api_key": "sk-fake", "api_base": "https://max-parallel.local/v1", "max_parallel_requests": 1, - "max_parallel_requests_queue_size": 1, }, - "model_info": {"id": "queue-bounded-deployment"}, + "model_info": {"id": "capped-deployment"}, }, { "model_name": "gpt-5.6", @@ -16067,7 +16078,7 @@ async def test_router_max_parallel_requests_queue_size_turns_overflow_into_429(m "api_key": "sk-fake", "api_base": "https://max-parallel-sibling.local/v1", }, - "model_info": {"id": "queue-sibling-deployment"}, + "model_info": {"id": "sibling-deployment"}, }, ], num_retries=0, @@ -16094,7 +16105,7 @@ async def test_router_max_parallel_requests_queue_size_turns_overflow_into_429(m results: Final = await asyncio.wait_for( asyncio.gather( *( - router.acompletion(model="queue-bounded-deployment", messages=[{"role": "user", "content": "hi"}]) + router.acompletion(model="capped-deployment", messages=[{"role": "user", "content": "hi"}]) for _ in range(3) ), return_exceptions=True, @@ -16103,19 +16114,18 @@ async def test_router_max_parallel_requests_queue_size_turns_overflow_into_429(m ) rejected: Final = [r for r in results if isinstance(r, BaseException)] - assert len(rejected) == 1 and len(results) == 3 - assert isinstance(rejected[0], litellm.RateLimitError) - assert rejected[0].status_code == 429 - assert "queue-bounded-deployment" in rejected[0].message - assert "max_parallel_requests_queue_size=1" in rejected[0].message - assert route.call_count == 2 + assert len(rejected) == 2 and len(results) == 3 + assert all(isinstance(r, litellm.RateLimitError) and r.status_code == 429 for r in rejected) + assert all("capped-deployment" in r.message and "max_parallel_requests=1" in r.message for r in rejected) + assert route.call_count == 1 assert sibling_route.call_count == 0 - assert all("max_parallel_requests_queue_size" not in call.request.content.decode() for call in route.calls) assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] @pytest.mark.asyncio -async def test_router_embedding_path_honors_max_parallel_requests_queue_size(monkeypatch: pytest.MonkeyPatch): +async def test_router_embedding_path_rejects_past_max_parallel_requests_without_orphan_coroutines( + monkeypatch: pytest.MonkeyPatch, +): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) router: Final = Router( model_list=[ @@ -16127,10 +16137,9 @@ async def test_router_embedding_path_honors_max_parallel_requests_queue_size(mon "api_base": "https://max-parallel-embed.local/v1", "max_parallel_requests": 1, }, - "model_info": {"id": "embed-bounded-deployment"}, + "model_info": {"id": "embed-capped-deployment"}, } ], - default_max_parallel_requests_queue_size=1, num_retries=0, ) @@ -16159,15 +16168,15 @@ async def test_router_embedding_path_honors_max_parallel_requests_queue_size(mon gc.collect() rejected: Final = [r for r in results if isinstance(r, BaseException)] - assert len(rejected) == 1 and len(results) == 3 - assert isinstance(rejected[0], litellm.RateLimitError) and rejected[0].status_code == 429 - assert "embed-bounded-deployment" in rejected[0].message - assert route.call_count == 2 + assert len(rejected) == 2 and len(results) == 3 + assert all(isinstance(r, litellm.RateLimitError) and r.status_code == 429 for r in rejected) + assert all("embed-capped-deployment" in r.message for r in rejected) + assert route.call_count == 1 assert [str(w.message) for w in caught if "never awaited" in str(w.message)] == [] @pytest.mark.asyncio -async def test_router_max_parallel_requests_queue_overflow_takes_the_ordinary_429_fallback_path( +async def test_router_max_parallel_requests_overflow_takes_the_ordinary_429_fallback_path( monkeypatch: pytest.MonkeyPatch, ): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -16180,9 +16189,8 @@ async def test_router_max_parallel_requests_queue_overflow_takes_the_ordinary_42 "api_key": "sk-fake", "api_base": "https://max-parallel-primary.local/v1", "max_parallel_requests": 1, - "max_parallel_requests_queue_size": 0, }, - "model_info": {"id": "queue-primary-deployment"}, + "model_info": {"id": "capped-primary-deployment"}, }, { "model_name": "gpt-5.6-fallback", @@ -16191,7 +16199,7 @@ async def test_router_max_parallel_requests_queue_overflow_takes_the_ordinary_42 "api_key": "sk-fake", "api_base": "https://max-parallel-fallback.local/v1", }, - "model_info": {"id": "queue-fallback-deployment"}, + "model_info": {"id": "fallback-deployment"}, }, ], fallbacks=[{"gpt-5.6": ["gpt-5.6-fallback"]}], @@ -16232,7 +16240,7 @@ async def test_router_max_parallel_requests_queue_overflow_takes_the_ordinary_42 @pytest.mark.asyncio -async def test_router_deployment_slot_rejects_once_queue_is_full_and_frees_slot_on_exit(): +async def test_router_deployment_slot_rejects_while_held_and_frees_slot_on_exit(): router: Final = Router( model_list=[ { @@ -16241,7 +16249,6 @@ async def test_router_deployment_slot_rejects_once_queue_is_full_and_frees_slot_ "model": "openai/gpt-5.6", "api_key": "sk-fake", "max_parallel_requests": 1, - "max_parallel_requests_queue_size": 0, }, "model_info": {"id": "slot-deployment"}, } diff --git a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx index f2740bbd1e0..1875085231a 100644 --- a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx @@ -137,41 +137,6 @@ describe("RouterSettings", () => { ); }); - it("should save default_max_parallel_requests_queue_size as a number and an empty field as null", async () => { - vi.mocked(getCallbacksCall).mockResolvedValue({ - router_settings: { ...mockCallbacksResponse.router_settings, default_max_parallel_requests_queue_size: null }, - }); - const user = userEvent.setup(); - renderWithProviders(); - - await findStrategySelect(); - - const queueSize = await screen.findByRole("textbox", { name: /default_max_parallel_requests_queue_size/i }); - fireEvent.change(queueSize, { target: { value: "4" } }); - await user.click(screen.getByRole("button", { name: /save changes/i })); - - await waitFor(() => - expect(setCallbacksCall).toHaveBeenLastCalledWith( - "test-token", - expect.objectContaining({ - router_settings: expect.objectContaining({ default_max_parallel_requests_queue_size: 4 }), - }), - ), - ); - - fireEvent.change(queueSize, { target: { value: "" } }); - await user.click(screen.getByRole("button", { name: /save changes/i })); - - await waitFor(() => - expect(setCallbacksCall).toHaveBeenLastCalledWith( - "test-token", - expect.objectContaining({ - router_settings: expect.objectContaining({ default_max_parallel_requests_queue_size: null }), - }), - ), - ); - }); - it("should show a success notification after saving", async () => { const user = userEvent.setup(); renderWithProviders(); diff --git a/ui/litellm-dashboard/src/components/router_settings/index.tsx b/ui/litellm-dashboard/src/components/router_settings/index.tsx index 4170d48361d..53d35b81cec 100644 --- a/ui/litellm-dashboard/src/components/router_settings/index.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/index.tsx @@ -86,15 +86,7 @@ const RouterSettings: React.FC = ({ accessToken, userRole, const router_settings = formValue.routerSettings; - const numberKeys = new Set([ - "allowed_fails", - "cooldown_time", - "num_retries", - "timeout", - "retry_after", - "default_max_parallel_requests_queue_size", - ]); - const unsettableNumberKeys = new Set(["default_max_parallel_requests_queue_size"]); + const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]); const jsonKeys = new Set(["model_group_alias"]); // retry_policy and model_group_retry_policy are owned by the Model Retry Settings tab; // routing_groups is owned by the Routing Groups tab. This page must not read or write them. @@ -108,7 +100,6 @@ const RouterSettings: React.FC = ({ accessToken, userRole, if (v.toLowerCase() === "null") return null; if (numberKeys.has(key)) { - if (v === "" && unsettableNumberKeys.has(key)) return null; const n = Number(v); return Number.isNaN(n) ? fallback : n; } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 84ca93eccfe..872875cc535 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30441,8 +30441,6 @@ export interface components { max_budget?: number | null; /** Max File Size Mb */ max_file_size_mb?: number | null; - /** Max Parallel Requests Queue Size */ - max_parallel_requests_queue_size?: number | null; /** Max Retries */ max_retries?: number | null; /** @@ -40895,8 +40893,6 @@ export interface components { max_budget?: number | null; /** Max File Size Mb */ max_file_size_mb?: number | null; - /** Max Parallel Requests Queue Size */ - max_parallel_requests_queue_size?: number | null; /** Max Retries */ max_retries?: number | null; /**