diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dc78ec75a6e..216a146143d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -776,6 +776,7 @@ from litellm.types.router import ( RoutingPlugin, SearchToolTypedDict, updateDeployment, + validate_max_parallel_requests_queue_size, ) from litellm.types.router import ModelInfo as RouterModelInfo from litellm.types.scheduler import DefaultPriorities @@ -16936,6 +16937,17 @@ async def update_config( ) }, ) + raw_queue_size: Final = raw_router_settings.get("default_max_parallel_requests_queue_size") + try: + validate_max_parallel_requests_queue_size(raw_queue_size) + except ValueError as invalid_queue_size: + raise HTTPException( + status_code=400, + detail=( + f"default_max_parallel_requests_queue_size={raw_queue_size!r} is not valid, " + "it must be a non-negative integer or null" + ), + ) from invalid_queue_size if prisma_client is None: raise Exception("No DB Connected") diff --git a/litellm/router.py b/litellm/router.py index 132f48730df..97325e6c450 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -263,6 +263,7 @@ from litellm.types.router import ( RoutingStrategy, SearchToolTypedDict, TaggedPreRoutingStrategy, + validate_max_parallel_requests_queue_size, ) from litellm.types.services import ServiceTypes from litellm.types.utils import ( @@ -936,7 +937,9 @@ class Router: None # use this to track the users default deployment, when they want to use model = * ) self.default_max_parallel_requests = default_max_parallel_requests - self._default_max_parallel_requests_queue_size = default_max_parallel_requests_queue_size + self._default_max_parallel_requests_queue_size = validate_max_parallel_requests_queue_size( + default_max_parallel_requests_queue_size + ) self.provider_default_deployment_ids: list[str] = [] self.pattern_router = PatternMatchRouter() self.team_pattern_routers: dict[str, PatternMatchRouter] = {} # {"TEAM_ID": PatternMatchRouter} @@ -11852,7 +11855,7 @@ class Router: @default_max_parallel_requests_queue_size.setter def default_max_parallel_requests_queue_size(self, queue_size: int | None) -> None: - self._default_max_parallel_requests_queue_size = None if queue_size is None else int(queue_size) + self._default_max_parallel_requests_queue_size = validate_max_parallel_requests_queue_size(queue_size) InitalizeCachedClient.apply_default_max_parallel_requests_queue_size( litellm_router_instance=self, queue_size=self._default_max_parallel_requests_queue_size ) diff --git a/litellm/router_utils/client_initalization_utils.py b/litellm/router_utils/client_initalization_utils.py index 72854cba028..be5f71a4e70 100644 --- a/litellm/router_utils/client_initalization_utils.py +++ b/litellm/router_utils/client_initalization_utils.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_router_logger from litellm.exceptions import RateLimitError, RateLimitErrorCategory, RateLimitType -from litellm.types.router import RouterErrors +from litellm.types.router import RouterErrors, validate_max_parallel_requests_queue_size from litellm.utils import calculate_max_parallel_requests if TYPE_CHECKING: @@ -26,7 +26,7 @@ class DeploymentSemaphore: self.max_parallel_requests: Final = max_parallel_requests self.model_id: Final = model_id self.model_group: Final = model_group - self.queue_size = queue_size + self.queue_size = validate_max_parallel_requests_queue_size(queue_size) self.waiting = 0 def locked(self) -> bool: diff --git a/litellm/types/router.py b/litellm/types/router.py index 8f788b5f933..848dd28aaac 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -6,10 +6,10 @@ import datetime import enum from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints +from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints import httpx -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator, model_validator from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger @@ -314,6 +314,14 @@ class CredentialLiteLLMParams(BaseModel): _RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"}) +MaxParallelRequestsQueueSize = Annotated[int, Field(strict=True, ge=0)] +_MAX_PARALLEL_REQUESTS_QUEUE_SIZE_ADAPTER: Final = TypeAdapter(MaxParallelRequestsQueueSize | None) + + +def validate_max_parallel_requests_queue_size(value: object) -> int | None: + return _MAX_PARALLEL_REQUESTS_QUEUE_SIZE_ADAPTER.validate_python(value) + + class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): """ LiteLLM Params without 'model' arg (used across completion / assistants api) @@ -324,6 +332,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): rpm: int | None = None itpm: int | None = None otpm: int | None = None + max_parallel_requests_queue_size: MaxParallelRequestsQueueSize | None = None timeout: float | str | httpx.Timeout | None = None # if str, pass in as os.environ/ stream_timeout: float | str | None = None # timeout when making stream=True calls, if str, pass in as os.environ/ max_retries: int | None = None @@ -497,7 +506,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): order: int | None weight: int | None max_parallel_requests: int | None - max_parallel_requests_queue_size: ReadOnly[int | None] + max_parallel_requests_queue_size: ReadOnly[MaxParallelRequestsQueueSize | None] api_key: str | None api_base: str | None api_version: str | None diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 057e82a24c8..582977d613b 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -88,7 +88,7 @@ ignored_function_names = [ "_resolve_claude_code_session_router", # Tested through Claude Code session routing in test_router.py "_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py "_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name) - "default_max_parallel_requests_queue_size", # Property, so its reads and assignments in test_router.py are never an ast.Call + "default_max_parallel_requests_queue_size", ] diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 5754301ac4a..5a2e76039e8 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -9393,6 +9393,25 @@ def test_update_config_router_settings_null_clears_max_parallel_requests_queue_s restore() +@pytest.mark.parametrize("invalid_queue_size", [-1, 2.5, "3"]) +def test_update_config_rejects_invalid_max_parallel_requests_queue_size_before_persisting( + _update_config_setup, invalid_queue_size +): + client, prisma, restore = _update_config_setup( + initial_rows={"router_settings": {"default_max_parallel_requests_queue_size": 3}}, + ) + try: + resp = client.post( + "/config/update", + json={"router_settings": {"default_max_parallel_requests_queue_size": invalid_queue_size}}, + ) + assert resp.status_code == 400 + assert "default_max_parallel_requests_queue_size" in resp.json()["error"]["message"] + assert prisma.db.litellm_config.rows["router_settings"] == {"default_max_parallel_requests_queue_size": 3} + finally: + restore() + + def test_update_config_success_callback_normalizes_existing_mixed_case( _update_config_setup, ): diff --git a/tests/test_litellm/router_utils/test_client_initalization_utils.py b/tests/test_litellm/router_utils/test_client_initalization_utils.py index 332f2f1503a..a6626d2f975 100644 --- a/tests/test_litellm/router_utils/test_client_initalization_utils.py +++ b/tests/test_litellm/router_utils/test_client_initalization_utils.py @@ -2,6 +2,7 @@ import asyncio from typing import Final import pytest +from pydantic import ValidationError import litellm from litellm import Router @@ -111,6 +112,37 @@ def _router_semaphore(router: Router, model_name: str) -> DeploymentSemaphore: return client +@pytest.mark.parametrize("invalid_queue_size", [-1, 2.5, True, "3"]) +def test_invalid_queue_sizes_are_rejected_instead_of_coerced(invalid_queue_size: object): + """A negative bound would reject every busy request and a fraction would be truncated, so + neither may reach a semaphore, the router default, or a live update of that default.""" + with pytest.raises(ValidationError): + _semaphore(queue_size=invalid_queue_size) + model_list: Final = [{"model_name": "gpt-5.6", "litellm_params": {"model": "openai/gpt-5.6", "rpm": 1}}] + with pytest.raises(ValidationError): + Router(model_list=model_list, default_max_parallel_requests_queue_size=invalid_queue_size) + with pytest.raises(ValidationError): + Router( + model_list=[ + { + "model_name": "gpt-5.6", + "litellm_params": { + "model": "openai/gpt-5.6", + "rpm": 1, + "max_parallel_requests_queue_size": invalid_queue_size, + }, + } + ] + ) + + router: Final = Router(model_list=model_list, default_max_parallel_requests_queue_size=4) + semaphore: Final = _router_semaphore(router, "gpt-5.6") + with pytest.raises(ValidationError): + router.update_settings(default_max_parallel_requests_queue_size=invalid_queue_size) + assert router.default_max_parallel_requests_queue_size == 4 + assert semaphore.queue_size == 4 + + @pytest.mark.asyncio async def test_deployment_queue_size_overrides_router_default_and_zero_is_honored(): router: Final = Router( @@ -170,7 +202,7 @@ async def test_update_settings_applies_default_queue_size_to_live_semaphores_wit pinned: Final = _router_semaphore(router, "pinned") assert router.get_settings()["default_max_parallel_requests_queue_size"] is None - router.update_settings(default_max_parallel_requests_queue_size="0") + router.update_settings(default_max_parallel_requests_queue_size=0) assert router.get_settings()["default_max_parallel_requests_queue_size"] == 0 assert (inherits.queue_size, pinned.queue_size) == (0, 5) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 872875cc535..84ca93eccfe 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30441,6 +30441,8 @@ export interface components { max_budget?: number | null; /** Max File Size Mb */ max_file_size_mb?: number | null; + /** Max Parallel Requests Queue Size */ + max_parallel_requests_queue_size?: number | null; /** Max Retries */ max_retries?: number | null; /** @@ -40893,6 +40895,8 @@ export interface components { max_budget?: number | null; /** Max File Size Mb */ max_file_size_mb?: number | null; + /** Max Parallel Requests Queue Size */ + max_parallel_requests_queue_size?: number | null; /** Max Retries */ max_retries?: number | null; /**