diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index a659d62e3eb..5a1d6bec5d5 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -4,7 +4,7 @@ Dynamic rate limiter v3 - Saturation-aware priority-based rate limiting import os from datetime import datetime -from typing import Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union from fastapi import HTTPException @@ -24,6 +24,25 @@ from litellm.proxy.utils import InternalUsageCache from litellm.types.router import ModelGroupInfo from litellm.types.utils import CallTypesLiteral +if TYPE_CHECKING: + from litellm.types.utils import PriorityReservationSettings + + +def _get_priority_settings() -> "PriorityReservationSettings": + """ + Get the priority reservation settings, guaranteed to be non-None. + + The settings are lazy-loaded in litellm.__init__ and always return an instance. + This helper provides proper type narrowing for mypy. + """ + settings = litellm.priority_reservation_settings + if settings is None: + # This should never happen due to lazy loading, but satisfy mypy + from litellm.types.utils import PriorityReservationSettings + + return PriorityReservationSettings() + return settings + class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): """ @@ -60,7 +79,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): def _get_saturation_check_cache_ttl(self) -> int: """Get the configurable TTL for local cache when reading saturation values.""" - return litellm.priority_reservation_settings.saturation_check_cache_ttl + return _get_priority_settings().saturation_check_cache_ttl async def _get_saturation_value_from_cache( self, @@ -91,7 +110,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): self, priority: Optional[str], model_info: Optional[ModelGroupInfo] = None ) -> float: """Get the weight for a given priority from litellm.priority_reservation""" - weight: float = litellm.priority_reservation_settings.default_priority + weight: float = _get_priority_settings().default_priority if ( litellm.priority_reservation is None or priority not in litellm.priority_reservation @@ -201,7 +220,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): priority_key = f"{model}:{priority}" else: # No explicit priority: share the default_priority pool with ALL other default keys - priority_weight = litellm.priority_reservation_settings.default_priority + priority_weight = _get_priority_settings().default_priority # Use shared key for all default-priority requests priority_key = f"{model}:default_pool" @@ -418,9 +437,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): """ import json - saturation_threshold = ( - litellm.priority_reservation_settings.saturation_threshold - ) + saturation_threshold = _get_priority_settings().saturation_threshold should_enforce_priority = saturation >= saturation_threshold # Build ALL descriptors upfront @@ -593,9 +610,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): # STEP 1: Check current saturation level saturation = await self._check_model_saturation(model, model_group_info) - saturation_threshold = ( - litellm.priority_reservation_settings.saturation_threshold - ) + saturation_threshold = _get_priority_settings().saturation_threshold verbose_proxy_logger.debug( f"[Dynamic Rate Limiter] Model={model}, Saturation={saturation:.1%}, " diff --git a/litellm/proxy/policy_engine/policy_validator.py b/litellm/proxy/policy_engine/policy_validator.py index f79fb8bfa9c..47787655cba 100644 --- a/litellm/proxy/policy_engine/policy_validator.py +++ b/litellm/proxy/policy_engine/policy_validator.py @@ -163,6 +163,7 @@ class PolicyValidator: policy_name: str, policies: Dict[str, Policy], visited: Optional[Set[str]] = None, + max_depth: int = 100, ) -> List[PolicyValidationError]: """ Validate the inheritance chain for a policy. @@ -170,17 +171,31 @@ class PolicyValidator: Checks for: - Parent policy exists - No circular inheritance + - Max depth not exceeded Args: policy_name: Name of the policy to validate policies: All policies visited: Set of already visited policy names (for cycle detection) + max_depth: Maximum recursion depth to prevent infinite loops Returns: List of validation errors """ errors: List[PolicyValidationError] = [] + # Prevent infinite recursion + if max_depth <= 0: + errors.append( + PolicyValidationError( + policy_name=policy_name, + error_type=PolicyValidationErrorType.CIRCULAR_INHERITANCE, + message=f"Inheritance chain too deep (exceeded max depth of 100)", + field="inherit", + ) + ) + return errors + if visited is None: visited = set() @@ -211,10 +226,12 @@ class PolicyValidator: ) ) else: - # Recursively check parent + # Recursively check parent with decremented depth visited.add(policy_name) errors.extend( - self._validate_inheritance_chain(policy.inherit, policies, visited) + self._validate_inheritance_chain( + policy.inherit, policies, visited, max_depth - 1 + ) ) return errors diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 2a460f621b2..d5640f4256c 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -39,6 +39,7 @@ IGNORE_FUNCTIONS = [ "_delete_nested_value_custom", # max depth set (bounded by number of path segments). "filter_exceptions_from_params", # max depth set (default 20) to prevent infinite recursion. "__getattr__", # lazy loading pattern in litellm/__init__.py with proper caching to prevent infinite recursion. + "_validate_inheritance_chain", # max depth set (default 100) to prevent infinite recursion in policy inheritance validation. ]