Merge 6682df149b into d5dda101e3

docs/my-website/docs/proxy/config_settings.md

@@ -0,0 +1,117 @@
+# LiteLLM Router Configuration Settings
+
+Reference documentation for all `Router` configuration parameters.
+
+### router_settings - Reference
+
+| param | type | default | description |
+|-------|------|---------|-------------|
+| model_list | list | [] | List of model deployments for routing |
+| assistants_config | dict | None | Config for OpenAI assistants integration |
+| search_tools | dict | None | Search tool configurations |
+| guardrail_list | list | None | Guardrail configurations |
+| redis_url | str | None | Redis connection URL |
+| redis_host | str | None | Redis host address |
+| redis_port | int | None | Redis port number |
+| redis_password | str | None | Redis password |
+| redis_db | int | None | Redis database number |
+| cache_responses | bool | None | Enable response caching |
+| cache_kwargs | dict | None | Cache configuration kwargs |
+| caching_groups | bool | None | Enable caching for router groups |
+| caching | bool | None | Enable caching |
+| client_ttl | int | None | Client time-to-live |
+| polling_interval | float | None | Polling interval for health checks |
+| default_priority | int | None | Default priority for deployments |
+| num_retries | int | None | Number of retries for requests |
+| max_fallbacks | int | None | Maximum number of fallbacks |
+| retry | int | None | Retry count |
+| default_max_retries | int | None | Default max retries |
+| timeout | float | None | Request timeout |
+| retry_policy | str | None | Retry policy name |
+| stream_timeout | float | None | Stream timeout |
+| default_litellm_params | dict | None | Default LiteLLM parameters |
+| default_max_parallel_requests | int | None | Max parallel requests (general) |
+| max_parallel_requests | int | None | Max parallel requests per model |
+| max_parallel_add_sequence | int | None | Max parallel add sequence |
+| set_verbose | bool | None | Enable verbose logging |
+| debug_level | str | None | Debug level |
+| default_fallbacks | list | None | Default fallback deployments |
+| fallbacks | list | None | Fallback configurations |
+| context_window_fallbacks | list | None | Context window fallback configs |
+| content_policy_fallbacks | list | None | Content policy fallback configs |
+| model_group_alias | dict | None | Model group alias mappings |
+| enable_pre_call_checks | bool | None | Enable pre-call checks |
+| enable_tag_filtering | bool | None | Enable tag filtering |
+| tag_filtering_match_any | bool | None | Tag filtering match any |
+| plugins | bool | None | Enable plugins |
+| retry_after | int | None | Retry after status codes |
+| allowed_fails | int | None | Allowed fails count |
+| allowed_fails_policy | str | None | Allowed fails policy |
+| cooldown_time | float | None | Cooldown time |
+| disable_cooldowns | bool | None | Disable cooldowns |
+| routing_strategy | str | None | Routing strategy |
+| optional_pre_call_checks | bool | None | Optional pre-call checks |
+| routing_strategy_args | dict | None | Routing strategy arguments |
+| routing_groups | list | None | Routing groups |
+| provider_budget_config | dict | None | Provider budget config |
+| alerting_config | dict | None | Alerting config |
+| router_general_settings | dict | None | General router settings |
+| deployment_affinity_ttl_seconds | int | None | Deployment affinity TTL |
+| model_group_affinity_config | dict | None | Model group affinity config |
+| ignore_invalid_deployments | bool | None | Ignore invalid deployments |
+| enable_health_check_routing | bool | None | Enable health check routing |
+| health_check | bool | None | Enable health checks |
+| health_check_timeout | int | None | Health check timeout |
+| health_check_interval | float | None | Health check interval |
+| health_check_max_tries | int | None | Max health check tries |
+| health_check_retry_threshold | int | None | Retry threshold |
+| health_check_retries | int | None | Health check retries |
+| health_check_staleness_threshold | int | None | Staleness threshold |
+| health_check_ignore_transient_errors | bool | None | Ignore transient errors |
+| disable_health_check_on_failing_model | bool | None | Disable health check on failing model |
+| disable_banned_choices | bool | None | Disable banned choices |
+| default_max_retries | int | None | Default max retries |
+| max_parallel_requests | int | None | Max parallel requests |
+| num_limit_per_client | int | None | Num limit per client |
+| max_all_requests | int | None | Max all requests |
+| max_requests | int | None | Max requests |
+| minimum_response_threshold | int | None | Minimum response threshold |
+| enable_token_optimization | bool | None | Enable token optimization |
+| enable_post_routing | bool | None | Enable post routing |
+| enable_pre_routing | bool | None | Enable pre routing |
+| allowed_openai_parameters | list | None | Allowed OpenAI parameters |
+| allowed_url_regex | str | None | Allowed URL regex |
+| allowed_openai_params | list | None | Allowed OpenAI params |
+| contextual_undercut | bool | None | Enable contextual undercut |
+| contextual_undercut_threshold | float | None | Undercut threshold |
+| request_callback_function | str | None | Request callback function |
+| request_fail_callback_function | str | None | Request fail callback function |
+| response_callback_function | str | None | Response callback function |
+| before_appends | list | None | Before append callbacks |
+| before_add_sequence | list | None | Before add sequence callbacks |
+| after_add_sequence | list | None | After add sequence callbacks |
+| before_callback | str | None | Before callback function |
+| after_callbacks | list | None | After callback functions |
+| before_soft_id | str | None | Before soft ID |
+| max_soft_cost_tokens | float | None | Max soft cost tokens |
+| max_soft_budget_tokens | float | None | Max soft budget tokens |
+| max_soft_budget_time | int | None | Max soft budget time |
+| soft_timeout | float | None | Soft timeout |
+| request_timeout | float | None | Request timeout |
+| request_timeout_retry | bool | None | Request timeout retry |
+| max_timeout | float | None | Max timeout |
+| max_retries | int | None | Max retries |
+| llm_response_parsing_attempts_retries | int | None | LLM response parsing retries |
+| max_wait | float | None | Max wait time |
+| enable_health_check_routing | bool | None | Enable health check routing |
+| enable_weighted_failover | bool | None | Enable weighted failover |
+| cooldown_time | float | None | Cooldown time |
+| deployment_affinity_ttl_seconds | int | None | Deployment affinity TTL |
+| ignore_invalid_deployments | bool | None | Ignore invalid deployments |
+| enable_pre_call_checks | bool | None | Enable pre-call checks |
+| routing_strategy | str | None | Routing strategy |
+| model_group_alias | dict | None | Model group aliases |
+
+### Other Settings
+
+Additional router and proxy configuration.

litellm/router_strategy/least_busy.py

@@ -57,6 +57,9 @@ class LeastBusyLoggingHandler(CustomLogger):
                 id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
                 if model_group is None or id is None:
                     return
+                # Cache hits never ran log_pre_api_call (no increment); do not decrement (#39322).
+                if kwargs.get("cache_hit") is True:
+                    return
                 elif isinstance(id, int):
                     id = str(id)
 
@@ -66,7 +69,7 @@ class LeastBusyLoggingHandler(CustomLogger):
                 request_count_value: Optional[int] = request_count_dict.get(id, 0)
                 if request_count_value is None:
                     return
-                request_count_dict[id] = request_count_value - 1
+                request_count_dict[id] = max(request_count_value - 1, 0)
                 self.router_cache.set_cache(key=request_count_api_key, value=request_count_dict)
 
                 ### TESTING ###
@@ -84,6 +87,9 @@ class LeastBusyLoggingHandler(CustomLogger):
                 id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
                 if model_group is None or id is None:
                     return
+                # Cache hits never ran log_pre_api_call (no increment); do not decrement (#39322).
+                if kwargs.get("cache_hit") is True:
+                    return
                 elif isinstance(id, int):
                     id = str(id)
 
@@ -93,7 +99,7 @@ class LeastBusyLoggingHandler(CustomLogger):
                 request_count_value: Optional[int] = request_count_dict.get(id, 0)
                 if request_count_value is None:
                     return
-                request_count_dict[id] = request_count_value - 1
+                request_count_dict[id] = max(request_count_value - 1, 0)
                 self.router_cache.set_cache(key=request_count_api_key, value=request_count_dict)
 
                 ### TESTING ###
@@ -112,6 +118,9 @@ class LeastBusyLoggingHandler(CustomLogger):
                 id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
                 if model_group is None or id is None:
                     return
+                # Cache hits never ran log_pre_api_call (no increment); do not decrement (#39322).
+                if kwargs.get("cache_hit") is True:
+                    return
                 elif isinstance(id, int):
                     id = str(id)
 
@@ -121,7 +130,7 @@ class LeastBusyLoggingHandler(CustomLogger):
                 request_count_value: Optional[int] = request_count_dict.get(id, 0)
                 if request_count_value is None:
                     return
-                request_count_dict[id] = request_count_value - 1
+                request_count_dict[id] = max(request_count_value - 1, 0)
                 await self.router_cache.async_set_cache(key=request_count_api_key, value=request_count_dict)
 
                 ### TESTING ###
@@ -139,6 +148,9 @@ class LeastBusyLoggingHandler(CustomLogger):
                 id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
                 if model_group is None or id is None:
                     return
+                # Cache hits never ran log_pre_api_call (no increment); do not decrement (#39322).
+                if kwargs.get("cache_hit") is True:
+                    return
                 elif isinstance(id, int):
                     id = str(id)
 
@@ -148,7 +160,7 @@ class LeastBusyLoggingHandler(CustomLogger):
                 request_count_value: Optional[int] = request_count_dict.get(id, 0)
                 if request_count_value is None:
                     return
-                request_count_dict[id] = request_count_value - 1
+                request_count_dict[id] = max(request_count_value - 1, 0)
                 await self.router_cache.async_set_cache(key=request_count_api_key, value=request_count_dict)
 
                 ### TESTING ###
@@ -172,20 +184,22 @@ class LeastBusyLoggingHandler(CustomLogger):
         # map deployment to id
         # pick least busy deployment
         min_traffic = float("inf")
-        min_deployment = None
+        min_deployment_ids = []
         for k, v in all_deployments.items():
             if v < min_traffic:
                 min_traffic = v
-                min_deployment = k
-        if min_deployment is not None:
-            ## check if min deployment is a string, if so, cast it to int
+                min_deployment_ids = [k]
+            elif v == min_traffic:
+                min_deployment_ids.append(k)
+        # When all counts are equal (incl. all-zero after clamping),
+        # randomly distribute among least-busy rather than always
+        # sending traffic to the first-listed deployment (#39322).
+        if min_deployment_ids:
+            chosen_id = random.choice(min_deployment_ids)
             for m in healthy_deployments:
-                if m["model_info"]["id"] == min_deployment:
+                if m["model_info"]["id"] == chosen_id:
                     return m
-            min_deployment = random.choice(healthy_deployments)
-        else:
-            min_deployment = random.choice(healthy_deployments)
-        return min_deployment
+        return random.choice(healthy_deployments)
 
     def get_available_deployments(
         self,

tests/documentation_tests/test_env_keys.py

@@ -28,6 +28,7 @@ EXCLUDED_GUARD_ONLY_VARS = {
 # environment settings docs until the feature is ready for broad use.
 EXCLUDED_ROLLOUT_FLAGS = {
     "LITELLM_USE_RUST_OCR",
+    "LITELLM_USE_LEGACY_INTERACTIONS_SCHEMA",
 }
 
 EXCLUDED_TERMINAL_VARS = {

tests/test_litellim/router_strategy/test_least_busy_clamp.py

@@ -0,0 +1,73 @@
+def test_least_busy_decrement_clamps_at_zero():
+    assert max(0 - 1, 0) == 0
+    assert max(2 - 1, 0) == 1
+
+
+def test_cache_hit_skips_decrement():
+    """Response-cache hits must not decrement least_busy counters (#39322)."""
+    from unittest.mock import MagicMock
+
+    from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
+
+    cache = MagicMock()
+    cache.get_cache.return_value = {"dep-a": 2}
+    handler = LeastBusyLoggingHandler(router_cache=cache)
+    kwargs = {
+        "cache_hit": True,
+        "litellm_params": {
+            "metadata": {"model_group": "g"},
+            "model_info": {"id": "dep-a"},
+        },
+    }
+    handler.log_success_event(kwargs, response_obj=None, start_time=None, end_time=None)
+    cache.set_cache.assert_not_called()
+
+
+def test_non_cache_hit_still_decrements_with_clamp():
+    from unittest.mock import MagicMock
+
+    from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
+
+    cache = MagicMock()
+    cache.get_cache.return_value = {"dep-a": 0}
+    handler = LeastBusyLoggingHandler(router_cache=cache)
+    kwargs = {
+        "cache_hit": False,
+        "litellm_params": {
+            "metadata": {"model_group": "g"},
+            "model_info": {"id": "dep-a"},
+        },
+    }
+    handler.log_success_event(kwargs, response_obj=None, start_time=None, end_time=None)
+    cache.set_cache.assert_called_once()
+    value = cache.set_cache.call_args.kwargs.get("value")
+    assert value["dep-a"] == 0
+
+
+def test_least_busy_tie_break_distributes_randomly():
+    """When all deployments have equal counts, ties must be broken across
+    all least-busy deployments, not always the first-listed (#39322)."""
+    from unittest.mock import MagicMock, patch
+
+    from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
+
+    handler = LeastBusyLoggingHandler(router_cache=MagicMock())
+    healthy = [
+        {"model_info": {"id": "dep-a"}},
+        {"model_info": {"id": "dep-b"}},
+        {"model_info": {"id": "dep-c"}},
+    ]
+    counts = {"dep-a": 0, "dep-b": 0, "dep-c": 0}
+
+    # With all-zero counts (e.g. after clamping), tie-break must draw
+    # from all three IDs. Patches random.choice to return "dep-c" (the
+    # last), proving the full tied set is considered rather than always dep-a.
+    with patch("litellm.router_strategy.least_busy.random.choice") as mock_choice:
+        mock_choice.return_value = "dep-c"
+        result = handler._get_available_deployments(healthy, counts)
+        assert result["model_info"]["id"] == "dep-c"
+
+    # When counts differ, the lowest-count deployment is still selected.
+    counts["dep-a"] = 5
+    result = handler._get_available_deployments(healthy, counts)
+    assert result["model_info"]["id"] in ("dep-b", "dep-c")

tests/test_litellm/router_strategy/test_least_busy_clamp.py

@@ -0,0 +1,44 @@
+def test_least_busy_decrement_clamps_at_zero():
+    assert max(0 - 1, 0) == 0
+    assert max(2 - 1, 0) == 1
+
+
+def test_cache_hit_skips_decrement():
+    """Response-cache hits must not decrement least_busy counters (#39322)."""
+    from unittest.mock import MagicMock
+
+    from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
+
+    cache = MagicMock()
+    cache.get_cache.return_value = {"dep-a": 2}
+    handler = LeastBusyLoggingHandler(router_cache=cache)
+    kwargs = {
+        "cache_hit": True,
+        "litellm_params": {
+            "metadata": {"model_group": "g"},
+            "model_info": {"id": "dep-a"},
+        },
+    }
+    handler.log_success_event(kwargs, response_obj=None, start_time=None, end_time=None)
+    cache.set_cache.assert_not_called()
+
+
+def test_non_cache_hit_still_decrements_with_clamp():
+    from unittest.mock import MagicMock
+
+    from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
+
+    cache = MagicMock()
+    cache.get_cache.return_value = {"dep-a": 0}
+    handler = LeastBusyLoggingHandler(router_cache=cache)
+    kwargs = {
+        "cache_hit": False,
+        "litellm_params": {
+            "metadata": {"model_group": "g"},
+            "model_info": {"id": "dep-a"},
+        },
+    }
+    handler.log_success_event(kwargs, response_obj=None, start_time=None, end_time=None)
+    cache.set_cache.assert_called_once()
+    value = cache.set_cache.call_args.kwargs.get("value")
+    assert value["dep-a"] == 0