scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

fix allow SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD

Browse source
This commit is contained in:
Ishaan Jaff 2025-02-17 19:12:59 -08:00
parent 3d0f67b260
commit b5709ca408
4 changed files with 21 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/__init__.py
View file

@ -7,6 +7,7 @@ import threading
import os
from typing import Callable, List, Optional, Dict, Union, Any, Literal, get_args
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm import constants
from litellm.caching.caching import Cache, DualCache, RedisCache, InMemoryCache
from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES
from litellm.types.utils import (

13
litellm/proxy/proxy_server.py
View file

@ -1972,6 +1972,13 @@ class ProxyConfig:
raise Exception(
f"Invalid value set for upperbound_key_generate_params - value={value}"
)
elif key == "constants":
# allow overriding constants
for constant_key, constant_value in value.items():
verbose_proxy_logger.debug(
f"{blue_color_code} setting litellm.constants.{constant_key}={constant_value}{reset_color_code}"
)
setattr(litellm.constants, constant_key, constant_value)
else:
verbose_proxy_logger.debug(
f"{blue_color_code} setting litellm.{key}={value}{reset_color_code}"
@ -8850,3 +8857,9 @@ app.include_router(openai_files_router)
app.include_router(team_callback_router)
app.include_router(budget_management_router)
app.include_router(model_management_router)
app.include_router(ui_crud_endpoints_router)
app.include_router(openai_files_router)
app.include_router(team_callback_router)
app.include_router(budget_management_router)
app.include_router(model_management_router)

5
litellm/router.py
View file

@ -68,7 +68,6 @@ from litellm.router_utils.batch_utils import (
from litellm.router_utils.client_initalization_utils import InitalizeOpenAISDKClient
from litellm.router_utils.cooldown_cache import CooldownCache
from litellm.router_utils.cooldown_handlers import (
DEFAULT_COOLDOWN_TIME_SECONDS,
_async_get_cooldown_deployments,
_async_get_cooldown_deployments_with_debug_info,
_get_cooldown_deployments,
@ -383,7 +382,9 @@ class Router:
self.allowed_fails = allowed_fails
else:
self.allowed_fails = litellm.allowed_fails
self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
self.cooldown_time = (
cooldown_time or litellm.constants.DEFAULT_COOLDOWN_TIME_SECONDS
)
self.cooldown_cache = CooldownCache(
cache=self.cache, default_cooldown_time=self.cooldown_time
)

12
litellm/router_utils/cooldown_handlers.py
View file

@ -11,11 +11,6 @@ from typing import TYPE_CHECKING, Any, List, Optional, Union
import litellm
from litellm._logging import verbose_router_logger
from litellm.constants import (
DEFAULT_COOLDOWN_TIME_SECONDS,
DEFAULT_FAILURE_THRESHOLD_PERCENT,
SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD,
)
from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback
from .router_callbacks.track_deployment_metrics import (
@ -197,12 +192,12 @@ def _should_cooldown_deployment(
elif (
percent_fails == 1.0
and total_requests_this_minute
>= SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD
>= litellm.constants.SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD
):
# Cooldown if all requests failed and we have reasonable traffic
return True
elif (
percent_fails > DEFAULT_FAILURE_THRESHOLD_PERCENT
percent_fails > litellm.constants.DEFAULT_FAILURE_THRESHOLD_PERCENT
and not is_single_deployment_model_group # by default we should avoid cooldowns on single deployment model groups
):
return True
@ -377,7 +372,8 @@ def should_cooldown_based_on_allowed_fails_policy(
or litellm_router_instance.allowed_fails
)
cooldown_time = (
litellm_router_instance.cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
litellm_router_instance.cooldown_time
or litellm.constants.DEFAULT_COOLDOWN_TIME_SECONDS
)
current_fails = litellm_router_instance.failed_calls.get_cache(key=deployment) or 0