mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-03 07:23:33 +00:00
3e18a4b55d
* fix(router): record cooled deployments on the all-cooled failure row A RouterRateLimitError now carries the cooled deployments of the route the request resolves to for the caller's team (model group, alias, wildcard or team route), each with its masked api_base. get_error_information copies them into error_information.error_cooldown_deployments so the SpendLogs failure row and every logging callback can tell which deployments were cooling * fix(router): name cooled deployments from the router's own resolved route Use the deployments the router resolved for the request, before health and cooldown filtering, instead of re-deriving the route. Alias chains, the provider-model fallback, wildcards and team filtering now match routing by construction. A routing-strategy refusal records null. * fix(router): name only the cooled deployments tag and region routing allow The refusal's cooled-deployment list now leaves out deployments the request's tags or the end user's allowed region keep it from, so a tenant's refusal row no longer shows another tenant's api_base * fix(router): keep failure logging working for refusal subclasses that skip init A RouterRateLimitError subclass that never calls its parent init now logs a null cooled-deployment list instead of breaking the failure payload * test(router): call the refusal routing helpers directly The router coverage check needs each router function called by name from a unit test --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com>
107 lines
3.7 KiB
Python
107 lines
3.7 KiB
Python
from collections.abc import Collection
|
|
from typing import TYPE_CHECKING, Any, Final
|
|
|
|
from litellm._logging import redact_secrets, verbose_router_logger
|
|
from litellm.constants import MAX_EXCEPTION_MESSAGE_LENGTH
|
|
from litellm.router_utils.cooldown_handlers import (
|
|
async_get_cooldown_deployments_with_debug_info,
|
|
get_cooldown_deployment_info,
|
|
)
|
|
from litellm.types.integrations.slack_alerting import AlertType
|
|
from litellm.types.router import RouterRateLimitError
|
|
|
|
if TYPE_CHECKING:
|
|
from opentelemetry.trace import Span as _Span
|
|
|
|
from litellm.router import Router as _Router
|
|
|
|
LitellmRouter = _Router
|
|
Span = _Span
|
|
else:
|
|
LitellmRouter = Any
|
|
Span = Any
|
|
|
|
|
|
async def send_llm_exception_alert(
|
|
litellm_router_instance: LitellmRouter,
|
|
request_kwargs: dict,
|
|
error_traceback_str: str,
|
|
original_exception,
|
|
):
|
|
"""
|
|
Only runs if router.slack_alerting_logger is set
|
|
Sends a Slack / MS Teams alert for the LLM API call failure. Only if router.slack_alerting_logger is set.
|
|
|
|
Parameters:
|
|
litellm_router_instance (_Router): The LitellmRouter instance.
|
|
original_exception (Any): The original exception that occurred.
|
|
|
|
Returns:
|
|
None
|
|
"""
|
|
if litellm_router_instance is None:
|
|
return
|
|
|
|
if not hasattr(litellm_router_instance, "slack_alerting_logger"):
|
|
return
|
|
|
|
if litellm_router_instance.slack_alerting_logger is None:
|
|
return
|
|
|
|
if "proxy_server_request" in request_kwargs:
|
|
# Do not send any alert if it's a request from litellm proxy server request
|
|
# the proxy is already instrumented to send LLM API call failures
|
|
return
|
|
|
|
litellm_debug_info: Final = getattr(original_exception, "litellm_debug_info", None)
|
|
exception_str = str(original_exception)
|
|
if litellm_debug_info is not None:
|
|
exception_str += litellm_debug_info
|
|
exception_str += f"\n\n{error_traceback_str[:MAX_EXCEPTION_MESSAGE_LENGTH]}"
|
|
|
|
# Redact secrets before sending to external service (Slack / MS Teams)
|
|
exception_str = redact_secrets(exception_str)
|
|
|
|
await litellm_router_instance.slack_alerting_logger.send_alert(
|
|
message=f"LLM API call failed: `{exception_str}`",
|
|
level="High",
|
|
alert_type=AlertType.llm_exceptions,
|
|
alerting_metadata={},
|
|
)
|
|
|
|
|
|
async def async_raise_no_deployment_exception(
|
|
litellm_router_instance: LitellmRouter,
|
|
model: str,
|
|
parent_otel_span: Span | None,
|
|
route_model_ids: Collection[str] | None = None,
|
|
):
|
|
"""
|
|
Raises a RouterRateLimitError if no deployment is found for the given model.
|
|
"""
|
|
verbose_router_logger.info("get_available_deployment for model: %s, No deployment available", model)
|
|
model_ids: Final = litellm_router_instance.get_model_ids(model_name=model)
|
|
_cooldown_time: Final = litellm_router_instance.cooldown_cache.get_min_cooldown(
|
|
model_ids=model_ids, parent_otel_span=parent_otel_span
|
|
)
|
|
_cooldown_list: Final = await async_get_cooldown_deployments_with_debug_info(
|
|
litellm_router_instance=litellm_router_instance,
|
|
parent_otel_span=parent_otel_span,
|
|
)
|
|
verbose_router_logger.info(
|
|
"No deployment found for model: %s, cooldown_list with debug info: %s", model, _cooldown_list
|
|
)
|
|
|
|
cooldown_list_ids: Final = [cooldown_model[0] for cooldown_model in (_cooldown_list or [])]
|
|
return RouterRateLimitError(
|
|
model=model,
|
|
cooldown_time=_cooldown_time,
|
|
enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks,
|
|
cooldown_list=cooldown_list_ids,
|
|
model_ids=model_ids,
|
|
cooldown_deployments=(
|
|
None
|
|
if route_model_ids is None
|
|
else get_cooldown_deployment_info(litellm_router_instance, route_model_ids, cooldown_list_ids)
|
|
),
|
|
)
|