mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor(router): add router entry points and keep the old members as a deprecated mixin
Router gains eight entry points (registry, settings, health, select, apply/aapply, observe, register_plugin, retire) in litellm/router_api.py. The 123 public members outside code used become DeprecatedRouterMembers (litellm/router_deprecated.py): same signatures and results, a DeprecationWarning naming the replacement and a use counter for callers outside the router. Router internals move to _core_ names; subclass overrides of old members are still called by the router.
This commit is contained in:
parent
ab1022dbb9
commit
cbb43d2fee
8 changed files with 3275 additions and 713 deletions
1459
litellm/router.py
1459
litellm/router.py
File diff suppressed because it is too large
Load diff
1262
litellm/router_api.py
Normal file
1262
litellm/router_api.py
Normal file
File diff suppressed because it is too large
Load diff
759
litellm/router_deprecated.py
Normal file
759
litellm/router_deprecated.py
Normal file
|
|
@ -0,0 +1,759 @@
|
|||
"""
|
||||
DeprecatedRouterMembers: the Router members that outside code used before the entry points.
|
||||
|
||||
``litellm.Router`` mixes this class in so existing callers keep working while they move to the
|
||||
entry points (``registry()``, ``settings()``, ``health()``, ``select()``, ``apply()``,
|
||||
``observe()``, ``register_plugin()``, ``retire()``; see litellm/router_api.py). Every member here
|
||||
returns what it returned before, with the type it had before, and names its replacement in a
|
||||
``DeprecationWarning``. Reads go to the same Router implementation (``Router._core_<name>``) the
|
||||
entry point uses; writes go through ``apply`` / ``aapply`` so the registry version moves.
|
||||
|
||||
Access from the router itself and from its own modules (``litellm.router*``) is not counted:
|
||||
those are the router's internals and move behind the core interface separately. Every other access
|
||||
is counted in ``DEPRECATED_ROUTER_MEMBER_USES`` so the remaining users show up before this class is
|
||||
deleted (planned about a month after the core swap).
|
||||
"""
|
||||
|
||||
# pyright: reportPrivateUsage=false, reportMissingTypeArgument=false, reportUnknownParameterType=false
|
||||
# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false
|
||||
# pyright: reportUnknownLambdaType=false
|
||||
# This shim is part of the router (it reads Router's ``_core_*`` members, as Router's own methods do)
|
||||
# and repeats each old member's signature exactly, untyped parts included (``dict``, ``list``), so
|
||||
# callers keep the types they had. Typing those parts is the core's job, not the shim's.
|
||||
|
||||
import contextvars
|
||||
import functools
|
||||
import inspect
|
||||
import sys
|
||||
import warnings
|
||||
from collections import Counter
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload
|
||||
|
||||
from litellm.router_api import SETTING_FIELDS, ChangeSet
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import Any # noqa: TID251 # only to repeat two old signatures that had Any, unchanged
|
||||
|
||||
from opentelemetry.trace import Span
|
||||
|
||||
from litellm.caching.caching import RedisCache
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery
|
||||
from litellm.router_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.types.router import (
|
||||
CustomRoutingStrategyBase,
|
||||
Deployment,
|
||||
DeploymentModelListingInfo,
|
||||
DeploymentTypedDict,
|
||||
GuardrailTypedDict,
|
||||
ModelGroupInfo,
|
||||
PreRoutingHookResponse,
|
||||
SearchToolTypedDict,
|
||||
)
|
||||
from litellm.types.utils import ModelInfo as ModelMapInfo
|
||||
|
||||
_T: Final = TypeVar("_T")
|
||||
|
||||
DEPRECATED_ROUTER_MEMBER_USES: Final[Counter[str]] = Counter() # mutable-ok: the live tally _note increments
|
||||
|
||||
_INTERNAL_PREFIX: Final = "litellm.router"
|
||||
|
||||
|
||||
def _note(name: str, replacement: str) -> None:
|
||||
caller: Final = sys._getframe(2)
|
||||
module: Final = caller.f_globals.get("__name__")
|
||||
if isinstance(module, str) and module.startswith(_INTERNAL_PREFIX):
|
||||
return
|
||||
DEPRECATED_ROUTER_MEMBER_USES[name] += 1
|
||||
warnings.warn(
|
||||
f"Router.{name} is deprecated; use Router.{replacement}",
|
||||
DeprecationWarning,
|
||||
stacklevel=3,
|
||||
)
|
||||
|
||||
|
||||
def _as_router(obj: "DeprecatedRouterMembers") -> "Router":
|
||||
"""The Router this mixin is part of (it is mixed into ``litellm.Router`` only)."""
|
||||
from litellm.router import Router
|
||||
|
||||
if isinstance(obj, Router):
|
||||
return obj
|
||||
raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only")
|
||||
|
||||
|
||||
def _public_attribute_error(obj: object, core_name: str, public_name: str, error: AttributeError) -> AttributeError:
|
||||
"""
|
||||
``error`` renamed to the public attribute when it is the router's own ``core_name`` that is
|
||||
missing, so the message is the one reading the old attribute raised; any other error unchanged.
|
||||
"""
|
||||
if error.name not in (core_name, public_name) or error.obj is not obj:
|
||||
return error
|
||||
return AttributeError(f"'{type(obj).__name__}' object has no attribute '{public_name}'", name=public_name, obj=obj)
|
||||
|
||||
|
||||
class _DeprecatedAttribute(Generic[_T]):
|
||||
"""A former Router attribute, read through an entry point. Assignment still replaces it."""
|
||||
|
||||
def __init__(self, replacement: str, read: Callable[["Router"], _T]) -> None:
|
||||
self.replacement = replacement
|
||||
self.read = read
|
||||
self.name = ""
|
||||
|
||||
def __set_name__(self, owner: type, name: str) -> None:
|
||||
# An alias (``_routing_groups = routing_groups``) keeps the first name.
|
||||
self.name = self.name or name
|
||||
|
||||
@overload
|
||||
def __get__(self, obj: None, owner: type) -> object: ...
|
||||
|
||||
@overload
|
||||
def __get__(self, obj: "DeprecatedRouterMembers", owner: type) -> _T: ...
|
||||
|
||||
def __get__(self, obj: "DeprecatedRouterMembers | None", owner: type) -> object:
|
||||
if obj is None:
|
||||
return getattr(owner, "_core_" + self.name, self)
|
||||
_note(self.name, self.replacement)
|
||||
router: Final = _as_router(obj)
|
||||
try:
|
||||
return self.read(router)
|
||||
except AttributeError as e:
|
||||
raise _public_attribute_error(router, "_core_" + self.name, self.name, e) from None
|
||||
|
||||
def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None:
|
||||
_note(self.name, self.replacement)
|
||||
if self.name in SETTING_FIELDS:
|
||||
_as_router(obj).apply(ChangeSet(assign={self.name: value}))
|
||||
else:
|
||||
setattr(obj, "_core_" + self.name, value)
|
||||
|
||||
def __delete__(self, obj: "DeprecatedRouterMembers") -> None:
|
||||
_note(self.name, self.replacement)
|
||||
try:
|
||||
delattr(obj, "_core_" + self.name)
|
||||
except AttributeError:
|
||||
# The only attribute this deletes; Python's own message for it, which sets no name/obj.
|
||||
raise AttributeError(f"'{type(obj).__name__}' object has no attribute '{self.name}'") from None
|
||||
|
||||
|
||||
def _setting(name: str, read: Callable[["Router"], _T]) -> _DeprecatedAttribute[_T]:
|
||||
"""A former settings attribute: ``read`` returns the router's own value, typed as before."""
|
||||
|
||||
def read_setting(router: "Router") -> _T:
|
||||
if not router.settings().is_set(name):
|
||||
raise AttributeError(f"'{type(router).__name__}' object has no attribute {name!r}", name=name, obj=router)
|
||||
return read(router)
|
||||
|
||||
return _DeprecatedAttribute(f"settings().{name}", read_setting)
|
||||
|
||||
|
||||
def _attr(replacement: str, read: Callable[["Router"], _T]) -> _DeprecatedAttribute[_T]:
|
||||
return _DeprecatedAttribute(replacement, read)
|
||||
|
||||
|
||||
# (id(router), member name) pairs whose subclass override is running in this context.
|
||||
_OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] = contextvars.ContextVar(
|
||||
"router_overrides_running", default=frozenset()
|
||||
)
|
||||
|
||||
|
||||
def _route_override(cls: type, name: str, override: Callable[..., object]) -> Callable[..., object]:
|
||||
"""
|
||||
Make the router's own calls to ``_core_<name>`` reach a subclass override of ``<name>``.
|
||||
|
||||
When the override calls ``super().<name>()``, the call comes back to ``_core_<name>``; it is
|
||||
then sent on to the parent's implementation instead of back into the override.
|
||||
"""
|
||||
core: Final = "_core_" + name
|
||||
|
||||
def parent(router: object) -> Callable[..., object]:
|
||||
return getattr(super(cls, router), core)
|
||||
|
||||
if inspect.iscoroutinefunction(override):
|
||||
async_override: Final[Callable[..., Awaitable[object]]] = override
|
||||
|
||||
@functools.wraps(override)
|
||||
async def routed_async(
|
||||
router: object,
|
||||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
|
||||
) -> object:
|
||||
key: Final = (id(router), name)
|
||||
running: Final = _OVERRIDES_RUNNING.get()
|
||||
if key in running:
|
||||
parent_async: Final[Callable[..., Awaitable[object]]] = getattr(super(cls, router), core)
|
||||
return await parent_async(*args, **kwargs)
|
||||
token: Final = _OVERRIDES_RUNNING.set(running | {key})
|
||||
try:
|
||||
return await async_override(router, *args, **kwargs)
|
||||
finally:
|
||||
_OVERRIDES_RUNNING.reset(token)
|
||||
|
||||
return routed_async
|
||||
|
||||
@functools.wraps(override)
|
||||
def routed(
|
||||
router: object,
|
||||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
|
||||
) -> object:
|
||||
key: Final = (id(router), name)
|
||||
running: Final = _OVERRIDES_RUNNING.get()
|
||||
if key in running:
|
||||
return parent(router)(*args, **kwargs)
|
||||
token: Final = _OVERRIDES_RUNNING.set(running | {key})
|
||||
try:
|
||||
return override(router, *args, **kwargs)
|
||||
finally:
|
||||
_OVERRIDES_RUNNING.reset(token)
|
||||
|
||||
return routed
|
||||
|
||||
|
||||
class DeprecatedRouterMembers:
|
||||
"""Old Router members, kept for about a month. Each names its replacement."""
|
||||
|
||||
def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: class keywords, passed on to super()
|
||||
# A Router subclass that overrides an old member (say async_pre_routing_hook) keeps
|
||||
# having its override called by the router, which now calls _core_<name> internally.
|
||||
super().__init_subclass__(**kwargs)
|
||||
for name, value in list(vars(cls).items()):
|
||||
if (
|
||||
inspect.isfunction(value)
|
||||
and not name.startswith("_core_")
|
||||
and name in vars(DeprecatedRouterMembers)
|
||||
and hasattr(cls, "_core_" + name)
|
||||
):
|
||||
setattr(cls, "_core_" + name, _route_override(cls, name, value))
|
||||
|
||||
# -- registry: attributes --------------------------------------------------------------------
|
||||
model_list = _attr("registry().all()", lambda r: r._core_model_list)
|
||||
model_names = _attr("registry().names(kind='group_set')", lambda r: r._core_model_names)
|
||||
deployment_names = _attr("registry().names(kind='deployment_names')", lambda r: r._core_deployment_names)
|
||||
team_public_model_names = _attr("registry().names(kind='team_public')", lambda r: r._core_team_public_model_names)
|
||||
model_name_to_deployment_indices = _attr(
|
||||
"registry().index('model_name')", lambda r: r._core_model_name_to_deployment_indices
|
||||
)
|
||||
team_model_to_deployment_indices = _attr(
|
||||
"registry().index('team_model')", lambda r: r._core_team_model_to_deployment_indices
|
||||
)
|
||||
cached_deployment_model_info = _attr(
|
||||
"registry().deployment_model_info(id, model_name, cached=True)",
|
||||
lambda r: r._core_cached_deployment_model_info,
|
||||
)
|
||||
model_group_alias = _attr("registry().aliases()", lambda r: r._core_model_group_alias)
|
||||
pattern_router = _attr("registry().patterns()", lambda r: r._core_pattern_router)
|
||||
team_pattern_routers = _attr("registry().patterns(team=True)", lambda r: r._core_team_pattern_routers)
|
||||
routing_groups = _attr("registry().routing_groups()", lambda r: r._core_routing_groups)
|
||||
_routing_groups = routing_groups
|
||||
auto_routers = _attr("registry().plugins('auto')", lambda r: r._core_auto_routers)
|
||||
complexity_routers = _attr("registry().plugins('complexity')", lambda r: r._core_complexity_routers)
|
||||
quality_routers = _attr("registry().plugins('quality')", lambda r: r._core_quality_routers)
|
||||
adaptive_routers = _attr("registry().plugins('adaptive')", lambda r: r._core_adaptive_routers)
|
||||
routing_plugins = _attr("registry().plugins('routing')", lambda r: r._core_routing_plugins)
|
||||
prompt_caching_pin_cache = _attr(
|
||||
"registry().plugins('prompt_caching_pins')", lambda r: r._core_prompt_caching_pin_cache
|
||||
)
|
||||
|
||||
# -- health and selection: attributes --------------------------------------------------------
|
||||
cache = _attr("health().store", lambda r: r._core_cache)
|
||||
cooldown_cache = _attr("health().cooldowns", lambda r: r._core_cooldown_cache)
|
||||
health_state_cache = _attr("health().health_checks", lambda r: r._core_health_state_cache)
|
||||
deployment_latency_map = _attr("health().latency", lambda r: r._core_deployment_latency_map)
|
||||
async_get_available_deployment = _attr(
|
||||
"select(model, request_kwargs)",
|
||||
lambda r: r._core_async_get_available_deployment,
|
||||
)
|
||||
|
||||
# -- observers ------------------------------------------------------------------------------
|
||||
slack_alerting_logger = _attr("observe().slack_alerting_logger", lambda r: r._core_slack_alerting_logger)
|
||||
|
||||
# -- settings -------------------------------------------------------------------------------
|
||||
routing_strategy = _setting("routing_strategy", lambda r: r._core_routing_strategy)
|
||||
context_window_fallbacks = _setting("context_window_fallbacks", lambda r: r._core_context_window_fallbacks)
|
||||
content_policy_fallbacks = _setting("content_policy_fallbacks", lambda r: r._core_content_policy_fallbacks)
|
||||
default_deployment = _setting("default_deployment", lambda r: r._core_default_deployment)
|
||||
default_litellm_params = _setting("default_litellm_params", lambda r: r._core_default_litellm_params)
|
||||
default_max_parallel_requests = _setting(
|
||||
"default_max_parallel_requests", lambda r: r._core_default_max_parallel_requests
|
||||
)
|
||||
cooldown_time = _setting("cooldown_time", lambda r: r._core_cooldown_time)
|
||||
allowed_fails = _setting("allowed_fails", lambda r: r._core_allowed_fails)
|
||||
allowed_fails_policy = _setting("allowed_fails_policy", lambda r: r._core_allowed_fails_policy)
|
||||
disable_cooldowns = _setting("disable_cooldowns", lambda r: r._core_disable_cooldowns)
|
||||
enable_pre_call_checks = _setting("enable_pre_call_checks", lambda r: r._core_enable_pre_call_checks)
|
||||
enable_health_check_routing = _setting("enable_health_check_routing", lambda r: r._core_enable_health_check_routing)
|
||||
health_check_ignore_transient_errors = _setting(
|
||||
"health_check_ignore_transient_errors", lambda r: r._core_health_check_ignore_transient_errors
|
||||
)
|
||||
background_health_check_model_groups = _setting(
|
||||
"background_health_check_model_groups", lambda r: r._core_background_health_check_model_groups
|
||||
)
|
||||
enable_tag_filtering = _setting("enable_tag_filtering", lambda r: r._core_enable_tag_filtering)
|
||||
tag_filtering_match_any = _setting("tag_filtering_match_any", lambda r: r._core_tag_filtering_match_any)
|
||||
tag_routing_prefix = _setting("tag_routing_prefix", lambda r: r._core_tag_routing_prefix)
|
||||
router_general_settings = _setting("router_general_settings", lambda r: r._core_router_general_settings)
|
||||
provider_budget_config = _setting("provider_budget_config", lambda r: r._core_provider_budget_config)
|
||||
provider_default_deployment_ids = _setting(
|
||||
"provider_default_deployment_ids", lambda r: r._core_provider_default_deployment_ids
|
||||
)
|
||||
fallback_access_check = _setting("fallback_access_check", lambda r: r._core_fallback_access_check)
|
||||
fallback_budget_check = _setting("fallback_budget_check", lambda r: r._core_fallback_budget_check)
|
||||
# These three are declared with the attribute type Router gave them rather than the descriptor's:
|
||||
# protocols in router_utils and the proxy match Router on them, and protocol matching does not
|
||||
# see through a descriptor. Reads and writes still go through _setting at runtime.
|
||||
fallbacks: list | list[dict[str, list[str] | list | None]] | None = _setting( # pyright: ignore[reportAssignmentType] # see above # mutable-ok: the old type
|
||||
"fallbacks", lambda r: r._core_fallbacks
|
||||
)
|
||||
kubernetes_pod_discovery: "KubernetesPodDiscovery" = _setting( # pyright: ignore[reportAssignmentType] # see above
|
||||
"kubernetes_pod_discovery", lambda r: r._core_kubernetes_pod_discovery
|
||||
)
|
||||
search_tools: "list[SearchToolTypedDict]" = _setting( # pyright: ignore[reportAssignmentType] # see above # mutable-ok: the old type
|
||||
"search_tools", lambda r: r._core_search_tools
|
||||
)
|
||||
guardrail_list = _setting("guardrail_list", lambda r: r._core_guardrail_list)
|
||||
cache_responses = _setting("cache_responses", lambda r: r._core_cache_responses)
|
||||
|
||||
# -- registry: one deployment ---------------------------------------------------------------
|
||||
def get_deployment(self, model_id: str) -> "Deployment | None":
|
||||
_note("get_deployment", "registry().deployment(id)")
|
||||
return _as_router(self)._core_get_deployment(model_id=model_id)
|
||||
|
||||
def has_model_id(self, candidate_id: str) -> bool:
|
||||
_note("has_model_id", "registry().has_deployment(id)")
|
||||
return _as_router(self)._core_has_model_id(candidate_id)
|
||||
|
||||
def get_model_info(self, id: str) -> dict | None: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_info", "registry().deployment_dict(id)")
|
||||
return _as_router(self)._core_get_model_info(id=id)
|
||||
|
||||
def get_deployment_model_info(self, model_id: str, model_name: str) -> "ModelMapInfo | None":
|
||||
_note("get_deployment_model_info", "registry().deployment_model_info(id, model_name)")
|
||||
return _as_router(self)._core_get_deployment_model_info(model_id=model_id, model_name=model_name)
|
||||
|
||||
@overload
|
||||
def get_router_model_info(
|
||||
self,
|
||||
deployment: "dict | Deployment", # mutable-ok: the old parameter type, unchanged
|
||||
received_model_name: str,
|
||||
id: None = None,
|
||||
) -> "ModelMapInfo": ...
|
||||
|
||||
@overload
|
||||
def get_router_model_info(self, deployment: None, received_model_name: str, id: str) -> "ModelMapInfo": ...
|
||||
|
||||
def get_router_model_info(
|
||||
self,
|
||||
deployment: "dict | Deployment | None", # mutable-ok: the old parameter type, unchanged
|
||||
received_model_name: str,
|
||||
id: str | None = None,
|
||||
) -> "ModelMapInfo":
|
||||
_note("get_router_model_info", "registry().router_model_info(...)")
|
||||
router: Final = _as_router(self)
|
||||
# The core's overloads split (deployment, id) by which one is None; this forwards either form unchanged.
|
||||
return router._core_get_router_model_info( # pyright: ignore[reportCallIssue] # see above
|
||||
deployment=deployment, # pyright: ignore[reportArgumentType] # see above
|
||||
received_model_name=received_model_name,
|
||||
id=id, # pyright: ignore[reportArgumentType] # see above
|
||||
)
|
||||
|
||||
def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]:
|
||||
_note("get_discovered_model_info", "registry().discovered_model_info(id)")
|
||||
return _as_router(self)._core_get_discovered_model_info(model_id)
|
||||
|
||||
# -- registry: many deployments -------------------------------------------------------------
|
||||
def get_model_list(
|
||||
self, model_name: str | None = None, team_id: str | None = None
|
||||
) -> "list[DeploymentTypedDict] | None": # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_list", "registry().deployments(model_name, team_id=...)")
|
||||
return _as_router(self)._core_get_model_list(model_name=model_name, team_id=team_id)
|
||||
|
||||
def get_all_deployments(
|
||||
self, model_name: str, model_alias: str | None = None, team_id: str | None = None
|
||||
) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged
|
||||
_note("get_all_deployments", "registry().deployments(model_name, source='all')")
|
||||
return _as_router(self)._core_get_all_deployments(
|
||||
model_name=model_name, model_alias=model_alias, team_id=team_id
|
||||
)
|
||||
|
||||
_get_all_deployments = get_all_deployments
|
||||
|
||||
def config_deployments(self) -> Iterator[Mapping[str, object]]:
|
||||
_note("config_deployments", "registry().deployments(source='config')")
|
||||
return _as_router(self)._core_config_deployments()
|
||||
|
||||
def deployments_named(self, model_name: str) -> "tuple[DeploymentTypedDict, ...] | None":
|
||||
_note("deployments_named", "registry().deployments(model_name, source='named')")
|
||||
return _as_router(self)._core_deployments_named(model_name)
|
||||
|
||||
def get_model_list_from_model_alias(
|
||||
self, model_name: str | None = None
|
||||
) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_list_from_model_alias", "registry().deployments(model_name, source='alias')")
|
||||
return _as_router(self)._core_get_model_list_from_model_alias(model_name=model_name)
|
||||
|
||||
def get_model_list_from_routing_groups(self, model_name: str | None = None) -> "Sequence[DeploymentTypedDict]":
|
||||
_note("get_model_list_from_routing_groups", "registry().deployments(model_name, source='routing_groups')")
|
||||
return _as_router(self)._core_get_model_list_from_routing_groups(model_name=model_name)
|
||||
|
||||
def get_model_list_of_routed_group(
|
||||
self, model_group: str
|
||||
) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_list_of_routed_group", "registry().deployments(model_group, source='routed_group')")
|
||||
return _as_router(self)._core_get_model_list_of_routed_group(model_group)
|
||||
|
||||
def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]:
|
||||
_note("wildcard_route_deployments", "registry().deployments(model, source='wildcard')")
|
||||
return _as_router(self)._core_wildcard_route_deployments(model, team_id)
|
||||
|
||||
def get_routable_deployments(self, model: str) -> "tuple[DeploymentTypedDict, ...]":
|
||||
_note("get_routable_deployments", "registry().deployments(model, source='routable')")
|
||||
return _as_router(self)._core_get_routable_deployments(model)
|
||||
|
||||
def get_deployment_by_model_group_name(self, model_group_name: str) -> "Deployment | None":
|
||||
_note("get_deployment_by_model_group_name", "registry().deployments(name, source='first_in_group')")
|
||||
return _as_router(self)._core_get_deployment_by_model_group_name(model_group_name=model_group_name)
|
||||
|
||||
def deployments_for_request(
|
||||
self, model: str, request_kwargs: Mapping[str, object]
|
||||
) -> "Sequence[DeploymentTypedDict]":
|
||||
_note("deployments_for_request", "registry().deployments(model, source='for_request', request=...)")
|
||||
return _as_router(self)._core_deployments_for_request(model, request_kwargs)
|
||||
|
||||
def should_include_deployment(
|
||||
self,
|
||||
model_name: str,
|
||||
model: "DeploymentTypedDict | dict", # mutable-ok: the old parameter type, unchanged
|
||||
team_id: str | None = None,
|
||||
) -> bool:
|
||||
_note("should_include_deployment", "registry().includes(deployment, model_name=..., team_id=...)")
|
||||
return _as_router(self)._core_should_include_deployment(model_name=model_name, model=model, team_id=team_id)
|
||||
|
||||
@staticmethod
|
||||
def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool:
|
||||
_note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)")
|
||||
from litellm.router import Router
|
||||
|
||||
return Router._core_deployment_usable_by_team(model, team_id)
|
||||
|
||||
_deployment_usable_by_team = deployment_usable_by_team
|
||||
|
||||
# -- registry: names -------------------------------------------------------------------------
|
||||
def get_model_names(self, team_id: str | None = None) -> list[str]: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_names", "registry().names(team_id=...)")
|
||||
return _as_router(self)._core_get_model_names(team_id=team_id)
|
||||
|
||||
def get_model_ids(
|
||||
self, model_name: str | None = None, exclude_team_models: bool = False
|
||||
) -> list[str]: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_ids", "registry().names(model_name, kind='ids')")
|
||||
return _as_router(self)._core_get_model_ids(model_name=model_name, exclude_team_models=exclude_team_models)
|
||||
|
||||
def resolved_litellm_models(self, model_name: str, team_id: str | None = None) -> tuple[str, ...]:
|
||||
_note("resolved_litellm_models", "registry().names(model_name, kind='litellm_models')")
|
||||
return _as_router(self)._core_resolved_litellm_models(model_name, team_id=team_id)
|
||||
|
||||
def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]:
|
||||
_note("wildcard_routed_models", "registry().names(model, kind='wildcard')")
|
||||
return _as_router(self)._core_wildcard_routed_models(model, team_id)
|
||||
|
||||
def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]:
|
||||
_note("get_candidate_model_ids_for_route", "registry().names(model, kind='route_candidates')")
|
||||
return _as_router(self)._core_get_candidate_model_ids_for_route(model, team_id=team_id)
|
||||
|
||||
# -- registry: resolving names ---------------------------------------------------------------
|
||||
def routable_model_group(self, model_name: str) -> str:
|
||||
_note("routable_model_group", "registry().resolve(name)")
|
||||
return _as_router(self)._core_routable_model_group(model_name)
|
||||
|
||||
def get_model_from_alias(self, model: str) -> str | None:
|
||||
_note("get_model_from_alias", "registry().resolve(name, kind='alias')")
|
||||
return _as_router(self)._core_get_model_from_alias(model=model)
|
||||
|
||||
_get_model_from_alias = get_model_from_alias
|
||||
|
||||
def resolve_served_model_name(self, model_name: str, served_model: str) -> str:
|
||||
_note("resolve_served_model_name", "registry().resolve(name, kind='served', served_model=...)")
|
||||
return _as_router(self)._core_resolve_served_model_name(model_name, served_model)
|
||||
|
||||
def resolve_model_name_from_model_id(
|
||||
self, model_id: str | None, custom_llm_provider: str | None = None
|
||||
) -> str | None:
|
||||
_note("resolve_model_name_from_model_id", "registry().resolve(id, kind='from_id')")
|
||||
return _as_router(self)._core_resolve_model_name_from_model_id(
|
||||
model_id, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
def map_team_model(
|
||||
self, team_model_name: str | None, team_id: str, custom_llm_provider: str | None = None
|
||||
) -> str | None:
|
||||
_note("map_team_model", "registry().resolve(name, kind='team_model', team_id=...)")
|
||||
return _as_router(self)._core_map_team_model(team_model_name, team_id, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
def is_recognized_model(self, model: str) -> bool:
|
||||
_note("is_recognized_model", "registry().resolve(name, kind='recognized')")
|
||||
return _as_router(self)._core_is_recognized_model(model)
|
||||
|
||||
def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None:
|
||||
_note("get_routable_upstream_model", "registry().resolve(name, kind='upstream')")
|
||||
return _as_router(self)._core_get_routable_upstream_model(model_name, team_id=team_id)
|
||||
|
||||
def get_model_group(self, id: str) -> list | None: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_group", "registry().resolve(id, kind='group_of_id')")
|
||||
return _as_router(self)._core_get_model_group(id=id)
|
||||
|
||||
def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool:
|
||||
_note("is_model_access_group_for_wildcard_route", "registry().resolve(group, kind='wildcard_access_group')")
|
||||
return _as_router(self)._core_is_model_access_group_for_wildcard_route(model_access_group=model_access_group)
|
||||
|
||||
_is_model_access_group_for_wildcard_route = is_model_access_group_for_wildcard_route
|
||||
|
||||
def team_model_has_alternatives(self, deployment_id: str) -> bool:
|
||||
_note("team_model_has_alternatives", "registry().resolve(id, kind='team_model_alternatives')")
|
||||
return _as_router(self)._core_team_model_has_alternatives(deployment_id)
|
||||
|
||||
def routing_group_has_alternatives(self, model_group: str | None) -> bool:
|
||||
_note("routing_group_has_alternatives", "registry().resolve(group, kind='routing_group_alternatives')")
|
||||
return _as_router(self)._core_routing_group_has_alternatives(model_group)
|
||||
|
||||
# -- registry: groups, access groups, credentials, plugins -----------------------------------
|
||||
def get_model_group_info(
|
||||
self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None
|
||||
) -> "ModelGroupInfo | None":
|
||||
_note("get_model_group_info", "registry().group(name)")
|
||||
return _as_router(self)._core_get_model_group_info(model_group, include_hidden=include_hidden, team_id=team_id)
|
||||
|
||||
def cached_model_group_info(self, model_group: str) -> "ModelGroupInfo | None":
|
||||
_note("cached_model_group_info", "registry().group(name, field='cached_info')")
|
||||
return _as_router(self)._core_cached_model_group_info(model_group)
|
||||
|
||||
def get_configured_display_name(self, model_name: str) -> str | None:
|
||||
_note("get_configured_display_name", "registry().group(name, field='display_name')")
|
||||
return _as_router(self)._core_get_configured_display_name(model_name)
|
||||
|
||||
def get_configured_mode(self, model_name: str) -> str | None:
|
||||
_note("get_configured_mode", "registry().group(name, field='mode')")
|
||||
return _as_router(self)._core_get_configured_mode(model_name)
|
||||
|
||||
def get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]:
|
||||
_note("get_configured_service_tiers", "registry().group(name, field='service_tiers')")
|
||||
return _as_router(self)._core_get_configured_service_tiers(model_name, team_id=team_id)
|
||||
|
||||
def get_configured_token_limits(self, model_name: str) -> tuple[int | None, int | None]:
|
||||
_note("get_configured_token_limits", "registry().group(name, field='token_limits')")
|
||||
return _as_router(self)._core_get_configured_token_limits(model_name)
|
||||
|
||||
def get_model_listing_info(self, model_name: str) -> "DeploymentModelListingInfo | None":
|
||||
_note("get_model_listing_info", "registry().group(name, field='listing_info')")
|
||||
return _as_router(self)._core_get_model_listing_info(model_name)
|
||||
|
||||
def get_model_access_groups(
|
||||
self,
|
||||
model_name: str | None = None,
|
||||
model_access_group: str | None = None,
|
||||
team_id: str | None = None,
|
||||
) -> dict[str, list[str]]: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_access_groups", "registry().access_groups(...)")
|
||||
return _as_router(self)._core_get_model_access_groups(
|
||||
model_name=model_name, model_access_group=model_access_group, team_id=team_id
|
||||
)
|
||||
|
||||
def get_deployment_credentials(self, model_id: str) -> dict | None: # mutable-ok: the old return type, unchanged
|
||||
_note("get_deployment_credentials", "registry().credentials(id)")
|
||||
return _as_router(self)._core_get_deployment_credentials(model_id=model_id)
|
||||
|
||||
def get_deployment_credentials_with_provider(
|
||||
self, model_id: str, team_id: str | None = None
|
||||
) -> "dict[str, Any] | None": # pyright: ignore[reportExplicitAny] # the old return type, unchanged # mutable-ok: same
|
||||
_note("get_deployment_credentials_with_provider", "registry().credentials(id, form='with_provider')")
|
||||
return _as_router(self)._core_get_deployment_credentials_with_provider(model_id=model_id, team_id=team_id)
|
||||
|
||||
def get_credential_deployment(self, model_id: str, team_id: str | None = None) -> "Deployment | None":
|
||||
_note("get_credential_deployment", "registry().credentials(id, form='deployment')")
|
||||
return _as_router(self)._core_get_credential_deployment(model_id=model_id, team_id=team_id)
|
||||
|
||||
def get_router_deployment_budget_limiter(self) -> "RouterBudgetLimiting | None":
|
||||
_note("get_router_deployment_budget_limiter", "registry().plugins('budget_limiter')")
|
||||
return _as_router(self)._core_get_router_deployment_budget_limiter()
|
||||
|
||||
_get_router_deployment_budget_limiter = get_router_deployment_budget_limiter
|
||||
|
||||
@staticmethod
|
||||
def generate_model_id(model_group: str, litellm_params: dict) -> str: # mutable-ok: the old parameter type
|
||||
_note("generate_model_id", "litellm.router_api.deployment_id(model_group, litellm_params)")
|
||||
from litellm.router_api import deployment_id
|
||||
|
||||
return deployment_id(model_group, litellm_params)
|
||||
|
||||
# -- settings: methods -----------------------------------------------------------------------
|
||||
def get_settings(self) -> dict[str, object]: # mutable-ok: Router._core_get_settings's type, a fresh dict
|
||||
_note("get_settings", "settings().as_dict()")
|
||||
return _as_router(self)._core_get_settings()
|
||||
|
||||
def get_allowed_fails_from_policy(self, exception: Exception) -> int | None:
|
||||
_note("get_allowed_fails_from_policy", "settings().allowed_fails_for(exception)")
|
||||
return _as_router(self)._core_get_allowed_fails_from_policy(exception=exception)
|
||||
|
||||
def get_available_guardrail(self, guardrail_name: str) -> "GuardrailTypedDict":
|
||||
_note("get_available_guardrail", "settings().guardrail(name)")
|
||||
return _as_router(self)._core_get_available_guardrail(guardrail_name)
|
||||
|
||||
# -- health ----------------------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def is_deployment_blocked(deployment: "Deployment") -> bool:
|
||||
_note("is_deployment_blocked", "health().is_blocked(deployment)")
|
||||
from litellm.router_api import RouterHealth
|
||||
|
||||
return RouterHealth.is_blocked(deployment)
|
||||
|
||||
_is_deployment_blocked = is_deployment_blocked
|
||||
|
||||
@staticmethod
|
||||
def are_all_deployments_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool:
|
||||
_note("are_all_deployments_blocked", "health().all_blocked(deployments)")
|
||||
from litellm.router_api import RouterHealth
|
||||
|
||||
return RouterHealth.all_blocked(deployments)
|
||||
|
||||
_are_all_deployments_blocked = are_all_deployments_blocked
|
||||
|
||||
def get_fully_blocked_model_names(self) -> set[str]: # mutable-ok: the old return type, unchanged
|
||||
_note("get_fully_blocked_model_names", "health().fully_blocked_model_names()")
|
||||
return _as_router(self)._core_get_fully_blocked_model_names()
|
||||
|
||||
async def async_get_fully_unhealthy_model_names(self) -> set[str]: # mutable-ok: the old return type, unchanged
|
||||
_note("async_get_fully_unhealthy_model_names", "health().fully_unhealthy_model_names()")
|
||||
return await _as_router(self)._core_async_get_fully_unhealthy_model_names()
|
||||
|
||||
async def get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]:
|
||||
_note("get_model_group_usage", "health().usage(group)")
|
||||
return await _as_router(self)._core_get_model_group_usage(model_group=model_group)
|
||||
|
||||
async def get_remaining_model_group_usage(
|
||||
self, model_group: str
|
||||
) -> dict[str, int]: # mutable-ok: the old return type, unchanged
|
||||
_note("get_remaining_model_group_usage", "health().usage(group, kind='remaining')")
|
||||
return await _as_router(self)._core_get_remaining_model_group_usage(model_group=model_group)
|
||||
|
||||
def team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None":
|
||||
_note("team_ptu_ceiling", "health().team_ptu_ceiling(team_id, group)")
|
||||
return _as_router(self)._core_team_ptu_ceiling(team_id, model_group)
|
||||
|
||||
def split_request_priority(
|
||||
self, model: str, kwargs: Mapping[str, object]
|
||||
) -> tuple[int | None, Mapping[str, object]]:
|
||||
_note("split_request_priority", "health().split_priority(model, kwargs)")
|
||||
return _as_router(self)._core_split_request_priority(model, kwargs)
|
||||
|
||||
# -- selection -------------------------------------------------------------------------------
|
||||
async def async_get_healthy_deployments(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict, # mutable-ok: the old parameter type, unchanged
|
||||
messages: list[dict[str, str]] | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
input: str | list | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
specific_deployment: bool | None = False,
|
||||
parent_otel_span: "Span | None" = None,
|
||||
health_check_probe: bool = False,
|
||||
) -> list[dict] | dict: # mutable-ok: the old return type, unchanged
|
||||
_note("async_get_healthy_deployments", "select(model, request_kwargs, kind='healthy')")
|
||||
return await _as_router(self)._core_async_get_healthy_deployments(
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
parent_otel_span=parent_otel_span,
|
||||
health_check_probe=health_check_probe,
|
||||
)
|
||||
|
||||
def get_available_deployment_for_pass_through( # noqa: ANN201 # the old member had no return annotation; its inferred type is kept by inferring it from the core
|
||||
self,
|
||||
model: str,
|
||||
messages: list[dict[str, str]] | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
input: str | list | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
specific_deployment: bool | None = False,
|
||||
request_kwargs: dict | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
):
|
||||
_note("get_available_deployment_for_pass_through", "select(model, kind='pass_through')")
|
||||
return _as_router(self)._core_get_available_deployment_for_pass_through(
|
||||
model=model,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
request_kwargs=request_kwargs,
|
||||
)
|
||||
|
||||
# -- writes ----------------------------------------------------------------------------------
|
||||
def upsert_deployment(self, deployment: "Deployment") -> "Deployment | None":
|
||||
_note("upsert_deployment", "apply(ChangeSet(upsert=[deployment]))")
|
||||
return _as_router(self).apply(ChangeSet(upsert=[deployment])).upserted[0]
|
||||
|
||||
def delete_deployment(self, id: str) -> "Deployment | None":
|
||||
_note("delete_deployment", "apply(ChangeSet(delete=[id]))")
|
||||
return _as_router(self).apply(ChangeSet(delete=[id])).deleted[0]
|
||||
|
||||
def update_settings(self, **kwargs: object) -> None: # kwargs-ok: the old signature, unchanged
|
||||
_note("update_settings", "apply(ChangeSet(settings={...}))")
|
||||
_as_router(self).apply(ChangeSet(settings=kwargs))
|
||||
|
||||
def update_redis_cache(self, cache: "RedisCache") -> None:
|
||||
_note("update_redis_cache", "apply(ChangeSet(redis_cache=cache))")
|
||||
_as_router(self).apply(ChangeSet(redis_cache=cache))
|
||||
|
||||
_update_redis_cache = update_redis_cache
|
||||
|
||||
async def arefresh_model_info(self, *, client: "AsyncHTTPHandler | None" = None) -> None:
|
||||
_note("arefresh_model_info", "aapply(ChangeSet(refresh_model_info=True))")
|
||||
await _as_router(self).aapply(ChangeSet(refresh_model_info=True, refresh_client=client))
|
||||
|
||||
# -- plugins, observers, lifecycle -----------------------------------------------------------
|
||||
def set_custom_routing_strategy(self, CustomRoutingStrategy: "CustomRoutingStrategyBase") -> None:
|
||||
_note("set_custom_routing_strategy", "register_plugin(strategy)")
|
||||
_as_router(self).register_plugin(CustomRoutingStrategy)
|
||||
|
||||
def serve_through_request_logging(self) -> None:
|
||||
_note("serve_through_request_logging", "observe(request_scoped=True)")
|
||||
_as_router(self).observe(request_scoped=True)
|
||||
|
||||
def bind_to_request_logging(self, logging_obj: "LiteLLMLogging") -> None:
|
||||
_note("bind_to_request_logging", "observe(logging_obj)")
|
||||
# Straight to the core, as before: observe() skips binding when logging_obj is None, the old member did not.
|
||||
_as_router(self)._core_bind_to_request_logging(logging_obj)
|
||||
|
||||
def discard(self) -> None:
|
||||
_note("discard", "retire()")
|
||||
_as_router(self).retire()
|
||||
|
||||
# -- plugin context (router strategies) ------------------------------------------------------
|
||||
def arm_routing_read_prefetch(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict[str, object] | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
) -> None:
|
||||
_note("arm_routing_read_prefetch", "RoutingContext.arm_read_prefetch")
|
||||
_as_router(self)._core_arm_routing_read_prefetch(model, request_kwargs)
|
||||
|
||||
async def async_pre_routing_hook(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict[str, object], # mutable-ok: the old parameter type, unchanged
|
||||
messages: "list[dict[str, Any]] | None" = None, # pyright: ignore[reportExplicitAny] # the old type # mutable-ok: same
|
||||
input: str | list | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
specific_deployment: bool | None = False,
|
||||
) -> "PreRoutingHookResponse | None":
|
||||
_note("async_pre_routing_hook", "RoutingContext.pre_routing_hook")
|
||||
return await _as_router(self)._core_async_pre_routing_hook(
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
)
|
||||
|
||||
def log_retry(self, kwargs: dict, e: Exception) -> dict: # mutable-ok: the old signature, unchanged
|
||||
_note("log_retry", "RoutingContext.log_retry")
|
||||
return _as_router(self)._core_log_retry(kwargs=kwargs, e=e)
|
||||
|
|
@ -37,10 +37,12 @@ ALLOWLIST = os.path.join(os.path.dirname(__file__), "router_surface_allowlist.js
|
|||
# Files that make up the router itself. Their access to Router members is internal.
|
||||
ROUTER_FILES = {
|
||||
"litellm/router.py",
|
||||
"litellm/router_api.py",
|
||||
"litellm/router_deprecated.py",
|
||||
}
|
||||
# Classes whose members count as Router members (Router and the mixins it is built from).
|
||||
ROUTER_CLASS_FILES = ["litellm/router.py"]
|
||||
ROUTER_CLASS_NAMES = {"Router"}
|
||||
ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_deprecated.py"]
|
||||
ROUTER_CLASS_NAMES = {"Router", "DeprecatedRouterMembers"}
|
||||
|
||||
SCAN_ROOTS = ["litellm", "enterprise"]
|
||||
SKIP_DIRS = {
|
||||
|
|
|
|||
|
|
@ -53,10 +53,10 @@ class TestFilterHealthCheckUnhealthyDeployments:
|
|||
|
||||
class FakeRouter:
|
||||
def __init__(self):
|
||||
self.enable_health_check_routing = enable
|
||||
self.health_state_cache = health_cache
|
||||
self.allowed_fails_policy = None
|
||||
self.background_health_check_model_groups = model_groups
|
||||
self._core_enable_health_check_routing = enable
|
||||
self._core_health_state_cache = health_cache
|
||||
self._core_allowed_fails_policy = None
|
||||
self._core_background_health_check_model_groups = model_groups
|
||||
|
||||
# Import the actual method and bind it
|
||||
from litellm.router import Router
|
||||
|
|
@ -169,10 +169,10 @@ class TestAsyncFilterHealthCheckUnhealthyDeployments:
|
|||
|
||||
class FakeRouter:
|
||||
def __init__(self):
|
||||
self.enable_health_check_routing = enable
|
||||
self.health_state_cache = health_cache
|
||||
self.allowed_fails_policy = None
|
||||
self.background_health_check_model_groups = model_groups
|
||||
self._core_enable_health_check_routing = enable
|
||||
self._core_health_state_cache = health_cache
|
||||
self._core_allowed_fails_policy = None
|
||||
self._core_background_health_check_model_groups = model_groups
|
||||
|
||||
fake = FakeRouter()
|
||||
fake._async_filter_health_check_unhealthy_deployments = (
|
||||
|
|
|
|||
|
|
@ -2290,7 +2290,7 @@ def test_with_router_resolved_session_model(session, expected):
|
|||
|
||||
def test_router_get_model_access_groups_team_only_models():
|
||||
"""
|
||||
Test that Router.get_model_access_groups returns the correct response for team-only models
|
||||
Test that Router._core_get_model_access_groups returns the correct response for team-only models
|
||||
"""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
|
|
@ -2409,7 +2409,7 @@ def test_model_group_info_cost_from_db_model_info():
|
|||
)
|
||||
|
||||
with patch.object(
|
||||
router, "get_deployment_model_info", side_effect=Exception("not found")
|
||||
router, "_core_get_deployment_model_info", side_effect=Exception("not found")
|
||||
):
|
||||
result = router._cached_get_model_group_info("my-custom-model")
|
||||
assert result is not None
|
||||
|
|
@ -2439,7 +2439,7 @@ def test_model_group_info_cost_none_when_db_model_info_has_no_cost():
|
|||
)
|
||||
|
||||
with patch.object(
|
||||
router, "get_deployment_model_info", side_effect=Exception("not found")
|
||||
router, "_core_get_deployment_model_info", side_effect=Exception("not found")
|
||||
):
|
||||
result = router._cached_get_model_group_info("my-custom-model-no-cost")
|
||||
assert result is not None
|
||||
|
|
@ -2823,7 +2823,7 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto
|
|||
},
|
||||
)
|
||||
assert router.routing_strategy == "lar1"
|
||||
assert "async_get_available_deployment" in router.__dict__
|
||||
assert "_core_async_get_available_deployment" in router.__dict__
|
||||
|
||||
router._switch_routing_strategy("usage-based-routing-v2", {})
|
||||
assert router.lowesttpm_logger_v2 is not None
|
||||
|
|
@ -2894,7 +2894,7 @@ def test_model_group_info_with_stringified_cost_values():
|
|||
return None
|
||||
|
||||
with patch.object(
|
||||
router, "get_deployment_model_info", side_effect=_model_info_with_str_costs
|
||||
router, "_core_get_deployment_model_info", side_effect=_model_info_with_str_costs
|
||||
):
|
||||
result = router._set_model_group_info(
|
||||
model_group="my-custom-model",
|
||||
|
|
@ -2942,7 +2942,7 @@ def test_model_group_info_db_fallback_with_stringified_cost_values():
|
|||
)
|
||||
|
||||
with patch.object(
|
||||
router, "get_deployment_model_info", side_effect=Exception("not found")
|
||||
router, "_core_get_deployment_model_info", side_effect=Exception("not found")
|
||||
):
|
||||
result = router._set_model_group_info(
|
||||
model_group="my-custom-model",
|
||||
|
|
@ -7371,7 +7371,7 @@ async def test_async_function_with_fallbacks_common_utils():
|
|||
|
||||
|
||||
def test_should_include_deployment():
|
||||
"""Test that Router.should_include_deployment returns the correct response"""
|
||||
"""Test that Router._core_should_include_deployment returns the correct response"""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -7426,7 +7426,7 @@ def test_pre_call_checks_skips_token_count_without_max_input_tokens(monkeypatch)
|
|||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: {})
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: {})
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -7460,7 +7460,7 @@ def test_pre_call_checks_counts_once_and_filters_on_max_input_tokens(monkeypatch
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
calls = []
|
||||
|
|
@ -7494,7 +7494,7 @@ def test_pre_call_checks_uses_precounted_tokens(monkeypatch):
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
calls = []
|
||||
|
|
@ -7528,7 +7528,7 @@ async def test_async_get_healthy_deployments_counts_tokens_off_the_event_loop(mo
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 1_000_000}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 1_000_000}
|
||||
)
|
||||
|
||||
counting_threads = []
|
||||
|
|
@ -7578,7 +7578,7 @@ def test_pre_call_checks_need_token_count(monkeypatch, model_info, expected):
|
|||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: model_info)
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: model_info)
|
||||
|
||||
deployments = [
|
||||
{"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "d1"}},
|
||||
|
|
@ -7604,7 +7604,7 @@ def test_deployment_max_input_tokens_survives_an_unmappable_deployment(monkeypat
|
|||
raise ValueError("This model isn't mapped yet.")
|
||||
return {"max_input_tokens": 100}
|
||||
|
||||
monkeypatch.setattr(router, "get_router_model_info", flaky_model_info)
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", flaky_model_info)
|
||||
|
||||
unmapped = {"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "unmapped"}}
|
||||
mapped = {"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "mapped"}}
|
||||
|
|
@ -7626,7 +7626,7 @@ def test_pre_call_checks_does_not_recount_inline_after_an_off_loop_failure(monke
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
calls = []
|
||||
|
|
@ -7661,7 +7661,7 @@ async def test_async_get_healthy_deployments_never_recounts_on_the_loop(monkeypa
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
counting_threads = []
|
||||
|
|
@ -7698,7 +7698,7 @@ async def test_acount_pre_call_check_tokens_leaves_the_event_loop_free(monkeypat
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
deployments = [
|
||||
|
|
@ -7732,7 +7732,7 @@ async def test_acount_pre_call_check_tokens_skips_without_max_input_tokens(monke
|
|||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: {})
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: {})
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -7766,7 +7766,7 @@ def test_pre_call_checks_counts_tokens_from_responses_input_string(monkeypatch):
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 1}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 1}
|
||||
)
|
||||
|
||||
deployments = [
|
||||
|
|
@ -7793,7 +7793,7 @@ def test_pre_call_checks_counts_tokens_from_responses_input_list(monkeypatch):
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 1}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 1}
|
||||
)
|
||||
|
||||
deployments = [
|
||||
|
|
@ -7837,7 +7837,7 @@ def test_pre_call_checks_counts_responses_instructions_tokens(monkeypatch):
|
|||
assert with_instructions_tokens > input_only_tokens
|
||||
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": input_only_tokens}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": input_only_tokens}
|
||||
)
|
||||
with pytest.raises(litellm.ContextWindowExceededError):
|
||||
router._pre_call_checks(
|
||||
|
|
@ -7908,7 +7908,7 @@ def test_pre_call_checks_counts_tool_definition_tokens(monkeypatch, prompt_kwarg
|
|||
messages=prompt_kwargs.get("messages"), input=prompt_kwargs.get("input")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": prompt_only_tokens}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": prompt_only_tokens}
|
||||
)
|
||||
|
||||
assert len(router._pre_call_checks(model="m", healthy_deployments=deployments, **prompt_kwargs)) == 1
|
||||
|
|
@ -7949,7 +7949,7 @@ def test_pre_call_checks_counts_anthropic_system_tokens(monkeypatch, system):
|
|||
messages = [{"role": "user", "content": "hi"}]
|
||||
|
||||
messages_only_tokens = router._count_pre_call_check_tokens(messages=messages, input=None)
|
||||
monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": messages_only_tokens})
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": messages_only_tokens})
|
||||
|
||||
assert len(router._pre_call_checks(model="m", healthy_deployments=deployments, messages=messages)) == 1
|
||||
with pytest.raises(litellm.ContextWindowExceededError):
|
||||
|
|
@ -8068,7 +8068,7 @@ def test_pre_call_checks_no_messages_or_input_does_not_crash(monkeypatch):
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
counted: list[dict] = []
|
||||
|
|
@ -9415,7 +9415,7 @@ def test_deployment_usable_by_team_uses_dynamic_model_info_get():
|
|||
def get(self, key: str) -> str | None:
|
||||
return "team-b" if key == "team_id" else None
|
||||
|
||||
assert litellm.Router.deployment_usable_by_team({"model_info": ModelInfo()}, "team-a") is False
|
||||
assert litellm.Router._core_deployment_usable_by_team({"model_info": ModelInfo()}, "team-a") is False
|
||||
|
||||
|
||||
def test_get_deployment_credentials_with_provider_skips_other_team_wildcard():
|
||||
|
|
@ -11062,7 +11062,7 @@ def test_access_group_filter_empty_does_not_bypass_via_litellm_model_fallback(
|
|||
team_id=team_id,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(router, "get_model_access_groups", fake_get_model_access_groups)
|
||||
monkeypatch.setattr(router, "_core_get_model_access_groups", fake_get_model_access_groups)
|
||||
|
||||
scoped_key = UserAPIKeyAuth(
|
||||
api_key="hashed-key",
|
||||
|
|
@ -11139,7 +11139,7 @@ def test_access_group_block_does_not_silently_use_default_fallback_model(
|
|||
team_id=team_id,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(router, "get_model_access_groups", fake_get_model_access_groups)
|
||||
monkeypatch.setattr(router, "_core_get_model_access_groups", fake_get_model_access_groups)
|
||||
|
||||
scoped_key = UserAPIKeyAuth(
|
||||
api_key="hashed-key",
|
||||
|
|
@ -11208,7 +11208,7 @@ def test_access_group_block_via_litellm_model_branch_does_not_use_default_fallba
|
|||
team_id=team_id,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(router, "get_model_access_groups", fake_get_model_access_groups)
|
||||
monkeypatch.setattr(router, "_core_get_model_access_groups", fake_get_model_access_groups)
|
||||
|
||||
scoped_key = UserAPIKeyAuth(
|
||||
api_key="hashed-key",
|
||||
|
|
@ -11709,7 +11709,7 @@ def test_get_deployment_credentials_with_provider_returns_none_for_blocked_deplo
|
|||
|
||||
def test_is_deployment_blocked_static_helper_reflects_blocked_flag():
|
||||
"""
|
||||
Exercises Router.is_deployment_blocked so router_code_coverage.py (AST call graph)
|
||||
Exercises Router._core_is_deployment_blocked so router_code_coverage.py (AST call graph)
|
||||
marks the helper as covered by router-named tests.
|
||||
"""
|
||||
import types
|
||||
|
|
@ -11720,15 +11720,15 @@ def test_is_deployment_blocked_static_helper_reflects_blocked_flag():
|
|||
blocked_dep = router.get_deployment("dep-0")
|
||||
unblocked_dep = router.get_deployment("dep-1")
|
||||
assert blocked_dep is not None and unblocked_dep is not None
|
||||
assert litellm.Router.is_deployment_blocked(blocked_dep) is True
|
||||
assert litellm.Router.is_deployment_blocked(unblocked_dep) is False
|
||||
assert litellm.Router._core_is_deployment_blocked(blocked_dep) is True
|
||||
assert litellm.Router._core_is_deployment_blocked(unblocked_dep) is False
|
||||
|
||||
# No model_info on deployment object → treated as not blocked
|
||||
assert litellm.Router.is_deployment_blocked(object()) is False
|
||||
assert litellm.Router._core_is_deployment_blocked(object()) is False
|
||||
missing_blocked = types.SimpleNamespace()
|
||||
assert litellm.Router.is_deployment_blocked(types.SimpleNamespace(model_info=missing_blocked)) is False
|
||||
assert litellm.Router._core_is_deployment_blocked(types.SimpleNamespace(model_info=missing_blocked)) is False
|
||||
assert (
|
||||
litellm.Router.is_deployment_blocked(types.SimpleNamespace(model_info=types.SimpleNamespace(blocked=True)))
|
||||
litellm.Router._core_is_deployment_blocked(types.SimpleNamespace(model_info=types.SimpleNamespace(blocked=True)))
|
||||
is True
|
||||
)
|
||||
|
||||
|
|
@ -14319,7 +14319,7 @@ def test_pre_call_checks_uses_deployment_model_when_model_info_lookup_raises(mon
|
|||
def _raise_unmapped(**kwargs):
|
||||
raise ValueError("This model isn't mapped yet")
|
||||
|
||||
monkeypatch.setattr(router, "get_router_model_info", _raise_unmapped)
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", _raise_unmapped)
|
||||
|
||||
seen: list[tuple] = []
|
||||
original_get_supported_openai_params = litellm.get_supported_openai_params
|
||||
|
|
@ -16510,7 +16510,7 @@ def test_model_group_info_intersects_supported_reasoning_efforts():
|
|||
"supports_xhigh_reasoning_effort": False,
|
||||
}
|
||||
|
||||
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
|
||||
with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info):
|
||||
result = router._set_model_group_info(
|
||||
model_group="smart-group",
|
||||
user_facing_model_group_name="smart-group",
|
||||
|
|
@ -16554,7 +16554,7 @@ def test_model_group_info_reasoning_efforts_are_unknown_when_any_deployment_is_o
|
|||
}
|
||||
return {"key": model_name, "litellm_provider": "openai", "mode": None, "supports_reasoning": None}
|
||||
|
||||
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
|
||||
with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info):
|
||||
result = router._set_model_group_info(
|
||||
model_group="smart-group",
|
||||
user_facing_model_group_name="smart-group",
|
||||
|
|
@ -16689,7 +16689,7 @@ def test_model_group_info_reasoning_efforts_empty_on_a_mapped_non_reasoning_depl
|
|||
}
|
||||
return {"key": model_name, "litellm_provider": "openai", "mode": "chat", "supports_reasoning": None}
|
||||
|
||||
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
|
||||
with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info):
|
||||
result = router._set_model_group_info(
|
||||
model_group="mixed-group",
|
||||
user_facing_model_group_name="mixed-group",
|
||||
|
|
@ -16730,7 +16730,7 @@ def test_model_group_info_reasoning_efforts_ignore_a_value_declared_in_model_inf
|
|||
info["supported_reasoning_efforts"] = ("high",)
|
||||
return info
|
||||
|
||||
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
|
||||
with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info):
|
||||
result = router._set_model_group_info(
|
||||
model_group="declared-group",
|
||||
user_facing_model_group_name="declared-group",
|
||||
|
|
@ -16764,7 +16764,7 @@ def test_model_group_info_survives_a_junk_typed_operator_effort_value():
|
|||
"supported_reasoning_efforts": "high",
|
||||
}
|
||||
|
||||
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
|
||||
with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info):
|
||||
result = router._set_model_group_info(
|
||||
model_group="junk-declared-group",
|
||||
user_facing_model_group_name="junk-declared-group",
|
||||
|
|
@ -23151,7 +23151,7 @@ def test_a_failed_routing_read_prefetch_logs_the_request_model_without_its_line_
|
|||
def fail_lookup(model_name: str | None = None, team_id: str | None = None) -> None:
|
||||
raise RuntimeError(f"no deployments for {model_name}")
|
||||
|
||||
monkeypatch.setattr(router, "get_model_list", fail_lookup)
|
||||
monkeypatch.setattr(router, "_core_get_model_list", fail_lookup)
|
||||
caplog.clear()
|
||||
|
||||
with caplog.at_level(logging.DEBUG, logger="LiteLLM Router"):
|
||||
|
|
@ -25161,7 +25161,7 @@ async def test_router_acancel_batch_removes_unresolved_credential_name():
|
|||
mock_response.status = "cancelled"
|
||||
|
||||
with (
|
||||
patch.object(router, "get_deployment_credentials_with_provider", return_value=None),
|
||||
patch.object(router, "_core_get_deployment_credentials_with_provider", return_value=None),
|
||||
patch.object(litellm, "acancel_batch", new_callable=AsyncMock) as mock_cancel,
|
||||
):
|
||||
mock_cancel.return_value = mock_response
|
||||
|
|
@ -28197,7 +28197,7 @@ def test_reset_custom_routing_strategy():
|
|||
router.set_custom_routing_strategy(SpecialEndpointRoutingStrategy(router))
|
||||
|
||||
assert "get_available_deployment" in router.__dict__
|
||||
assert "async_get_available_deployment" in router.__dict__
|
||||
assert "_core_async_get_available_deployment" in router.__dict__
|
||||
|
||||
router._reset_custom_routing_strategy()
|
||||
|
||||
|
|
@ -30306,7 +30306,7 @@ def test_team_model_has_alternatives_when_a_delete_starts_after_its_deployment_l
|
|||
has_alternatives: Final = _read_during_held_write(
|
||||
lambda: router.team_model_has_alternatives("deployment-3"),
|
||||
lambda: router.delete_deployment("deployment-0"),
|
||||
write_after=Router.get_deployment.__code__,
|
||||
write_after=Router._core_get_deployment.__code__,
|
||||
)
|
||||
|
||||
assert has_alternatives is True
|
||||
|
|
|
|||
386
tests/unit/test_router/test_router_entry_points.py
Normal file
386
tests/unit/test_router/test_router_entry_points.py
Normal file
|
|
@ -0,0 +1,386 @@
|
|||
"""
|
||||
Router entry points and DeprecatedRouterMembers.
|
||||
|
||||
Every member that moved into DeprecatedRouterMembers must return what its entry point returns,
|
||||
warn once per access from outside the router, and count the access. Internal access (the router's
|
||||
own modules) must stay silent.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.router_api import SETTING_FIELDS, ChangeSet, RouterSettings
|
||||
from litellm.router_deprecated import DEPRECATED_ROUTER_MEMBER_USES, DeprecatedRouterMembers
|
||||
from litellm.types.router import Deployment
|
||||
|
||||
|
||||
def _router() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"},
|
||||
"model_info": {"id": "dep-a", "access_groups": ["grp"]},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt",
|
||||
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-b"},
|
||||
"model_info": {"id": "dep-b"},
|
||||
},
|
||||
{
|
||||
"model_name": "team-internal",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-t"},
|
||||
"model_info": {"id": "dep-team", "team_id": "team-1", "team_public_model_name": "team-gpt"},
|
||||
},
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {"model": "openai/*", "api_key": "sk-w"},
|
||||
"model_info": {"id": "dep-wild"},
|
||||
},
|
||||
],
|
||||
model_group_alias={"alias-gpt": "gpt"},
|
||||
fallbacks=[{"gpt": ["team-internal"]}],
|
||||
cooldown_time=7,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def router():
|
||||
r = _router()
|
||||
yield r
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
r.retire()
|
||||
|
||||
|
||||
# (old member, how to read it through the old name, how to read it through the entry point)
|
||||
READS = [
|
||||
("model_list", lambda r: r.model_list, lambda r: r.registry().all()),
|
||||
("model_names", lambda r: r.model_names, lambda r: r.registry().names(kind="group_set")),
|
||||
("deployment_names", lambda r: r.deployment_names, lambda r: r.registry().names(kind="deployment_names")),
|
||||
("team_public_model_names", lambda r: r.team_public_model_names, lambda r: r.registry().names(kind="team_public")),
|
||||
(
|
||||
"model_name_to_deployment_indices",
|
||||
lambda r: r.model_name_to_deployment_indices,
|
||||
lambda r: r.registry().index("model_name"),
|
||||
),
|
||||
(
|
||||
"team_model_to_deployment_indices",
|
||||
lambda r: r.team_model_to_deployment_indices,
|
||||
lambda r: r.registry().index("team_model"),
|
||||
),
|
||||
("model_group_alias", lambda r: r.model_group_alias, lambda r: r.registry().aliases()),
|
||||
("pattern_router", lambda r: r.pattern_router, lambda r: r.registry().patterns()),
|
||||
("team_pattern_routers", lambda r: r.team_pattern_routers, lambda r: r.registry().patterns(team=True)),
|
||||
("routing_groups", lambda r: r.routing_groups, lambda r: r.registry().routing_groups()),
|
||||
("auto_routers", lambda r: r.auto_routers, lambda r: r.registry().plugins("auto")),
|
||||
("complexity_routers", lambda r: r.complexity_routers, lambda r: r.registry().plugins("complexity")),
|
||||
("quality_routers", lambda r: r.quality_routers, lambda r: r.registry().plugins("quality")),
|
||||
("adaptive_routers", lambda r: r.adaptive_routers, lambda r: r.registry().plugins("adaptive")),
|
||||
("routing_plugins", lambda r: r.routing_plugins, lambda r: r.registry().plugins("routing")),
|
||||
(
|
||||
"prompt_caching_pin_cache",
|
||||
lambda r: r.prompt_caching_pin_cache,
|
||||
lambda r: r.registry().plugins("prompt_caching_pins"),
|
||||
),
|
||||
("cache", lambda r: r.cache, lambda r: r.health().store),
|
||||
("cooldown_cache", lambda r: r.cooldown_cache, lambda r: r.health().cooldowns),
|
||||
("health_state_cache", lambda r: r.health_state_cache, lambda r: r.health().health_checks),
|
||||
("deployment_latency_map", lambda r: r.deployment_latency_map, lambda r: r.health().latency),
|
||||
("get_deployment", lambda r: r.get_deployment("dep-a"), lambda r: r.registry().deployment("dep-a")),
|
||||
("has_model_id", lambda r: r.has_model_id("dep-b"), lambda r: r.registry().has_deployment("dep-b")),
|
||||
("get_model_info", lambda r: r.get_model_info("dep-a"), lambda r: r.registry().deployment_dict("dep-a")),
|
||||
("get_model_list", lambda r: r.get_model_list("gpt"), lambda r: r.registry().deployments("gpt")),
|
||||
("get_model_list_all", lambda r: r.get_model_list(), lambda r: r.registry().deployments()),
|
||||
("get_model_list_alias", lambda r: r.get_model_list("alias-gpt"), lambda r: r.registry().deployments("alias-gpt")),
|
||||
(
|
||||
"get_all_deployments",
|
||||
lambda r: r.get_all_deployments("gpt"),
|
||||
lambda r: r.registry().deployments("gpt", source="all"),
|
||||
),
|
||||
(
|
||||
"config_deployments",
|
||||
lambda r: list(r.config_deployments()),
|
||||
lambda r: list(r.registry().deployments(source="config")),
|
||||
),
|
||||
(
|
||||
"deployments_named",
|
||||
lambda r: r.deployments_named("gpt"),
|
||||
lambda r: r.registry().deployments("gpt", source="named"),
|
||||
),
|
||||
(
|
||||
"get_model_list_from_model_alias",
|
||||
lambda r: r.get_model_list_from_model_alias("alias-gpt"),
|
||||
lambda r: r.registry().deployments("alias-gpt", source="alias"),
|
||||
),
|
||||
(
|
||||
"get_deployment_by_model_group_name",
|
||||
lambda r: r.get_deployment_by_model_group_name("gpt"),
|
||||
lambda r: r.registry().deployments("gpt", source="first_in_group"),
|
||||
),
|
||||
("get_model_names", lambda r: r.get_model_names(), lambda r: r.registry().names()),
|
||||
(
|
||||
"get_model_names_team",
|
||||
lambda r: r.get_model_names(team_id="team-1"),
|
||||
lambda r: r.registry().names(team_id="team-1"),
|
||||
),
|
||||
("get_model_ids", lambda r: r.get_model_ids("gpt"), lambda r: r.registry().names("gpt", kind="ids")),
|
||||
(
|
||||
"get_model_ids_all",
|
||||
lambda r: r.get_model_ids(exclude_team_models=True),
|
||||
lambda r: r.registry().names(kind="ids", exclude_team_models=True),
|
||||
),
|
||||
(
|
||||
"get_model_from_alias",
|
||||
lambda r: r.get_model_from_alias("alias-gpt"),
|
||||
lambda r: r.registry().resolve("alias-gpt", kind="alias"),
|
||||
),
|
||||
(
|
||||
"is_recognized_model",
|
||||
lambda r: r.is_recognized_model("gpt"),
|
||||
lambda r: r.registry().resolve("gpt", kind="recognized"),
|
||||
),
|
||||
(
|
||||
"routable_model_group",
|
||||
lambda r: r.routable_model_group("alias-gpt"),
|
||||
lambda r: r.registry().resolve("alias-gpt"),
|
||||
),
|
||||
(
|
||||
"map_team_model",
|
||||
lambda r: r.map_team_model("team-gpt", "team-1"),
|
||||
lambda r: r.registry().resolve("team-gpt", kind="team_model", team_id="team-1"),
|
||||
),
|
||||
(
|
||||
"get_model_group",
|
||||
lambda r: r.get_model_group("dep-a"),
|
||||
lambda r: r.registry().resolve("dep-a", kind="group_of_id"),
|
||||
),
|
||||
("get_model_access_groups", lambda r: r.get_model_access_groups(), lambda r: r.registry().access_groups()),
|
||||
(
|
||||
"get_deployment_credentials",
|
||||
lambda r: r.get_deployment_credentials("dep-a"),
|
||||
lambda r: r.registry().credentials("dep-a"),
|
||||
),
|
||||
(
|
||||
"get_deployment_credentials_with_provider",
|
||||
lambda r: r.get_deployment_credentials_with_provider("dep-a"),
|
||||
lambda r: r.registry().credentials("dep-a", form="with_provider"),
|
||||
),
|
||||
(
|
||||
"get_credential_deployment",
|
||||
lambda r: r.get_credential_deployment("dep-a"),
|
||||
lambda r: r.registry().credentials("dep-a", form="deployment"),
|
||||
),
|
||||
("get_configured_mode", lambda r: r.get_configured_mode("gpt"), lambda r: r.registry().group("gpt", field="mode")),
|
||||
(
|
||||
"get_configured_token_limits",
|
||||
lambda r: r.get_configured_token_limits("gpt"),
|
||||
lambda r: r.registry().group("gpt", field="token_limits"),
|
||||
),
|
||||
(
|
||||
"get_fully_blocked_model_names",
|
||||
lambda r: r.get_fully_blocked_model_names(),
|
||||
lambda r: r.health().fully_blocked_model_names(),
|
||||
),
|
||||
("get_settings", lambda r: r.get_settings(), lambda r: r.settings().as_dict()),
|
||||
("fallbacks", lambda r: r.fallbacks, lambda r: r.settings().fallbacks),
|
||||
("cooldown_time", lambda r: r.cooldown_time, lambda r: r.settings().cooldown_time),
|
||||
("routing_strategy", lambda r: r.routing_strategy, lambda r: r.settings().routing_strategy),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name, old, new", READS, ids=[r[0] for r in READS])
|
||||
def test_deprecated_read_matches_entry_point(router, name, old, new):
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
old_value = old(router)
|
||||
new_value = new(router)
|
||||
if isinstance(old_value, (list, dict, set)) or hasattr(old_value, "__dict__"):
|
||||
assert old_value == new_value
|
||||
else:
|
||||
assert old_value == new_value
|
||||
assert any(issubclass(w.category, DeprecationWarning) for w in caught), name
|
||||
|
||||
|
||||
def test_live_objects_not_copies(router):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert router.model_list is router.registry().all()
|
||||
assert router.fallbacks is router.settings().fallbacks
|
||||
assert router.cache is router.health().store
|
||||
assert router.model_group_alias is router.registry().aliases()
|
||||
|
||||
|
||||
def test_assignment_goes_through_apply(router):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
router.fallbacks = [{"gpt": ["openai/gpt-4.1"]}]
|
||||
assert router.settings().fallbacks == [{"gpt": ["openai/gpt-4.1"]}]
|
||||
router.search_tools = [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily"}}]
|
||||
assert router.settings().search_tools[0]["search_tool_name"] == "s"
|
||||
router.apply(ChangeSet(assign={"cooldown_time": 3}))
|
||||
assert router.settings().cooldown_time == 3
|
||||
with pytest.raises(ValueError, match="not assignable router settings"):
|
||||
router.apply(ChangeSet(assign={"not_a_setting": 1}))
|
||||
|
||||
|
||||
def test_apply_upsert_and_delete_bump_version(router):
|
||||
v0 = router.registry().version
|
||||
added = Deployment(
|
||||
model_name="new-group",
|
||||
litellm_params={"model": "openai/gpt-4o-mini", "api_key": "sk-n"},
|
||||
model_info={"id": "dep-new"},
|
||||
)
|
||||
result = router.apply(ChangeSet(upsert=[added]))
|
||||
assert result.version == v0 + 1
|
||||
assert router.registry().has_deployment("dep-new")
|
||||
assert router.registry().deployments("new-group")[0]["model_info"]["id"] == "dep-new"
|
||||
result = router.apply(ChangeSet(delete=["dep-new"]))
|
||||
assert result.version == v0 + 2
|
||||
assert result.deleted[0] is not None
|
||||
assert not router.registry().has_deployment("dep-new")
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert router.upsert_deployment(added) is added
|
||||
assert router.upsert_deployment(added) is None # unchanged
|
||||
assert router.delete_deployment("dep-new") is not None
|
||||
|
||||
|
||||
def test_update_settings_through_apply(router):
|
||||
router.apply(ChangeSet(settings={"cooldown_time": "11", "not_allowed": 1}))
|
||||
assert router.settings().cooldown_time == 11
|
||||
|
||||
|
||||
def test_settings_snapshot_is_frozen(router):
|
||||
snapshot = router.settings()
|
||||
assert isinstance(snapshot, RouterSettings)
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
snapshot.cooldown_time = 1 # type: ignore[misc]
|
||||
assert set(SETTING_FIELDS) <= set(RouterSettings.__dataclass_fields__)
|
||||
|
||||
|
||||
def test_unset_setting_raises_attribute_error_through_old_name():
|
||||
r = _router()
|
||||
try:
|
||||
del r.__dict__["_core_kubernetes_pod_discovery"]
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert getattr(r, "kubernetes_pod_discovery", "missing") == "missing"
|
||||
assert r.settings().is_set("kubernetes_pod_discovery") is False
|
||||
finally:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
r.retire()
|
||||
|
||||
|
||||
def test_unset_attribute_error_names_the_public_attribute(router):
|
||||
# Reading (or deleting) an old attribute the router never set raises what reading it raised before.
|
||||
router.__dict__.pop("_core_slack_alerting_logger", None)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
with pytest.raises(AttributeError) as read:
|
||||
_ = router.slack_alerting_logger
|
||||
with pytest.raises(AttributeError) as deleted:
|
||||
del router.slack_alerting_logger
|
||||
del router.__dict__["_core_kubernetes_pod_discovery"]
|
||||
with pytest.raises(AttributeError) as setting:
|
||||
_ = router.kubernetes_pod_discovery
|
||||
for caught, name in ((read, "slack_alerting_logger"), (setting, "kubernetes_pod_discovery")):
|
||||
assert str(caught.value) == f"'Router' object has no attribute '{name}'"
|
||||
assert caught.value.name == name
|
||||
assert caught.value.obj is router
|
||||
# A failed del sets no name/obj in Python either.
|
||||
assert str(deleted.value) == "'Router' object has no attribute 'slack_alerting_logger'"
|
||||
|
||||
|
||||
def test_bind_to_request_logging_none_raises_as_before(router):
|
||||
# The old member bound unconditionally, so None fails on the first callback registration.
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
with pytest.raises(AttributeError, match="add_dynamic_function_callbacks"):
|
||||
router.bind_to_request_logging(None)
|
||||
|
||||
|
||||
def test_select_picks_like_async_get_available_deployment(router):
|
||||
picked = asyncio.run(router.select("gpt", {}))
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
old = asyncio.run(router.async_get_available_deployment(model="gpt", request_kwargs={}))
|
||||
assert picked["model_name"] == old["model_name"] == "gpt"
|
||||
healthy = asyncio.run(router.select("gpt", {}, kind="healthy"))
|
||||
assert {d["model_info"]["id"] for d in healthy} == {"dep-a", "dep-b"}
|
||||
|
||||
|
||||
def test_counter_and_internal_silence(router):
|
||||
before = DEPRECATED_ROUTER_MEMBER_USES["get_model_ids"]
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
router.get_model_ids()
|
||||
# Router internals call the core names; a completion must not warn.
|
||||
asyncio.run(router.acompletion(model="gpt", messages=[{"role": "user", "content": "hi"}], mock_response="ok"))
|
||||
assert DEPRECATED_ROUTER_MEMBER_USES["get_model_ids"] == before + 1
|
||||
deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning) and "Router." in str(w.message)]
|
||||
assert [str(w.message).split(" ")[0] for w in deprecations] == ["Router.get_model_ids"]
|
||||
assert deprecations[0].filename == __file__
|
||||
|
||||
|
||||
def test_every_deprecated_member_names_a_replacement():
|
||||
import inspect
|
||||
|
||||
from litellm.router_deprecated import _DeprecatedAttribute
|
||||
|
||||
for name, member in vars(DeprecatedRouterMembers).items():
|
||||
if name.startswith("__"):
|
||||
continue
|
||||
if isinstance(member, _DeprecatedAttribute):
|
||||
assert member.replacement
|
||||
continue
|
||||
func = member.__func__ if isinstance(member, staticmethod) else member
|
||||
if callable(func):
|
||||
assert "_note(" in inspect.getsource(func), name
|
||||
|
||||
|
||||
def test_generate_model_id_is_the_module_function():
|
||||
from litellm.router_api import deployment_id
|
||||
|
||||
params = {"model": "openai/gpt-4o", "api_key": "k", "extra": {"a": 1}}
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert Router.generate_model_id("g", params) == deployment_id("g", params)
|
||||
|
||||
|
||||
def test_subclass_override_of_an_old_member_is_still_called():
|
||||
calls = []
|
||||
|
||||
class Sub(litellm.Router):
|
||||
async def async_pre_routing_hook(
|
||||
self, model, request_kwargs, messages=None, input=None, specific_deployment=False
|
||||
):
|
||||
calls.append(model)
|
||||
return await super().async_pre_routing_hook(model, request_kwargs, messages, input, specific_deployment)
|
||||
|
||||
r = Sub(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "m",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x", "mock_response": "ok"},
|
||||
}
|
||||
]
|
||||
)
|
||||
response = asyncio.run(r.acompletion(model="m", messages=[{"role": "user", "content": "hi"}]))
|
||||
assert response.choices[0].message.content == "ok"
|
||||
assert calls == ["m"]
|
||||
|
||||
|
||||
def test_alias_attribute_writes_the_same_storage(router):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
replacement = {}
|
||||
router._routing_groups = replacement
|
||||
assert router.routing_groups is replacement
|
||||
assert router.registry().routing_groups() is replacement
|
||||
|
|
@ -1158,7 +1158,7 @@ async def test_set_response_headers_passes_through_post_increment_counters(model
|
|||
_hidden_params: dict = {}
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-remaining-tokens": 958,
|
||||
"x-ratelimit-limit-tokens": 1000,
|
||||
|
|
@ -1539,7 +1539,7 @@ async def test_get_remaining_model_group_usage_merges_io_and_tpm_headers(model_l
|
|||
)
|
||||
)
|
||||
router.get_model_group_io_token_usage = AsyncMock(return_value=(100, 40))
|
||||
router.get_model_group_usage = AsyncMock(return_value=(500, 5))
|
||||
router._core_get_model_group_usage = AsyncMock(return_value=(500, 5))
|
||||
|
||||
headers = await router.get_remaining_model_group_usage("gpt-3.5-turbo")
|
||||
|
||||
|
|
@ -1566,7 +1566,7 @@ async def test_set_response_headers_native_input_token_header_does_not_suppress_
|
|||
_hidden_params: dict = {}
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-remaining-tokens": 1000,
|
||||
"x-ratelimit-remaining-requests": 100,
|
||||
|
|
@ -1596,7 +1596,7 @@ async def test_set_response_headers_native_token_header_does_not_suppress_io_hea
|
|||
_hidden_params: dict = {}
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-remaining-tokens": 1000,
|
||||
"x-ratelimit-remaining-requests": 100,
|
||||
|
|
@ -1628,7 +1628,7 @@ async def test_set_response_headers_handles_missing_usage(model_list):
|
|||
_hidden_params: dict = {}
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-remaining-tokens": 1000,
|
||||
"x-ratelimit-remaining-requests": 100,
|
||||
|
|
@ -1648,7 +1648,7 @@ async def test_set_response_headers_handles_missing_usage(model_list):
|
|||
async def test_set_response_headers_dict_anthropic_messages_response(model_list):
|
||||
"""Anthropic /v1/messages returns a dict; IO rate-limit headers must attach."""
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-limit-input-tokens": 25,
|
||||
"x-ratelimit-remaining-input-tokens": 20,
|
||||
|
|
@ -1688,7 +1688,7 @@ async def test_set_response_headers_wraps_bare_async_generator(model_list):
|
|||
yield {"type": "message_stop"}
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-limit-input-tokens": 25,
|
||||
"x-ratelimit-remaining-input-tokens": 20,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue