mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-30 06:43:27 +00:00
refactor(router): add router entry points and keep the old members as a deprecated mixin (#48840)
Router gains eight entry points (registry, settings, health, select, apply/aapply, observe, register_plugin, retire) in litellm/router_api.py. The 123 public members outside code used become DeprecatedRouterMembers (litellm/router_deprecated.py): same signatures and results, a DeprecationWarning naming the replacement and a use counter for callers outside the router. Router internals move to _core_ names; subclass overrides of old members are still called by the router.
This commit is contained in:
Nate Armstrong
GitHub
parent
feda51453a
commit
a659e8be70
10 changed files with 3913 additions and 724 deletions
1517
litellm/router.py
1517
litellm/router.py
File diff suppressed because it is too large
Load diff
1296
litellm/router_api.py
Normal file
1296
litellm/router_api.py
Normal file
File diff suppressed because it is too large
Load diff
852
litellm/router_deprecated.py
Normal file
852
litellm/router_deprecated.py
Normal file
|
|
@ -0,0 +1,852 @@
|
|||
"""
|
||||
DeprecatedRouterMembers: the Router members that outside code used before the entry points.
|
||||
|
||||
``litellm.Router`` mixes this class in so existing callers keep working while they move to the
|
||||
entry points (``registry()``, ``settings()``, ``health()``, ``select()``, ``apply()``,
|
||||
``observe()``, ``register_plugin()``, ``retire()``; see litellm/router_api.py). Every member here
|
||||
returns what it returned before, with the type it had before, and names its replacement in a
|
||||
``DeprecationWarning``. Reads go to the same Router implementation (``Router._core_<name>``) the
|
||||
entry point uses; writes go through ``apply`` / ``aapply`` so the registry version moves.
|
||||
|
||||
Access from the router itself and from its own modules (``litellm.router*``) is not counted:
|
||||
those are the router's internals and move behind the core interface separately. Every other access
|
||||
is counted in ``DEPRECATED_ROUTER_MEMBER_USES`` so the remaining users show up before this class is
|
||||
deleted (planned about a month after the core swap).
|
||||
"""
|
||||
|
||||
# pyright: reportPrivateUsage=false, reportMissingTypeArgument=false, reportUnknownParameterType=false
|
||||
# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false
|
||||
# pyright: reportUnknownLambdaType=false
|
||||
# This shim is part of the router (it reads Router's ``_core_*`` members, as Router's own methods do)
|
||||
# and repeats each old member's signature exactly, untyped parts included (``dict``, ``list``), so
|
||||
# callers keep the types they had. Typing those parts is the core's job, not the shim's.
|
||||
|
||||
import contextvars
|
||||
import functools
|
||||
import inspect
|
||||
import sys
|
||||
import warnings
|
||||
from collections import Counter
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload
|
||||
|
||||
from litellm.router_api import SETTING_FIELDS, ChangeSet
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import Any # noqa: TID251 # only to repeat two old signatures that had Any, unchanged
|
||||
|
||||
from opentelemetry.trace import Span
|
||||
|
||||
from litellm.caching.caching import RedisCache
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery
|
||||
from litellm.router_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.types.router import (
|
||||
CustomRoutingStrategyBase,
|
||||
Deployment,
|
||||
DeploymentModelListingInfo,
|
||||
DeploymentTypedDict,
|
||||
GuardrailTypedDict,
|
||||
ModelGroupInfo,
|
||||
PreRoutingHookResponse,
|
||||
SearchToolTypedDict,
|
||||
)
|
||||
from litellm.types.utils import ModelInfo as ModelMapInfo
|
||||
|
||||
_T: Final = TypeVar("_T")
|
||||
|
||||
DEPRECATED_ROUTER_MEMBER_USES: Final[Counter[str]] = Counter() # mutable-ok: the live tally _note increments
|
||||
|
||||
_INTERNAL_PREFIX: Final = "litellm.router"
|
||||
|
||||
|
||||
# Router's own attribute hooks: a write or delete through an old name passes through them first.
|
||||
_ATTRIBUTE_HOOKS: Final = frozenset({"__setattr__", "__delattr__"})
|
||||
|
||||
|
||||
def _note(name: str, replacement: str) -> None:
|
||||
depth = 2 # rebind-ok: grows by one per attribute hook frame below
|
||||
caller = sys._getframe(depth) # rebind-ok: follows depth up past those hooks
|
||||
while caller.f_code.co_name in _ATTRIBUTE_HOOKS and caller.f_globals.get("__name__") == "litellm.router":
|
||||
depth += 1
|
||||
caller = sys._getframe(depth)
|
||||
module: Final = caller.f_globals.get("__name__")
|
||||
if isinstance(module, str) and module.startswith(_INTERNAL_PREFIX):
|
||||
return
|
||||
DEPRECATED_ROUTER_MEMBER_USES[name] += 1
|
||||
warnings.warn(
|
||||
f"Router.{name} is deprecated; use Router.{replacement}",
|
||||
DeprecationWarning,
|
||||
stacklevel=depth + 1,
|
||||
)
|
||||
|
||||
|
||||
def _as_router(obj: "DeprecatedRouterMembers") -> "Router":
|
||||
"""The Router this mixin is part of (it is mixed into ``litellm.Router`` only)."""
|
||||
from litellm.router import Router
|
||||
|
||||
if isinstance(obj, Router):
|
||||
return obj
|
||||
raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only")
|
||||
|
||||
|
||||
def _public_attribute_error(obj: object, core_name: str, public_name: str, error: AttributeError) -> AttributeError:
|
||||
"""
|
||||
``error`` renamed to the public attribute when it is the router's own ``core_name`` that is
|
||||
missing, so the message is the one reading the old attribute raised; any other error unchanged.
|
||||
"""
|
||||
if error.name not in (core_name, public_name) or error.obj is not obj:
|
||||
return error
|
||||
return AttributeError(f"'{type(obj).__name__}' object has no attribute '{public_name}'", name=public_name, obj=obj)
|
||||
|
||||
|
||||
class _DeprecatedAttribute(Generic[_T]):
|
||||
"""A former Router attribute, read through an entry point. Assignment still replaces it."""
|
||||
|
||||
def __init__(self, replacement: str, read: Callable[["Router"], _T]) -> None:
|
||||
self.replacement = replacement
|
||||
self.read = read
|
||||
self.name = ""
|
||||
|
||||
def __set_name__(self, owner: type, name: str) -> None:
|
||||
# An alias (``_routing_groups = routing_groups``) keeps the first name.
|
||||
self.name = self.name or name
|
||||
|
||||
@overload
|
||||
def __get__(self, obj: None, owner: type) -> object: ...
|
||||
|
||||
@overload
|
||||
def __get__(self, obj: "DeprecatedRouterMembers", owner: type) -> _T: ...
|
||||
|
||||
def __get__(self, obj: "DeprecatedRouterMembers | None", owner: type) -> object:
|
||||
if obj is None:
|
||||
return getattr(owner, "_core_" + self.name, self)
|
||||
_note(self.name, self.replacement)
|
||||
router: Final = _as_router(obj)
|
||||
try:
|
||||
return self.read(router)
|
||||
except AttributeError as e:
|
||||
raise _public_attribute_error(router, "_core_" + self.name, self.name, e) from None
|
||||
|
||||
def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None:
|
||||
_note(self.name, self.replacement)
|
||||
if self.name in SETTING_FIELDS:
|
||||
_as_router(obj).apply(ChangeSet(assign={self.name: value}))
|
||||
else:
|
||||
setattr(obj, "_core_" + self.name, value)
|
||||
|
||||
def __delete__(self, obj: "DeprecatedRouterMembers") -> None:
|
||||
_note(self.name, self.replacement)
|
||||
try:
|
||||
delattr(obj, "_core_" + self.name)
|
||||
except AttributeError:
|
||||
# The only attribute this deletes; Python's own message for it, which sets no name/obj.
|
||||
raise AttributeError(f"'{type(obj).__name__}' object has no attribute '{self.name}'") from None
|
||||
|
||||
|
||||
def _setting(name: str, read: Callable[["Router"], _T]) -> _DeprecatedAttribute[_T]:
|
||||
"""A former settings attribute: ``read`` returns the router's own value, typed as before."""
|
||||
|
||||
def read_setting(router: "Router") -> _T:
|
||||
if not router.settings().is_set(name):
|
||||
raise AttributeError(f"'{type(router).__name__}' object has no attribute {name!r}", name=name, obj=router)
|
||||
return read(router)
|
||||
|
||||
return _DeprecatedAttribute(f"settings().{name}", read_setting)
|
||||
|
||||
|
||||
def _attr(replacement: str, read: Callable[["Router"], _T]) -> _DeprecatedAttribute[_T]:
|
||||
return _DeprecatedAttribute(replacement, read)
|
||||
|
||||
|
||||
# (id(router), member name) pairs whose subclass override is running in this context.
|
||||
_OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] = contextvars.ContextVar(
|
||||
"router_overrides_running", default=frozenset()
|
||||
)
|
||||
|
||||
|
||||
def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., object]:
|
||||
"""
|
||||
A direct call of a subclass override (``router.<name>()``) marks it running, so its
|
||||
``super().<name>()`` reaches the parent once instead of re-entering the override through
|
||||
``_core_<name>``.
|
||||
"""
|
||||
if inspect.iscoroutinefunction(override):
|
||||
async_override: Final[Callable[..., Awaitable[object]]] = override
|
||||
|
||||
@functools.wraps(override)
|
||||
async def entered_async(
|
||||
router: object,
|
||||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
|
||||
) -> object:
|
||||
token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)})
|
||||
try:
|
||||
return await async_override(router, *args, **kwargs)
|
||||
finally:
|
||||
_OVERRIDES_RUNNING.reset(token)
|
||||
|
||||
return entered_async
|
||||
|
||||
@functools.wraps(override)
|
||||
def entered(
|
||||
router: object,
|
||||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
|
||||
) -> object:
|
||||
token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)})
|
||||
try:
|
||||
return override(router, *args, **kwargs)
|
||||
finally:
|
||||
_OVERRIDES_RUNNING.reset(token)
|
||||
|
||||
return entered
|
||||
|
||||
|
||||
def _private_alias(name: str, member: Callable[..., object]) -> Callable[..., object]:
|
||||
"""
|
||||
A private alias of an old member (``_get_all_deployments``). It was bound to Router's own
|
||||
method, so it never reached a subclass override of ``name``, and still doesn't.
|
||||
"""
|
||||
return _enter_override(name, member)
|
||||
|
||||
|
||||
def _route_override(cls: type, name: str, override: Callable[..., object]) -> Callable[..., object]:
|
||||
"""
|
||||
Make the router's own calls to ``_core_<name>`` reach a subclass override of ``<name>``.
|
||||
|
||||
When the override calls ``super().<name>()``, the call comes back to ``_core_<name>``; it is
|
||||
then sent on to the parent's implementation instead of back into the override.
|
||||
"""
|
||||
core: Final = "_core_" + name
|
||||
|
||||
def parent(router: object) -> Callable[..., object]:
|
||||
return getattr(super(cls, router), core)
|
||||
|
||||
if inspect.iscoroutinefunction(override):
|
||||
async_override: Final[Callable[..., Awaitable[object]]] = override
|
||||
|
||||
@functools.wraps(override)
|
||||
async def routed_async(
|
||||
router: object,
|
||||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
|
||||
) -> object:
|
||||
key: Final = (id(router), name)
|
||||
running: Final = _OVERRIDES_RUNNING.get()
|
||||
if key in running:
|
||||
parent_async: Final[Callable[..., Awaitable[object]]] = getattr(super(cls, router), core)
|
||||
return await parent_async(*args, **kwargs)
|
||||
token: Final = _OVERRIDES_RUNNING.set(running | {key})
|
||||
try:
|
||||
return await async_override(router, *args, **kwargs)
|
||||
finally:
|
||||
_OVERRIDES_RUNNING.reset(token)
|
||||
|
||||
return routed_async
|
||||
|
||||
@functools.wraps(override)
|
||||
def routed(
|
||||
router: object,
|
||||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
|
||||
) -> object:
|
||||
key: Final = (id(router), name)
|
||||
running: Final = _OVERRIDES_RUNNING.get()
|
||||
if key in running:
|
||||
return parent(router)(*args, **kwargs)
|
||||
token: Final = _OVERRIDES_RUNNING.set(running | {key})
|
||||
try:
|
||||
return override(router, *args, **kwargs)
|
||||
finally:
|
||||
_OVERRIDES_RUNNING.reset(token)
|
||||
|
||||
return routed
|
||||
|
||||
|
||||
class DeprecatedRouterMembers:
|
||||
"""Old Router members, kept for about a month. Each names its replacement."""
|
||||
|
||||
def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: class keywords, passed on to super()
|
||||
# A Router subclass that overrides an old member (say async_pre_routing_hook) keeps
|
||||
# having its override called by the router, which now calls _core_<name> internally.
|
||||
super().__init_subclass__(**kwargs)
|
||||
for name, value in list(vars(cls).items()):
|
||||
if (
|
||||
name.startswith("_core_")
|
||||
or name not in vars(DeprecatedRouterMembers)
|
||||
or not hasattr(cls, "_core_" + name)
|
||||
):
|
||||
continue
|
||||
if isinstance(value, staticmethod):
|
||||
# Its super() call reaches the old member here, which calls Router's own implementation.
|
||||
setattr(cls, "_core_" + name, value)
|
||||
elif inspect.isfunction(value):
|
||||
setattr(cls, "_core_" + name, _route_override(cls, name, value))
|
||||
setattr(cls, name, _enter_override(name, value))
|
||||
|
||||
# -- registry: attributes --------------------------------------------------------------------
|
||||
model_list = _attr("registry().all()", lambda r: r._core_model_list)
|
||||
model_names = _attr("registry().names(kind='group_set')", lambda r: r._core_model_names)
|
||||
deployment_names = _attr("registry().names(kind='deployment_names')", lambda r: r._core_deployment_names)
|
||||
team_public_model_names = _attr("registry().names(kind='team_public')", lambda r: r._core_team_public_model_names)
|
||||
model_name_to_deployment_indices = _attr(
|
||||
"registry().index('model_name')", lambda r: r._core_model_name_to_deployment_indices
|
||||
)
|
||||
team_model_to_deployment_indices = _attr(
|
||||
"registry().index('team_model')", lambda r: r._core_team_model_to_deployment_indices
|
||||
)
|
||||
cached_deployment_model_info = _attr(
|
||||
"registry().deployment_model_info(id, model_name, cached=True)",
|
||||
lambda r: r._core_cached_deployment_model_info,
|
||||
)
|
||||
model_group_alias = _attr("registry().aliases()", lambda r: r._core_model_group_alias)
|
||||
pattern_router = _attr("registry().patterns()", lambda r: r._core_pattern_router)
|
||||
team_pattern_routers = _attr("registry().patterns(team=True)", lambda r: r._core_team_pattern_routers)
|
||||
routing_groups = _attr("registry().routing_groups()", lambda r: r._core_routing_groups)
|
||||
_routing_groups = routing_groups
|
||||
auto_routers = _attr("registry().plugins('auto')", lambda r: r._core_auto_routers)
|
||||
complexity_routers = _attr("registry().plugins('complexity')", lambda r: r._core_complexity_routers)
|
||||
quality_routers = _attr("registry().plugins('quality')", lambda r: r._core_quality_routers)
|
||||
adaptive_routers = _attr("registry().plugins('adaptive')", lambda r: r._core_adaptive_routers)
|
||||
routing_plugins = _attr("registry().plugins('routing')", lambda r: r._core_routing_plugins)
|
||||
prompt_caching_pin_cache = _attr(
|
||||
"registry().plugins('prompt_caching_pins')", lambda r: r._core_prompt_caching_pin_cache
|
||||
)
|
||||
|
||||
# -- health and selection: attributes --------------------------------------------------------
|
||||
cache = _attr("health().store", lambda r: r._core_cache)
|
||||
cooldown_cache = _attr("health().cooldowns", lambda r: r._core_cooldown_cache)
|
||||
health_state_cache = _attr("health().health_checks", lambda r: r._core_health_state_cache)
|
||||
deployment_latency_map = _attr("health().latency", lambda r: r._core_deployment_latency_map)
|
||||
async_get_available_deployment = _attr(
|
||||
"select(model, request_kwargs)",
|
||||
lambda r: r._core_async_get_available_deployment,
|
||||
)
|
||||
|
||||
# -- observers ------------------------------------------------------------------------------
|
||||
slack_alerting_logger = _attr("observe().slack_alerting_logger", lambda r: r._core_slack_alerting_logger)
|
||||
|
||||
# -- settings -------------------------------------------------------------------------------
|
||||
routing_strategy = _setting("routing_strategy", lambda r: r._core_routing_strategy)
|
||||
context_window_fallbacks = _setting("context_window_fallbacks", lambda r: r._core_context_window_fallbacks)
|
||||
content_policy_fallbacks = _setting("content_policy_fallbacks", lambda r: r._core_content_policy_fallbacks)
|
||||
default_deployment = _setting("default_deployment", lambda r: r._core_default_deployment)
|
||||
default_litellm_params = _setting("default_litellm_params", lambda r: r._core_default_litellm_params)
|
||||
default_max_parallel_requests = _setting(
|
||||
"default_max_parallel_requests", lambda r: r._core_default_max_parallel_requests
|
||||
)
|
||||
cooldown_time = _setting("cooldown_time", lambda r: r._core_cooldown_time)
|
||||
allowed_fails = _setting("allowed_fails", lambda r: r._core_allowed_fails)
|
||||
allowed_fails_policy = _setting("allowed_fails_policy", lambda r: r._core_allowed_fails_policy)
|
||||
disable_cooldowns = _setting("disable_cooldowns", lambda r: r._core_disable_cooldowns)
|
||||
enable_pre_call_checks = _setting("enable_pre_call_checks", lambda r: r._core_enable_pre_call_checks)
|
||||
enable_health_check_routing = _setting("enable_health_check_routing", lambda r: r._core_enable_health_check_routing)
|
||||
health_check_ignore_transient_errors = _setting(
|
||||
"health_check_ignore_transient_errors", lambda r: r._core_health_check_ignore_transient_errors
|
||||
)
|
||||
background_health_check_model_groups = _setting(
|
||||
"background_health_check_model_groups", lambda r: r._core_background_health_check_model_groups
|
||||
)
|
||||
enable_tag_filtering = _setting("enable_tag_filtering", lambda r: r._core_enable_tag_filtering)
|
||||
tag_filtering_match_any = _setting("tag_filtering_match_any", lambda r: r._core_tag_filtering_match_any)
|
||||
tag_routing_prefix = _setting("tag_routing_prefix", lambda r: r._core_tag_routing_prefix)
|
||||
router_general_settings = _setting("router_general_settings", lambda r: r._core_router_general_settings)
|
||||
provider_budget_config = _setting("provider_budget_config", lambda r: r._core_provider_budget_config)
|
||||
provider_default_deployment_ids = _setting(
|
||||
"provider_default_deployment_ids", lambda r: r._core_provider_default_deployment_ids
|
||||
)
|
||||
fallback_access_check = _setting("fallback_access_check", lambda r: r._core_fallback_access_check)
|
||||
fallback_budget_check = _setting("fallback_budget_check", lambda r: r._core_fallback_budget_check)
|
||||
# These three are declared with the attribute type Router gave them rather than the descriptor's:
|
||||
# protocols in router_utils and the proxy match Router on them, and protocol matching does not
|
||||
# see through a descriptor. Reads and writes still go through _setting at runtime.
|
||||
fallbacks: list | list[dict[str, list[str] | list | None]] | None = _setting( # pyright: ignore[reportAssignmentType] # see above # mutable-ok: the old type
|
||||
"fallbacks", lambda r: r._core_fallbacks
|
||||
)
|
||||
kubernetes_pod_discovery: "KubernetesPodDiscovery" = _setting( # pyright: ignore[reportAssignmentType] # see above
|
||||
"kubernetes_pod_discovery", lambda r: r._core_kubernetes_pod_discovery
|
||||
)
|
||||
search_tools: "list[SearchToolTypedDict]" = _setting( # pyright: ignore[reportAssignmentType] # see above # mutable-ok: the old type
|
||||
"search_tools", lambda r: r._core_search_tools
|
||||
)
|
||||
guardrail_list = _setting("guardrail_list", lambda r: r._core_guardrail_list)
|
||||
cache_responses = _setting("cache_responses", lambda r: r._core_cache_responses)
|
||||
|
||||
# -- registry: one deployment ---------------------------------------------------------------
|
||||
def get_deployment(self, model_id: str) -> "Deployment | None":
|
||||
_note("get_deployment", "registry().deployment(id)")
|
||||
return _as_router(self)._core_get_deployment(model_id=model_id)
|
||||
|
||||
def has_model_id(self, candidate_id: str) -> bool:
|
||||
_note("has_model_id", "registry().has_deployment(id)")
|
||||
return _as_router(self)._core_has_model_id(candidate_id)
|
||||
|
||||
def get_model_info(self, id: str) -> dict | None: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_info", "registry().deployment_dict(id)")
|
||||
return _as_router(self)._core_get_model_info(id=id)
|
||||
|
||||
def get_deployment_model_info(self, model_id: str, model_name: str) -> "ModelMapInfo | None":
|
||||
_note("get_deployment_model_info", "registry().deployment_model_info(id, model_name)")
|
||||
return _as_router(self)._core_get_deployment_model_info(model_id=model_id, model_name=model_name)
|
||||
|
||||
@overload
|
||||
def get_router_model_info(
|
||||
self,
|
||||
deployment: "dict | Deployment", # mutable-ok: the old parameter type, unchanged
|
||||
received_model_name: str,
|
||||
id: None = None,
|
||||
) -> "ModelMapInfo": ...
|
||||
|
||||
@overload
|
||||
def get_router_model_info(self, deployment: None, received_model_name: str, id: str) -> "ModelMapInfo": ...
|
||||
|
||||
def get_router_model_info(
|
||||
self,
|
||||
deployment: "dict | Deployment | None", # mutable-ok: the old parameter type, unchanged
|
||||
received_model_name: str,
|
||||
id: str | None = None,
|
||||
) -> "ModelMapInfo":
|
||||
_note("get_router_model_info", "registry().router_model_info(...)")
|
||||
router: Final = _as_router(self)
|
||||
# The core's overloads split (deployment, id) by which one is None; this forwards either form unchanged.
|
||||
return router._core_get_router_model_info( # pyright: ignore[reportCallIssue] # see above
|
||||
deployment=deployment, # pyright: ignore[reportArgumentType] # see above
|
||||
received_model_name=received_model_name,
|
||||
id=id, # pyright: ignore[reportArgumentType] # see above
|
||||
)
|
||||
|
||||
def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]:
|
||||
_note("get_discovered_model_info", "registry().discovered_model_info(id)")
|
||||
return _as_router(self)._core_get_discovered_model_info(model_id)
|
||||
|
||||
# -- registry: many deployments -------------------------------------------------------------
|
||||
def get_model_list(
|
||||
self, model_name: str | None = None, team_id: str | None = None
|
||||
) -> "list[DeploymentTypedDict] | None": # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_list", "registry().deployments(model_name, team_id=...)")
|
||||
return _as_router(self)._core_get_model_list(model_name=model_name, team_id=team_id)
|
||||
|
||||
def get_all_deployments(
|
||||
self, model_name: str, model_alias: str | None = None, team_id: str | None = None
|
||||
) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged
|
||||
_note("get_all_deployments", "registry().deployments(model_name, source='all')")
|
||||
return _as_router(self)._core_get_all_deployments(
|
||||
model_name=model_name, model_alias=model_alias, team_id=team_id
|
||||
)
|
||||
|
||||
_get_all_deployments = _private_alias("get_all_deployments", get_all_deployments)
|
||||
|
||||
def config_deployments(self) -> Iterator[Mapping[str, object]]:
|
||||
_note("config_deployments", "registry().deployments(source='config')")
|
||||
return _as_router(self)._core_config_deployments()
|
||||
|
||||
def deployments_named(self, model_name: str) -> "tuple[DeploymentTypedDict, ...] | None":
|
||||
_note("deployments_named", "registry().deployments(model_name, source='named')")
|
||||
return _as_router(self)._core_deployments_named(model_name)
|
||||
|
||||
def get_model_list_from_model_alias(
|
||||
self, model_name: str | None = None
|
||||
) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_list_from_model_alias", "registry().deployments(model_name, source='alias')")
|
||||
return _as_router(self)._core_get_model_list_from_model_alias(model_name=model_name)
|
||||
|
||||
def get_model_list_from_routing_groups(self, model_name: str | None = None) -> "Sequence[DeploymentTypedDict]":
|
||||
_note("get_model_list_from_routing_groups", "registry().deployments(model_name, source='routing_groups')")
|
||||
return _as_router(self)._core_get_model_list_from_routing_groups(model_name=model_name)
|
||||
|
||||
def get_model_list_of_routed_group(
|
||||
self, model_group: str
|
||||
) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_list_of_routed_group", "registry().deployments(model_group, source='routed_group')")
|
||||
return _as_router(self)._core_get_model_list_of_routed_group(model_group)
|
||||
|
||||
def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]:
|
||||
_note("wildcard_route_deployments", "registry().deployments(model, source='wildcard')")
|
||||
return _as_router(self)._core_wildcard_route_deployments(model, team_id)
|
||||
|
||||
def get_routable_deployments(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: Mapping[str, object] = MappingProxyType({}),
|
||||
call_type: str | None = None,
|
||||
) -> "tuple[DeploymentTypedDict, ...]":
|
||||
_note(
|
||||
"get_routable_deployments", "registry().deployments(model, source='routable', request=..., call_type=...)"
|
||||
)
|
||||
return _as_router(self)._core_get_routable_deployments(model, request_kwargs, call_type)
|
||||
|
||||
def get_deployment_by_model_group_name(self, model_group_name: str) -> "Deployment | None":
|
||||
_note("get_deployment_by_model_group_name", "registry().deployments(name, source='first_in_group')")
|
||||
return _as_router(self)._core_get_deployment_by_model_group_name(model_group_name=model_group_name)
|
||||
|
||||
def deployments_for_request(
|
||||
self, model: str, request_kwargs: Mapping[str, object]
|
||||
) -> "Sequence[DeploymentTypedDict]":
|
||||
_note("deployments_for_request", "registry().deployments(model, source='for_request', request=...)")
|
||||
return _as_router(self)._core_deployments_for_request(model, request_kwargs)
|
||||
|
||||
def should_include_deployment(
|
||||
self,
|
||||
model_name: str,
|
||||
model: "DeploymentTypedDict | dict", # mutable-ok: the old parameter type, unchanged
|
||||
team_id: str | None = None,
|
||||
) -> bool:
|
||||
_note("should_include_deployment", "registry().includes(deployment, model_name=..., team_id=...)")
|
||||
return _as_router(self)._core_should_include_deployment(model_name=model_name, model=model, team_id=team_id)
|
||||
|
||||
@staticmethod
|
||||
def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool:
|
||||
_note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)")
|
||||
from litellm.router import Router
|
||||
|
||||
return Router._core_deployment_usable_by_team(model, team_id)
|
||||
|
||||
_deployment_usable_by_team = deployment_usable_by_team
|
||||
|
||||
# -- registry: names -------------------------------------------------------------------------
|
||||
def get_model_names(self, team_id: str | None = None) -> list[str]: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_names", "registry().names(team_id=...)")
|
||||
return _as_router(self)._core_get_model_names(team_id=team_id)
|
||||
|
||||
def get_model_ids(
|
||||
self, model_name: str | None = None, exclude_team_models: bool = False
|
||||
) -> list[str]: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_ids", "registry().names(model_name, kind='ids')")
|
||||
return _as_router(self)._core_get_model_ids(model_name=model_name, exclude_team_models=exclude_team_models)
|
||||
|
||||
def resolved_litellm_models(
|
||||
self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False
|
||||
) -> tuple[str, ...]:
|
||||
_note("resolved_litellm_models", "registry().names(model_name, kind='litellm_models' or 'litellm_base_models')")
|
||||
return _as_router(self)._core_resolved_litellm_models(
|
||||
model_name, team_id=team_id, use_base_model=use_base_model
|
||||
)
|
||||
|
||||
def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]:
|
||||
_note("wildcard_routed_models", "registry().names(model, kind='wildcard')")
|
||||
return _as_router(self)._core_wildcard_routed_models(model, team_id)
|
||||
|
||||
def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]:
|
||||
_note("get_candidate_model_ids_for_route", "registry().names(model, kind='route_candidates')")
|
||||
return _as_router(self)._core_get_candidate_model_ids_for_route(model, team_id=team_id)
|
||||
|
||||
# -- registry: resolving names ---------------------------------------------------------------
|
||||
def routable_model_group(self, model_name: str) -> str:
|
||||
_note("routable_model_group", "registry().resolve(name)")
|
||||
return _as_router(self)._core_routable_model_group(model_name)
|
||||
|
||||
def get_model_from_alias(self, model: str) -> str | None:
|
||||
_note("get_model_from_alias", "registry().resolve(name, kind='alias')")
|
||||
return _as_router(self)._core_get_model_from_alias(model=model)
|
||||
|
||||
_get_model_from_alias = _private_alias("get_model_from_alias", get_model_from_alias)
|
||||
|
||||
def resolve_served_model_name(self, model_name: str, served_model: str) -> str:
|
||||
_note("resolve_served_model_name", "registry().resolve(name, kind='served', served_model=...)")
|
||||
return _as_router(self)._core_resolve_served_model_name(model_name, served_model)
|
||||
|
||||
def resolve_model_name_from_model_id(
|
||||
self, model_id: str | None, custom_llm_provider: str | None = None
|
||||
) -> str | None:
|
||||
_note("resolve_model_name_from_model_id", "registry().resolve(id, kind='from_id')")
|
||||
return _as_router(self)._core_resolve_model_name_from_model_id(
|
||||
model_id, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
def map_team_model(
|
||||
self, team_model_name: str | None, team_id: str, custom_llm_provider: str | None = None
|
||||
) -> str | None:
|
||||
_note("map_team_model", "registry().resolve(name, kind='team_model', team_id=...)")
|
||||
return _as_router(self)._core_map_team_model(team_model_name, team_id, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
def is_recognized_model(self, model: str) -> bool:
|
||||
_note("is_recognized_model", "registry().resolve(name, kind='recognized')")
|
||||
return _as_router(self)._core_is_recognized_model(model)
|
||||
|
||||
def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None:
|
||||
_note("get_routable_upstream_model", "registry().resolve(name, kind='upstream')")
|
||||
return _as_router(self)._core_get_routable_upstream_model(model_name, team_id=team_id)
|
||||
|
||||
def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]:
|
||||
_note("get_search_tool_support", "registry().resolve(name, kind='search_tool_support')")
|
||||
return _as_router(self)._core_get_search_tool_support(model_name, team_id=team_id)
|
||||
|
||||
def get_model_group(self, id: str) -> list | None: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_group", "registry().resolve(id, kind='group_of_id')")
|
||||
return _as_router(self)._core_get_model_group(id=id)
|
||||
|
||||
def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool:
|
||||
_note("is_model_access_group_for_wildcard_route", "registry().resolve(group, kind='wildcard_access_group')")
|
||||
return _as_router(self)._core_is_model_access_group_for_wildcard_route(model_access_group=model_access_group)
|
||||
|
||||
_is_model_access_group_for_wildcard_route = _private_alias(
|
||||
"is_model_access_group_for_wildcard_route", is_model_access_group_for_wildcard_route
|
||||
)
|
||||
|
||||
def team_model_has_alternatives(self, deployment_id: str) -> bool:
|
||||
_note("team_model_has_alternatives", "registry().resolve(id, kind='team_model_alternatives')")
|
||||
return _as_router(self)._core_team_model_has_alternatives(deployment_id)
|
||||
|
||||
def routing_group_has_alternatives(self, model_group: str | None) -> bool:
|
||||
_note("routing_group_has_alternatives", "registry().resolve(group, kind='routing_group_alternatives')")
|
||||
return _as_router(self)._core_routing_group_has_alternatives(model_group)
|
||||
|
||||
# -- registry: groups, access groups, credentials, plugins -----------------------------------
|
||||
def get_model_group_info(
|
||||
self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None
|
||||
) -> "ModelGroupInfo | None":
|
||||
_note("get_model_group_info", "registry().group(name)")
|
||||
return _as_router(self)._core_get_model_group_info(model_group, include_hidden=include_hidden, team_id=team_id)
|
||||
|
||||
def cached_model_group_info(self, model_group: str) -> "ModelGroupInfo | None":
|
||||
_note("cached_model_group_info", "registry().group(name, field='cached_info')")
|
||||
return _as_router(self)._core_cached_model_group_info(model_group)
|
||||
|
||||
def get_configured_display_name(self, model_name: str) -> str | None:
|
||||
_note("get_configured_display_name", "registry().group(name, field='display_name')")
|
||||
return _as_router(self)._core_get_configured_display_name(model_name)
|
||||
|
||||
def get_configured_mode(self, model_name: str) -> str | None:
|
||||
_note("get_configured_mode", "registry().group(name, field='mode')")
|
||||
return _as_router(self)._core_get_configured_mode(model_name)
|
||||
|
||||
def get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]:
|
||||
_note("get_configured_service_tiers", "registry().group(name, field='service_tiers')")
|
||||
return _as_router(self)._core_get_configured_service_tiers(model_name, team_id=team_id)
|
||||
|
||||
def get_configured_token_limits(self, model_name: str) -> tuple[int | None, int | None]:
|
||||
_note("get_configured_token_limits", "registry().group(name, field='token_limits')")
|
||||
return _as_router(self)._core_get_configured_token_limits(model_name)
|
||||
|
||||
def get_model_listing_info(self, model_name: str) -> "DeploymentModelListingInfo | None":
|
||||
_note("get_model_listing_info", "registry().group(name, field='listing_info')")
|
||||
return _as_router(self)._core_get_model_listing_info(model_name)
|
||||
|
||||
def get_model_access_groups(
|
||||
self,
|
||||
model_name: str | None = None,
|
||||
model_access_group: str | None = None,
|
||||
team_id: str | None = None,
|
||||
) -> dict[str, list[str]]: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_access_groups", "registry().access_groups(...)")
|
||||
return _as_router(self)._core_get_model_access_groups(
|
||||
model_name=model_name, model_access_group=model_access_group, team_id=team_id
|
||||
)
|
||||
|
||||
def get_deployment_credentials(self, model_id: str) -> dict | None: # mutable-ok: the old return type, unchanged
|
||||
_note("get_deployment_credentials", "registry().credentials(id)")
|
||||
return _as_router(self)._core_get_deployment_credentials(model_id=model_id)
|
||||
|
||||
def get_deployment_credentials_with_provider(
|
||||
self, model_id: str, team_id: str | None = None
|
||||
) -> "dict[str, Any] | None": # pyright: ignore[reportExplicitAny] # the old return type, unchanged # mutable-ok: same
|
||||
_note("get_deployment_credentials_with_provider", "registry().credentials(id, form='with_provider')")
|
||||
return _as_router(self)._core_get_deployment_credentials_with_provider(model_id=model_id, team_id=team_id)
|
||||
|
||||
def get_credential_deployment(self, model_id: str, team_id: str | None = None) -> "Deployment | None":
|
||||
_note("get_credential_deployment", "registry().credentials(id, form='deployment')")
|
||||
return _as_router(self)._core_get_credential_deployment(model_id=model_id, team_id=team_id)
|
||||
|
||||
def get_router_deployment_budget_limiter(self) -> "RouterBudgetLimiting | None":
|
||||
_note("get_router_deployment_budget_limiter", "registry().plugins('budget_limiter')")
|
||||
return _as_router(self)._core_get_router_deployment_budget_limiter()
|
||||
|
||||
_get_router_deployment_budget_limiter = _private_alias(
|
||||
"get_router_deployment_budget_limiter", get_router_deployment_budget_limiter
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def generate_model_id(model_group: str, litellm_params: dict) -> str: # mutable-ok: the old parameter type
|
||||
_note("generate_model_id", "litellm.router_api.deployment_id(model_group, litellm_params)")
|
||||
from litellm.router_api import deployment_id
|
||||
|
||||
return deployment_id(model_group, litellm_params)
|
||||
|
||||
# -- settings: methods -----------------------------------------------------------------------
|
||||
def get_settings(self) -> dict[str, object]: # mutable-ok: Router._core_get_settings's type, a fresh dict
|
||||
_note("get_settings", "settings().as_dict()")
|
||||
return _as_router(self)._core_get_settings()
|
||||
|
||||
def get_allowed_fails_from_policy(self, exception: Exception) -> int | None:
|
||||
_note("get_allowed_fails_from_policy", "settings().allowed_fails_for(exception)")
|
||||
return _as_router(self)._core_get_allowed_fails_from_policy(exception=exception)
|
||||
|
||||
def get_available_guardrail(self, guardrail_name: str) -> "GuardrailTypedDict":
|
||||
_note("get_available_guardrail", "settings().guardrail(name)")
|
||||
return _as_router(self)._core_get_available_guardrail(guardrail_name)
|
||||
|
||||
# -- health ----------------------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def is_deployment_blocked(deployment: "Deployment") -> bool:
|
||||
_note("is_deployment_blocked", "health().is_blocked(deployment)")
|
||||
from litellm.router_api import RouterHealth
|
||||
|
||||
return RouterHealth.is_blocked(deployment)
|
||||
|
||||
_is_deployment_blocked = is_deployment_blocked
|
||||
|
||||
@staticmethod
|
||||
def are_all_deployments_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool:
|
||||
_note("are_all_deployments_blocked", "health().all_blocked(deployments)")
|
||||
from litellm.router_api import RouterHealth
|
||||
|
||||
return RouterHealth.all_blocked(deployments)
|
||||
|
||||
_are_all_deployments_blocked = are_all_deployments_blocked
|
||||
|
||||
def get_fully_blocked_model_names(self) -> set[str]: # mutable-ok: the old return type, unchanged
|
||||
_note("get_fully_blocked_model_names", "health().fully_blocked_model_names()")
|
||||
return _as_router(self)._core_get_fully_blocked_model_names()
|
||||
|
||||
async def async_get_fully_unhealthy_model_names(self) -> set[str]: # mutable-ok: the old return type, unchanged
|
||||
_note("async_get_fully_unhealthy_model_names", "health().fully_unhealthy_model_names()")
|
||||
return await _as_router(self)._core_async_get_fully_unhealthy_model_names()
|
||||
|
||||
async def get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]:
|
||||
_note("get_model_group_usage", "health().usage(group)")
|
||||
return await _as_router(self)._core_get_model_group_usage(model_group=model_group)
|
||||
|
||||
async def get_remaining_model_group_usage(
|
||||
self, model_group: str
|
||||
) -> dict[str, int]: # mutable-ok: the old return type, unchanged
|
||||
_note("get_remaining_model_group_usage", "health().usage(group, kind='remaining')")
|
||||
return await _as_router(self)._core_get_remaining_model_group_usage(model_group=model_group)
|
||||
|
||||
def team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None":
|
||||
_note("team_ptu_ceiling", "health().team_ptu_ceiling(team_id, group)")
|
||||
return _as_router(self)._core_team_ptu_ceiling(team_id, model_group)
|
||||
|
||||
def split_request_priority(
|
||||
self, model: str, kwargs: Mapping[str, object]
|
||||
) -> tuple[int | None, Mapping[str, object]]:
|
||||
_note("split_request_priority", "health().split_priority(model, kwargs)")
|
||||
return _as_router(self)._core_split_request_priority(model, kwargs)
|
||||
|
||||
# -- selection -------------------------------------------------------------------------------
|
||||
async def async_get_healthy_deployments(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict, # mutable-ok: the old parameter type, unchanged
|
||||
messages: list[dict[str, str]] | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
input: str | list | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
specific_deployment: bool | None = False,
|
||||
parent_otel_span: "Span | None" = None,
|
||||
health_check_probe: bool = False,
|
||||
) -> list[dict] | dict: # mutable-ok: the old return type, unchanged
|
||||
_note("async_get_healthy_deployments", "select(model, request_kwargs, kind='healthy')")
|
||||
return await _as_router(self)._core_async_get_healthy_deployments(
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
parent_otel_span=parent_otel_span,
|
||||
health_check_probe=health_check_probe,
|
||||
)
|
||||
|
||||
def get_available_deployment_for_pass_through( # noqa: ANN201 # the old member had no return annotation; its inferred type is kept by inferring it from the core
|
||||
self,
|
||||
model: str,
|
||||
messages: list[dict[str, str]] | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
input: str | list | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
specific_deployment: bool | None = False,
|
||||
request_kwargs: dict | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
):
|
||||
_note("get_available_deployment_for_pass_through", "select(model, kind='pass_through')")
|
||||
return _as_router(self)._core_get_available_deployment_for_pass_through(
|
||||
model=model,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
request_kwargs=request_kwargs,
|
||||
)
|
||||
|
||||
# -- writes ----------------------------------------------------------------------------------
|
||||
def upsert_deployment(self, deployment: "Deployment") -> "Deployment | None":
|
||||
_note("upsert_deployment", "apply(ChangeSet(upsert=[deployment]))")
|
||||
return _as_router(self).apply(ChangeSet(upsert=[deployment])).upserted[0]
|
||||
|
||||
def delete_deployment(self, id: str) -> "Deployment | None":
|
||||
_note("delete_deployment", "apply(ChangeSet(delete=[id]))")
|
||||
return _as_router(self).apply(ChangeSet(delete=[id])).deleted[0]
|
||||
|
||||
def update_settings(self, **kwargs: object) -> None: # kwargs-ok: the old signature, unchanged
|
||||
_note("update_settings", "apply(ChangeSet(settings={...}))")
|
||||
_as_router(self).apply(ChangeSet(settings=kwargs))
|
||||
|
||||
def update_redis_cache(self, cache: "RedisCache") -> None:
|
||||
_note("update_redis_cache", "apply(ChangeSet(redis_cache=cache))")
|
||||
_as_router(self).apply(ChangeSet(redis_cache=cache))
|
||||
|
||||
_update_redis_cache = _private_alias("update_redis_cache", update_redis_cache)
|
||||
|
||||
async def arefresh_model_info(self, *, client: "AsyncHTTPHandler | None" = None) -> None:
|
||||
_note("arefresh_model_info", "aapply(ChangeSet(refresh_model_info=True))")
|
||||
await _as_router(self).aapply(ChangeSet(refresh_model_info=True, refresh_client=client))
|
||||
|
||||
# -- plugins, observers, lifecycle -----------------------------------------------------------
|
||||
def set_custom_routing_strategy(self, CustomRoutingStrategy: "CustomRoutingStrategyBase") -> None:
|
||||
_note("set_custom_routing_strategy", "register_plugin(strategy)")
|
||||
_as_router(self).register_plugin(CustomRoutingStrategy)
|
||||
|
||||
def serve_through_request_logging(self) -> None:
|
||||
_note("serve_through_request_logging", "observe(request_scoped=True)")
|
||||
_as_router(self).observe(request_scoped=True)
|
||||
|
||||
def bind_to_request_logging(self, logging_obj: "LiteLLMLogging") -> None:
|
||||
_note("bind_to_request_logging", "observe(logging_obj)")
|
||||
# Straight to the core, as before: observe() skips binding when logging_obj is None, the old member did not.
|
||||
_as_router(self)._core_bind_to_request_logging(logging_obj)
|
||||
|
||||
def discard(self) -> None:
|
||||
_note("discard", "retire()")
|
||||
_as_router(self).retire()
|
||||
|
||||
# -- plugin context (router strategies) ------------------------------------------------------
|
||||
def arm_routing_read_prefetch(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict[str, object] | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
) -> None:
|
||||
_note("arm_routing_read_prefetch", "RoutingContext.arm_read_prefetch")
|
||||
_as_router(self)._core_arm_routing_read_prefetch(model, request_kwargs)
|
||||
|
||||
async def async_pre_routing_hook(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict[str, object], # mutable-ok: the old parameter type, unchanged
|
||||
messages: "list[dict[str, Any]] | None" = None, # pyright: ignore[reportExplicitAny] # the old type # mutable-ok: same
|
||||
input: str | list | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
specific_deployment: bool | None = False,
|
||||
) -> "PreRoutingHookResponse | None":
|
||||
_note("async_pre_routing_hook", "RoutingContext.pre_routing_hook")
|
||||
return await _as_router(self)._core_async_pre_routing_hook(
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
)
|
||||
|
||||
def log_retry(self, kwargs: dict, e: Exception) -> dict: # mutable-ok: the old signature, unchanged
|
||||
_note("log_retry", "RoutingContext.log_retry")
|
||||
return _as_router(self)._core_log_retry(kwargs=kwargs, e=e)
|
||||
|
||||
|
||||
def _name_members_after_router() -> None:
|
||||
# The old members were Router methods, so errors that name the method (a wrong-arity TypeError,
|
||||
# or "litellm.router.Router.<name>() argument after ** must be a mapping") keep naming it the
|
||||
# way they did before the members moved here.
|
||||
for member in vars(DeprecatedRouterMembers).values():
|
||||
function = member.__func__ if isinstance(member, staticmethod) else member
|
||||
if inspect.isfunction(function):
|
||||
function.__qualname__ = f"Router.{function.__name__}"
|
||||
function.__module__ = "litellm.router"
|
||||
|
||||
|
||||
_name_members_after_router()
|
||||
|
|
@ -186,6 +186,14 @@ ignored_function_names = [
|
|||
]
|
||||
|
||||
|
||||
def public_name(function_name):
|
||||
"""
|
||||
The deprecated public name a ``_core_<name>`` router member is reached through. Tests that call the
|
||||
old name run the core member, since ``DeprecatedRouterMembers`` forwards it there.
|
||||
"""
|
||||
return function_name.removeprefix("_core_")
|
||||
|
||||
|
||||
def main():
|
||||
router_file = [
|
||||
"./litellm/router.py",
|
||||
|
|
@ -205,12 +213,16 @@ def main():
|
|||
router_functions.extend(get_functions_from_router(file))
|
||||
print("router_functions: ", router_functions)
|
||||
called_functions_in_tests = get_all_functions_called_in_tests(tests_dir)
|
||||
untested_functions = [fn for fn in router_functions if fn not in called_functions_in_tests]
|
||||
untested_functions = [
|
||||
fn
|
||||
for fn in router_functions
|
||||
if fn not in called_functions_in_tests and public_name(fn) not in called_functions_in_tests
|
||||
]
|
||||
|
||||
if untested_functions:
|
||||
all_untested_functions = []
|
||||
for func in untested_functions:
|
||||
if func not in ignored_function_names:
|
||||
if func not in ignored_function_names and public_name(func) not in ignored_function_names:
|
||||
all_untested_functions.append(func)
|
||||
untested_perc = (len(all_untested_functions)) / len(router_functions)
|
||||
print("untested_perc: ", untested_perc)
|
||||
|
|
|
|||
|
|
@ -37,9 +37,11 @@ ALLOWLIST = os.path.join(os.path.dirname(__file__), "router_surface_allowlist.js
|
|||
|
||||
ROUTER_FILES = {
|
||||
"litellm/router.py",
|
||||
"litellm/router_api.py",
|
||||
"litellm/router_deprecated.py",
|
||||
}
|
||||
ROUTER_CLASS_FILES = ["litellm/router.py"]
|
||||
ROUTER_CLASS_NAMES = {"Router"}
|
||||
ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_deprecated.py"]
|
||||
ROUTER_CLASS_NAMES = {"Router", "DeprecatedRouterMembers"}
|
||||
|
||||
SCAN_ROOTS = ["litellm", "enterprise"]
|
||||
SKIP_DIRS = {
|
||||
|
|
|
|||
|
|
@ -53,10 +53,10 @@ class TestFilterHealthCheckUnhealthyDeployments:
|
|||
|
||||
class FakeRouter:
|
||||
def __init__(self):
|
||||
self.enable_health_check_routing = enable
|
||||
self.health_state_cache = health_cache
|
||||
self.allowed_fails_policy = None
|
||||
self.background_health_check_model_groups = model_groups
|
||||
self._core_enable_health_check_routing = enable
|
||||
self._core_health_state_cache = health_cache
|
||||
self._core_allowed_fails_policy = None
|
||||
self._core_background_health_check_model_groups = model_groups
|
||||
|
||||
# Import the actual method and bind it
|
||||
from litellm.router import Router
|
||||
|
|
@ -169,10 +169,10 @@ class TestAsyncFilterHealthCheckUnhealthyDeployments:
|
|||
|
||||
class FakeRouter:
|
||||
def __init__(self):
|
||||
self.enable_health_check_routing = enable
|
||||
self.health_state_cache = health_cache
|
||||
self.allowed_fails_policy = None
|
||||
self.background_health_check_model_groups = model_groups
|
||||
self._core_enable_health_check_routing = enable
|
||||
self._core_health_state_cache = health_cache
|
||||
self._core_allowed_fails_policy = None
|
||||
self._core_background_health_check_model_groups = model_groups
|
||||
|
||||
fake = FakeRouter()
|
||||
fake._async_filter_health_check_unhealthy_deployments = (
|
||||
|
|
|
|||
|
|
@ -5242,10 +5242,11 @@ def test_public_aliases(
|
|||
resolved_owner: Final = _get_owner(module_path, owner_name)
|
||||
alias_owner: Final = _get_instance(resolved_owner, new_name) if use_instance else resolved_owner
|
||||
old_value, new_value = getattr(alias_owner, old_name), getattr(alias_owner, new_name)
|
||||
# Router's private aliases wrap the public method so they skip a subclass override of it.
|
||||
if isinstance(old_value, MethodType) and isinstance(new_value, MethodType):
|
||||
assert old_value.__func__ is new_value.__func__
|
||||
assert old_value.__func__ is new_value.__func__ or inspect.unwrap(old_value.__func__) is new_value.__func__
|
||||
else:
|
||||
assert old_value is new_value
|
||||
assert old_value is new_value or inspect.unwrap(old_value) is new_value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
|
|
@ -2321,7 +2321,7 @@ def test_with_router_resolved_session_model(session, expected):
|
|||
|
||||
def test_router_get_model_access_groups_team_only_models():
|
||||
"""
|
||||
Test that Router.get_model_access_groups returns the correct response for team-only models
|
||||
Test that Router._core_get_model_access_groups returns the correct response for team-only models
|
||||
"""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
|
|
@ -2440,7 +2440,7 @@ def test_model_group_info_cost_from_db_model_info():
|
|||
)
|
||||
|
||||
with patch.object(
|
||||
router, "get_deployment_model_info", side_effect=Exception("not found")
|
||||
router, "_core_get_deployment_model_info", side_effect=Exception("not found")
|
||||
):
|
||||
result = router._cached_get_model_group_info("my-custom-model")
|
||||
assert result is not None
|
||||
|
|
@ -2470,7 +2470,7 @@ def test_model_group_info_cost_none_when_db_model_info_has_no_cost():
|
|||
)
|
||||
|
||||
with patch.object(
|
||||
router, "get_deployment_model_info", side_effect=Exception("not found")
|
||||
router, "_core_get_deployment_model_info", side_effect=Exception("not found")
|
||||
):
|
||||
result = router._cached_get_model_group_info("my-custom-model-no-cost")
|
||||
assert result is not None
|
||||
|
|
@ -2854,7 +2854,7 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto
|
|||
},
|
||||
)
|
||||
assert router.routing_strategy == "lar1"
|
||||
assert "async_get_available_deployment" in router.__dict__
|
||||
assert "_core_async_get_available_deployment" in router.__dict__
|
||||
|
||||
router._switch_routing_strategy("usage-based-routing-v2", {})
|
||||
assert router.lowesttpm_logger_v2 is not None
|
||||
|
|
@ -2925,7 +2925,7 @@ def test_model_group_info_with_stringified_cost_values():
|
|||
return None
|
||||
|
||||
with patch.object(
|
||||
router, "get_deployment_model_info", side_effect=_model_info_with_str_costs
|
||||
router, "_core_get_deployment_model_info", side_effect=_model_info_with_str_costs
|
||||
):
|
||||
result = router._set_model_group_info(
|
||||
model_group="my-custom-model",
|
||||
|
|
@ -2973,7 +2973,7 @@ def test_model_group_info_db_fallback_with_stringified_cost_values():
|
|||
)
|
||||
|
||||
with patch.object(
|
||||
router, "get_deployment_model_info", side_effect=Exception("not found")
|
||||
router, "_core_get_deployment_model_info", side_effect=Exception("not found")
|
||||
):
|
||||
result = router._set_model_group_info(
|
||||
model_group="my-custom-model",
|
||||
|
|
@ -7828,7 +7828,7 @@ async def test_async_function_with_fallbacks_common_utils():
|
|||
|
||||
|
||||
def test_should_include_deployment():
|
||||
"""Test that Router.should_include_deployment returns the correct response"""
|
||||
"""Test that Router._core_should_include_deployment returns the correct response"""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -7883,7 +7883,7 @@ def test_pre_call_checks_skips_token_count_without_max_input_tokens(monkeypatch)
|
|||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: {})
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: {})
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -7917,7 +7917,7 @@ def test_pre_call_checks_counts_once_and_filters_on_max_input_tokens(monkeypatch
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
calls = []
|
||||
|
|
@ -8031,7 +8031,7 @@ def test_pre_call_checks_uses_precounted_tokens(monkeypatch):
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
calls = []
|
||||
|
|
@ -8065,7 +8065,7 @@ async def test_async_get_healthy_deployments_counts_tokens_off_the_event_loop(mo
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 1_000_000}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 1_000_000}
|
||||
)
|
||||
|
||||
counting_threads = []
|
||||
|
|
@ -8115,7 +8115,7 @@ def test_pre_call_checks_need_token_count(monkeypatch, model_info, expected):
|
|||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: model_info)
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: model_info)
|
||||
|
||||
deployments = [
|
||||
{"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "d1"}},
|
||||
|
|
@ -8141,7 +8141,7 @@ def test_deployment_max_input_tokens_survives_an_unmappable_deployment(monkeypat
|
|||
raise ValueError("This model isn't mapped yet.")
|
||||
return {"max_input_tokens": 100}
|
||||
|
||||
monkeypatch.setattr(router, "get_router_model_info", flaky_model_info)
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", flaky_model_info)
|
||||
|
||||
unmapped = {"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "unmapped"}}
|
||||
mapped = {"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "mapped"}}
|
||||
|
|
@ -8163,7 +8163,7 @@ def test_pre_call_checks_does_not_recount_inline_after_an_off_loop_failure(monke
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
calls = []
|
||||
|
|
@ -8198,7 +8198,7 @@ async def test_async_get_healthy_deployments_never_recounts_on_the_loop(monkeypa
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
counting_threads = []
|
||||
|
|
@ -8235,7 +8235,7 @@ async def test_acount_pre_call_check_tokens_leaves_the_event_loop_free(monkeypat
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
deployments = [
|
||||
|
|
@ -8269,7 +8269,7 @@ async def test_acount_pre_call_check_tokens_skips_without_max_input_tokens(monke
|
|||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: {})
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: {})
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -8303,7 +8303,7 @@ def test_pre_call_checks_counts_tokens_from_responses_input_string(monkeypatch):
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 1}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 1}
|
||||
)
|
||||
|
||||
deployments = [
|
||||
|
|
@ -8330,7 +8330,7 @@ def test_pre_call_checks_counts_tokens_from_responses_input_list(monkeypatch):
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 1}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 1}
|
||||
)
|
||||
|
||||
deployments = [
|
||||
|
|
@ -8374,7 +8374,7 @@ def test_pre_call_checks_counts_responses_instructions_tokens(monkeypatch):
|
|||
assert with_instructions_tokens > input_only_tokens
|
||||
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": input_only_tokens}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": input_only_tokens}
|
||||
)
|
||||
with pytest.raises(litellm.ContextWindowExceededError):
|
||||
router._pre_call_checks(
|
||||
|
|
@ -8445,7 +8445,7 @@ def test_pre_call_checks_counts_tool_definition_tokens(monkeypatch, prompt_kwarg
|
|||
messages=prompt_kwargs.get("messages"), input=prompt_kwargs.get("input")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": prompt_only_tokens}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": prompt_only_tokens}
|
||||
)
|
||||
|
||||
assert len(router._pre_call_checks(model="m", healthy_deployments=deployments, **prompt_kwargs)) == 1
|
||||
|
|
@ -8486,7 +8486,7 @@ def test_pre_call_checks_counts_anthropic_system_tokens(monkeypatch, system):
|
|||
messages = [{"role": "user", "content": "hi"}]
|
||||
|
||||
messages_only_tokens = router._count_pre_call_check_tokens(messages=messages, input=None)
|
||||
monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": messages_only_tokens})
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": messages_only_tokens})
|
||||
|
||||
assert len(router._pre_call_checks(model="m", healthy_deployments=deployments, messages=messages)) == 1
|
||||
with pytest.raises(litellm.ContextWindowExceededError):
|
||||
|
|
@ -8605,7 +8605,7 @@ def test_pre_call_checks_no_messages_or_input_does_not_crash(monkeypatch):
|
|||
enable_pre_call_checks=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5}
|
||||
)
|
||||
|
||||
counted: list[dict] = []
|
||||
|
|
@ -9952,7 +9952,7 @@ def test_deployment_usable_by_team_uses_dynamic_model_info_get():
|
|||
def get(self, key: str) -> str | None:
|
||||
return "team-b" if key == "team_id" else None
|
||||
|
||||
assert litellm.Router.deployment_usable_by_team({"model_info": ModelInfo()}, "team-a") is False
|
||||
assert litellm.Router._core_deployment_usable_by_team({"model_info": ModelInfo()}, "team-a") is False
|
||||
|
||||
|
||||
def test_get_deployment_credentials_with_provider_skips_other_team_wildcard():
|
||||
|
|
@ -11599,7 +11599,7 @@ def test_access_group_filter_empty_does_not_bypass_via_litellm_model_fallback(
|
|||
team_id=team_id,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(router, "get_model_access_groups", fake_get_model_access_groups)
|
||||
monkeypatch.setattr(router, "_core_get_model_access_groups", fake_get_model_access_groups)
|
||||
|
||||
scoped_key = UserAPIKeyAuth(
|
||||
api_key="hashed-key",
|
||||
|
|
@ -11676,7 +11676,7 @@ def test_access_group_block_does_not_silently_use_default_fallback_model(
|
|||
team_id=team_id,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(router, "get_model_access_groups", fake_get_model_access_groups)
|
||||
monkeypatch.setattr(router, "_core_get_model_access_groups", fake_get_model_access_groups)
|
||||
|
||||
scoped_key = UserAPIKeyAuth(
|
||||
api_key="hashed-key",
|
||||
|
|
@ -11745,7 +11745,7 @@ def test_access_group_block_via_litellm_model_branch_does_not_use_default_fallba
|
|||
team_id=team_id,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(router, "get_model_access_groups", fake_get_model_access_groups)
|
||||
monkeypatch.setattr(router, "_core_get_model_access_groups", fake_get_model_access_groups)
|
||||
|
||||
scoped_key = UserAPIKeyAuth(
|
||||
api_key="hashed-key",
|
||||
|
|
@ -12246,7 +12246,7 @@ def test_get_deployment_credentials_with_provider_returns_none_for_blocked_deplo
|
|||
|
||||
def test_is_deployment_blocked_static_helper_reflects_blocked_flag():
|
||||
"""
|
||||
Exercises Router.is_deployment_blocked so router_code_coverage.py (AST call graph)
|
||||
Exercises Router._core_is_deployment_blocked so router_code_coverage.py (AST call graph)
|
||||
marks the helper as covered by router-named tests.
|
||||
"""
|
||||
import types
|
||||
|
|
@ -12257,15 +12257,15 @@ def test_is_deployment_blocked_static_helper_reflects_blocked_flag():
|
|||
blocked_dep = router.get_deployment("dep-0")
|
||||
unblocked_dep = router.get_deployment("dep-1")
|
||||
assert blocked_dep is not None and unblocked_dep is not None
|
||||
assert litellm.Router.is_deployment_blocked(blocked_dep) is True
|
||||
assert litellm.Router.is_deployment_blocked(unblocked_dep) is False
|
||||
assert litellm.Router._core_is_deployment_blocked(blocked_dep) is True
|
||||
assert litellm.Router._core_is_deployment_blocked(unblocked_dep) is False
|
||||
|
||||
# No model_info on deployment object → treated as not blocked
|
||||
assert litellm.Router.is_deployment_blocked(object()) is False
|
||||
assert litellm.Router._core_is_deployment_blocked(object()) is False
|
||||
missing_blocked = types.SimpleNamespace()
|
||||
assert litellm.Router.is_deployment_blocked(types.SimpleNamespace(model_info=missing_blocked)) is False
|
||||
assert litellm.Router._core_is_deployment_blocked(types.SimpleNamespace(model_info=missing_blocked)) is False
|
||||
assert (
|
||||
litellm.Router.is_deployment_blocked(types.SimpleNamespace(model_info=types.SimpleNamespace(blocked=True)))
|
||||
litellm.Router._core_is_deployment_blocked(types.SimpleNamespace(model_info=types.SimpleNamespace(blocked=True)))
|
||||
is True
|
||||
)
|
||||
|
||||
|
|
@ -14991,7 +14991,7 @@ def test_pre_call_checks_uses_deployment_model_when_model_info_lookup_raises(mon
|
|||
def _raise_unmapped(**kwargs):
|
||||
raise ValueError("This model isn't mapped yet")
|
||||
|
||||
monkeypatch.setattr(router, "get_router_model_info", _raise_unmapped)
|
||||
monkeypatch.setattr(router, "_core_get_router_model_info", _raise_unmapped)
|
||||
|
||||
seen: list[tuple] = []
|
||||
original_get_supported_openai_params = litellm.get_supported_openai_params
|
||||
|
|
@ -17182,7 +17182,7 @@ def test_model_group_info_intersects_supported_reasoning_efforts():
|
|||
"supports_xhigh_reasoning_effort": False,
|
||||
}
|
||||
|
||||
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
|
||||
with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info):
|
||||
result = router._set_model_group_info(
|
||||
model_group="smart-group",
|
||||
user_facing_model_group_name="smart-group",
|
||||
|
|
@ -17226,7 +17226,7 @@ def test_model_group_info_reasoning_efforts_are_unknown_when_any_deployment_is_o
|
|||
}
|
||||
return {"key": model_name, "litellm_provider": "openai", "mode": None, "supports_reasoning": None}
|
||||
|
||||
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
|
||||
with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info):
|
||||
result = router._set_model_group_info(
|
||||
model_group="smart-group",
|
||||
user_facing_model_group_name="smart-group",
|
||||
|
|
@ -17361,7 +17361,7 @@ def test_model_group_info_reasoning_efforts_empty_on_a_mapped_non_reasoning_depl
|
|||
}
|
||||
return {"key": model_name, "litellm_provider": "openai", "mode": "chat", "supports_reasoning": None}
|
||||
|
||||
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
|
||||
with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info):
|
||||
result = router._set_model_group_info(
|
||||
model_group="mixed-group",
|
||||
user_facing_model_group_name="mixed-group",
|
||||
|
|
@ -17402,7 +17402,7 @@ def test_model_group_info_reasoning_efforts_ignore_a_value_declared_in_model_inf
|
|||
info["supported_reasoning_efforts"] = ("high",)
|
||||
return info
|
||||
|
||||
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
|
||||
with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info):
|
||||
result = router._set_model_group_info(
|
||||
model_group="declared-group",
|
||||
user_facing_model_group_name="declared-group",
|
||||
|
|
@ -17436,7 +17436,7 @@ def test_model_group_info_survives_a_junk_typed_operator_effort_value():
|
|||
"supported_reasoning_efforts": "high",
|
||||
}
|
||||
|
||||
with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
|
||||
with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info):
|
||||
result = router._set_model_group_info(
|
||||
model_group="junk-declared-group",
|
||||
user_facing_model_group_name="junk-declared-group",
|
||||
|
|
@ -23993,7 +23993,7 @@ def test_a_failed_routing_read_prefetch_logs_the_request_model_without_its_line_
|
|||
def fail_lookup(model_name: str | None = None, team_id: str | None = None) -> None:
|
||||
raise RuntimeError(f"no deployments for {model_name}")
|
||||
|
||||
monkeypatch.setattr(router, "get_model_list", fail_lookup)
|
||||
monkeypatch.setattr(router, "_core_get_model_list", fail_lookup)
|
||||
caplog.clear()
|
||||
|
||||
with caplog.at_level(logging.DEBUG, logger="LiteLLM Router"):
|
||||
|
|
@ -26003,7 +26003,7 @@ async def test_router_acancel_batch_removes_unresolved_credential_name():
|
|||
mock_response.status = "cancelled"
|
||||
|
||||
with (
|
||||
patch.object(router, "get_deployment_credentials_with_provider", return_value=None),
|
||||
patch.object(router, "_core_get_deployment_credentials_with_provider", return_value=None),
|
||||
patch.object(litellm, "acancel_batch", new_callable=AsyncMock) as mock_cancel,
|
||||
):
|
||||
mock_cancel.return_value = mock_response
|
||||
|
|
@ -29090,7 +29090,7 @@ def test_reset_custom_routing_strategy():
|
|||
router.set_custom_routing_strategy(SpecialEndpointRoutingStrategy(router))
|
||||
|
||||
assert "get_available_deployment" in router.__dict__
|
||||
assert "async_get_available_deployment" in router.__dict__
|
||||
assert "_core_async_get_available_deployment" in router.__dict__
|
||||
|
||||
router._reset_custom_routing_strategy()
|
||||
|
||||
|
|
@ -31179,7 +31179,7 @@ def test_team_model_has_alternatives_when_a_delete_starts_after_its_deployment_l
|
|||
has_alternatives: Final = _read_during_held_write(
|
||||
lambda: router.team_model_has_alternatives("deployment-3"),
|
||||
lambda: router.delete_deployment("deployment-0"),
|
||||
write_after=Router.get_deployment.__code__,
|
||||
write_after=Router._core_get_deployment.__code__,
|
||||
)
|
||||
|
||||
assert has_alternatives is True
|
||||
|
|
|
|||
829
tests/unit/test_router/test_router_entry_points.py
Normal file
829
tests/unit/test_router/test_router_entry_points.py
Normal file
|
|
@ -0,0 +1,829 @@
|
|||
"""
|
||||
Router entry points and DeprecatedRouterMembers.
|
||||
|
||||
Every member that moved into DeprecatedRouterMembers must return what its entry point returns,
|
||||
warn once per access from outside the router, and count the access. Internal access (the router's
|
||||
own modules) must stay silent.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import dataclasses
|
||||
import warnings
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.router_api import SETTING_FIELDS, ChangeSet, RouterSettings
|
||||
from litellm.router_deprecated import DEPRECATED_ROUTER_MEMBER_USES, DeprecatedRouterMembers
|
||||
from litellm.types.router import Deployment
|
||||
|
||||
|
||||
def _router() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"},
|
||||
"model_info": {"id": "dep-a", "access_groups": ["grp"]},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt",
|
||||
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-b"},
|
||||
"model_info": {"id": "dep-b"},
|
||||
},
|
||||
{
|
||||
"model_name": "team-internal",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-t"},
|
||||
"model_info": {"id": "dep-team", "team_id": "team-1", "team_public_model_name": "team-gpt"},
|
||||
},
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {"model": "openai/*", "api_key": "sk-w"},
|
||||
"model_info": {"id": "dep-wild"},
|
||||
},
|
||||
],
|
||||
model_group_alias={"alias-gpt": "gpt"},
|
||||
fallbacks=[{"gpt": ["team-internal"]}],
|
||||
cooldown_time=7,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def router():
|
||||
r = _router()
|
||||
yield r
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
r.retire()
|
||||
|
||||
|
||||
# (old member, how to read it through the old name, how to read it through the entry point)
|
||||
READS = [
|
||||
("model_list", lambda r: r.model_list, lambda r: r.registry().all()),
|
||||
("model_names", lambda r: r.model_names, lambda r: r.registry().names(kind="group_set")),
|
||||
("deployment_names", lambda r: r.deployment_names, lambda r: r.registry().names(kind="deployment_names")),
|
||||
("team_public_model_names", lambda r: r.team_public_model_names, lambda r: r.registry().names(kind="team_public")),
|
||||
(
|
||||
"model_name_to_deployment_indices",
|
||||
lambda r: r.model_name_to_deployment_indices,
|
||||
lambda r: r.registry().index("model_name"),
|
||||
),
|
||||
(
|
||||
"team_model_to_deployment_indices",
|
||||
lambda r: r.team_model_to_deployment_indices,
|
||||
lambda r: r.registry().index("team_model"),
|
||||
),
|
||||
("model_group_alias", lambda r: r.model_group_alias, lambda r: r.registry().aliases()),
|
||||
("pattern_router", lambda r: r.pattern_router, lambda r: r.registry().patterns()),
|
||||
("team_pattern_routers", lambda r: r.team_pattern_routers, lambda r: r.registry().patterns(team=True)),
|
||||
("routing_groups", lambda r: r.routing_groups, lambda r: r.registry().routing_groups()),
|
||||
("auto_routers", lambda r: r.auto_routers, lambda r: r.registry().plugins("auto")),
|
||||
("complexity_routers", lambda r: r.complexity_routers, lambda r: r.registry().plugins("complexity")),
|
||||
("quality_routers", lambda r: r.quality_routers, lambda r: r.registry().plugins("quality")),
|
||||
("adaptive_routers", lambda r: r.adaptive_routers, lambda r: r.registry().plugins("adaptive")),
|
||||
("routing_plugins", lambda r: r.routing_plugins, lambda r: r.registry().plugins("routing")),
|
||||
(
|
||||
"prompt_caching_pin_cache",
|
||||
lambda r: r.prompt_caching_pin_cache,
|
||||
lambda r: r.registry().plugins("prompt_caching_pins"),
|
||||
),
|
||||
("cache", lambda r: r.cache, lambda r: r.health().store),
|
||||
("cooldown_cache", lambda r: r.cooldown_cache, lambda r: r.health().cooldowns),
|
||||
("health_state_cache", lambda r: r.health_state_cache, lambda r: r.health().health_checks),
|
||||
("deployment_latency_map", lambda r: r.deployment_latency_map, lambda r: r.health().latency),
|
||||
("get_deployment", lambda r: r.get_deployment("dep-a"), lambda r: r.registry().deployment("dep-a")),
|
||||
("has_model_id", lambda r: r.has_model_id("dep-b"), lambda r: r.registry().has_deployment("dep-b")),
|
||||
("get_model_info", lambda r: r.get_model_info("dep-a"), lambda r: r.registry().deployment_dict("dep-a")),
|
||||
("get_model_list", lambda r: r.get_model_list("gpt"), lambda r: r.registry().deployments("gpt")),
|
||||
("get_model_list_all", lambda r: r.get_model_list(), lambda r: r.registry().deployments()),
|
||||
("get_model_list_alias", lambda r: r.get_model_list("alias-gpt"), lambda r: r.registry().deployments("alias-gpt")),
|
||||
(
|
||||
"get_all_deployments",
|
||||
lambda r: r.get_all_deployments("gpt"),
|
||||
lambda r: r.registry().deployments("gpt", source="all"),
|
||||
),
|
||||
(
|
||||
"config_deployments",
|
||||
lambda r: list(r.config_deployments()),
|
||||
lambda r: list(r.registry().deployments(source="config")),
|
||||
),
|
||||
(
|
||||
"deployments_named",
|
||||
lambda r: r.deployments_named("gpt"),
|
||||
lambda r: r.registry().deployments("gpt", source="named"),
|
||||
),
|
||||
(
|
||||
"get_model_list_from_model_alias",
|
||||
lambda r: r.get_model_list_from_model_alias("alias-gpt"),
|
||||
lambda r: r.registry().deployments("alias-gpt", source="alias"),
|
||||
),
|
||||
(
|
||||
"get_deployment_by_model_group_name",
|
||||
lambda r: r.get_deployment_by_model_group_name("gpt"),
|
||||
lambda r: r.registry().deployments("gpt", source="first_in_group"),
|
||||
),
|
||||
("get_model_names", lambda r: r.get_model_names(), lambda r: r.registry().names()),
|
||||
(
|
||||
"get_model_names_team",
|
||||
lambda r: r.get_model_names(team_id="team-1"),
|
||||
lambda r: r.registry().names(team_id="team-1"),
|
||||
),
|
||||
("get_model_ids", lambda r: r.get_model_ids("gpt"), lambda r: r.registry().names("gpt", kind="ids")),
|
||||
(
|
||||
"get_model_ids_all",
|
||||
lambda r: r.get_model_ids(exclude_team_models=True),
|
||||
lambda r: r.registry().names(kind="ids", exclude_team_models=True),
|
||||
),
|
||||
(
|
||||
"get_model_from_alias",
|
||||
lambda r: r.get_model_from_alias("alias-gpt"),
|
||||
lambda r: r.registry().resolve("alias-gpt", kind="alias"),
|
||||
),
|
||||
(
|
||||
"is_recognized_model",
|
||||
lambda r: r.is_recognized_model("gpt"),
|
||||
lambda r: r.registry().resolve("gpt", kind="recognized"),
|
||||
),
|
||||
(
|
||||
"routable_model_group",
|
||||
lambda r: r.routable_model_group("alias-gpt"),
|
||||
lambda r: r.registry().resolve("alias-gpt"),
|
||||
),
|
||||
(
|
||||
"map_team_model",
|
||||
lambda r: r.map_team_model("team-gpt", "team-1"),
|
||||
lambda r: r.registry().resolve("team-gpt", kind="team_model", team_id="team-1"),
|
||||
),
|
||||
(
|
||||
"get_model_group",
|
||||
lambda r: r.get_model_group("dep-a"),
|
||||
lambda r: r.registry().resolve("dep-a", kind="group_of_id"),
|
||||
),
|
||||
(
|
||||
"get_search_tool_support",
|
||||
lambda r: r.get_search_tool_support("gpt"),
|
||||
lambda r: r.registry().resolve("gpt", kind="search_tool_support"),
|
||||
),
|
||||
("get_model_access_groups", lambda r: r.get_model_access_groups(), lambda r: r.registry().access_groups()),
|
||||
(
|
||||
"get_deployment_credentials",
|
||||
lambda r: r.get_deployment_credentials("dep-a"),
|
||||
lambda r: r.registry().credentials("dep-a"),
|
||||
),
|
||||
(
|
||||
"get_deployment_credentials_with_provider",
|
||||
lambda r: r.get_deployment_credentials_with_provider("dep-a"),
|
||||
lambda r: r.registry().credentials("dep-a", form="with_provider"),
|
||||
),
|
||||
(
|
||||
"get_credential_deployment",
|
||||
lambda r: r.get_credential_deployment("dep-a"),
|
||||
lambda r: r.registry().credentials("dep-a", form="deployment"),
|
||||
),
|
||||
("get_configured_mode", lambda r: r.get_configured_mode("gpt"), lambda r: r.registry().group("gpt", field="mode")),
|
||||
(
|
||||
"get_configured_token_limits",
|
||||
lambda r: r.get_configured_token_limits("gpt"),
|
||||
lambda r: r.registry().group("gpt", field="token_limits"),
|
||||
),
|
||||
(
|
||||
"get_fully_blocked_model_names",
|
||||
lambda r: r.get_fully_blocked_model_names(),
|
||||
lambda r: r.health().fully_blocked_model_names(),
|
||||
),
|
||||
("get_settings", lambda r: r.get_settings(), lambda r: r.settings().as_dict()),
|
||||
("fallbacks", lambda r: r.fallbacks, lambda r: r.settings().fallbacks),
|
||||
("cooldown_time", lambda r: r.cooldown_time, lambda r: r.settings().cooldown_time),
|
||||
("routing_strategy", lambda r: r.routing_strategy, lambda r: r.settings().routing_strategy),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name, old, new", READS, ids=[r[0] for r in READS])
|
||||
def test_deprecated_read_matches_entry_point(router, name, old, new):
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
old_value = old(router)
|
||||
new_value = new(router)
|
||||
if isinstance(old_value, (list, dict, set)) or hasattr(old_value, "__dict__"):
|
||||
assert old_value == new_value
|
||||
else:
|
||||
assert old_value == new_value
|
||||
assert any(issubclass(w.category, DeprecationWarning) for w in caught), name
|
||||
|
||||
|
||||
def test_live_objects_not_copies(router):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert router.model_list is router.registry().all()
|
||||
assert router.fallbacks is router.settings().fallbacks
|
||||
assert router.cache is router.health().store
|
||||
assert router.model_group_alias is router.registry().aliases()
|
||||
|
||||
|
||||
def test_assignment_goes_through_apply(router):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
router.fallbacks = [{"gpt": ["openai/gpt-4.1"]}]
|
||||
assert router.settings().fallbacks == [{"gpt": ["openai/gpt-4.1"]}]
|
||||
router.search_tools = [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily"}}]
|
||||
assert router.settings().search_tools[0]["search_tool_name"] == "s"
|
||||
router.apply(ChangeSet(assign={"cooldown_time": 3}))
|
||||
assert router.settings().cooldown_time == 3
|
||||
with pytest.raises(ValueError, match="not assignable router settings"):
|
||||
router.apply(ChangeSet(assign={"not_a_setting": 1}))
|
||||
|
||||
|
||||
def test_apply_upsert_and_delete_bump_version(router):
|
||||
v0 = router.registry().version
|
||||
added = Deployment(
|
||||
model_name="new-group",
|
||||
litellm_params={"model": "openai/gpt-4o-mini", "api_key": "sk-n"},
|
||||
model_info={"id": "dep-new"},
|
||||
)
|
||||
result = router.apply(ChangeSet(upsert=[added]))
|
||||
assert result.version == v0 + 1
|
||||
assert router.registry().has_deployment("dep-new")
|
||||
assert router.registry().deployments("new-group")[0]["model_info"]["id"] == "dep-new"
|
||||
result = router.apply(ChangeSet(delete=["dep-new"]))
|
||||
assert result.version == v0 + 2
|
||||
assert result.deleted[0] is not None
|
||||
assert not router.registry().has_deployment("dep-new")
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert router.upsert_deployment(added) is added
|
||||
assert router.upsert_deployment(added) is None # unchanged
|
||||
assert router.delete_deployment("dep-new") is not None
|
||||
|
||||
|
||||
def test_old_deployment_writes_also_advance_the_version(router):
|
||||
v0 = router.registry().version
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
router.add_deployment(
|
||||
Deployment(
|
||||
model_name="added",
|
||||
litellm_params={"model": "openai/gpt-4o-mini", "api_key": "sk-x"},
|
||||
model_info={"id": "dep-added"},
|
||||
)
|
||||
)
|
||||
v1 = router.registry().version
|
||||
router.set_model_list([{"model_name": "only", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-o"}}])
|
||||
assert v0 < v1 < router.registry().version
|
||||
assert router.registry().names() == ["only"]
|
||||
v2 = router.registry().version
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
router.model_list = []
|
||||
assert router.registry().version > v2
|
||||
|
||||
|
||||
def test_update_settings_through_apply(router):
|
||||
router.apply(ChangeSet(settings={"cooldown_time": "11", "not_allowed": 1}))
|
||||
assert router.settings().cooldown_time == 11
|
||||
|
||||
|
||||
def test_aapply_applies_changes_then_refreshes_model_info(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "local",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/local-model",
|
||||
"api_base": "https://capacity.test/v1",
|
||||
"api_key": "local-key",
|
||||
},
|
||||
"model_info": {"id": "dep-local"},
|
||||
}
|
||||
]
|
||||
)
|
||||
with pytest.raises(TypeError, match="needs aapply"):
|
||||
router.apply(ChangeSet(refresh_model_info=True))
|
||||
|
||||
async def refresh() -> None:
|
||||
handler = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]})
|
||||
)
|
||||
) as client:
|
||||
handler.client = client
|
||||
await router.aapply(ChangeSet(assign={"cooldown_time": 4}, refresh_model_info=True, refresh_client=handler))
|
||||
|
||||
asyncio.run(refresh())
|
||||
assert router.settings().cooldown_time == 4
|
||||
assert router.registry().group("local", field="token_limits") == (4096, 4096)
|
||||
|
||||
|
||||
def test_settings_snapshot_is_reused_until_a_setting_is_written(router):
|
||||
first = router.settings()
|
||||
assert router.settings() is first
|
||||
router.apply(ChangeSet(assign={"cooldown_time": 7}))
|
||||
assert router.settings() is not first
|
||||
assert router.settings().cooldown_time == 7
|
||||
router.apply(ChangeSet(settings={"cooldown_time": "8"}))
|
||||
assert router.settings().cooldown_time == 8
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
router.fallbacks = [{"gpt": ["team-gpt"]}]
|
||||
assert router.settings().fallbacks == [{"gpt": ["team-gpt"]}]
|
||||
router._core_fallbacks = [{"gpt": ["other"]}] # the router's own writes invalidate it too
|
||||
assert router.settings().fallbacks == [{"gpt": ["other"]}]
|
||||
router.settings().fallbacks.append({"x": ["y"]}) # the snapshot holds the router's own objects
|
||||
assert router._core_fallbacks[-1] == {"x": ["y"]}
|
||||
unrelated = router.settings()
|
||||
router._core_model_list = list(router._core_model_list)
|
||||
assert router.settings() is unrelated
|
||||
|
||||
|
||||
def test_deleting_a_setting_shows_as_unset(router):
|
||||
router.apply(ChangeSet(assign={"cooldown_time": 3}))
|
||||
assert router.settings().is_set("cooldown_time")
|
||||
del router._core_cooldown_time
|
||||
assert router.settings().is_set("cooldown_time") is (hasattr(router, "_core_cooldown_time"))
|
||||
|
||||
|
||||
def test_settings_snapshot_is_frozen(router):
|
||||
snapshot = router.settings()
|
||||
assert isinstance(snapshot, RouterSettings)
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
snapshot.cooldown_time = 1 # type: ignore[misc]
|
||||
assert set(SETTING_FIELDS) <= set(RouterSettings.__dataclass_fields__)
|
||||
|
||||
|
||||
def test_unset_setting_raises_attribute_error_through_old_name():
|
||||
r = _router()
|
||||
try:
|
||||
del r.__dict__["_core_kubernetes_pod_discovery"]
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert getattr(r, "kubernetes_pod_discovery", "missing") == "missing"
|
||||
assert r.settings().is_set("kubernetes_pod_discovery") is False
|
||||
finally:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
r.retire()
|
||||
|
||||
|
||||
def test_unset_attribute_error_names_the_public_attribute(router):
|
||||
# Reading (or deleting) an old attribute the router never set raises what reading it raised before.
|
||||
router.__dict__.pop("_core_slack_alerting_logger", None)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
with pytest.raises(AttributeError) as read:
|
||||
_ = router.slack_alerting_logger
|
||||
with pytest.raises(AttributeError) as deleted:
|
||||
del router.slack_alerting_logger
|
||||
del router.__dict__["_core_kubernetes_pod_discovery"]
|
||||
with pytest.raises(AttributeError) as setting:
|
||||
_ = router.kubernetes_pod_discovery
|
||||
for caught, name in ((read, "slack_alerting_logger"), (setting, "kubernetes_pod_discovery")):
|
||||
assert str(caught.value) == f"'Router' object has no attribute '{name}'"
|
||||
assert caught.value.name == name
|
||||
assert caught.value.obj is router
|
||||
# A failed del sets no name/obj in Python either.
|
||||
assert str(deleted.value) == "'Router' object has no attribute 'slack_alerting_logger'"
|
||||
|
||||
|
||||
def test_bind_to_request_logging_none_raises_as_before(router):
|
||||
# The old member bound unconditionally, so None fails on the first callback registration.
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
with pytest.raises(AttributeError, match="add_dynamic_function_callbacks"):
|
||||
router.bind_to_request_logging(None)
|
||||
|
||||
|
||||
def test_select_picks_like_async_get_available_deployment(router):
|
||||
picked = asyncio.run(router.select("gpt", {}))
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
old = asyncio.run(router.async_get_available_deployment(model="gpt", request_kwargs={}))
|
||||
assert picked["model_name"] == old["model_name"] == "gpt"
|
||||
healthy = asyncio.run(router.select("gpt", {}, kind="healthy"))
|
||||
assert {d["model_info"]["id"] for d in healthy} == {"dep-a", "dep-b"}
|
||||
|
||||
|
||||
def test_counter_and_internal_silence(router):
|
||||
before = DEPRECATED_ROUTER_MEMBER_USES["get_model_ids"]
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
router.get_model_ids()
|
||||
# Router internals call the core names; a completion must not warn.
|
||||
asyncio.run(router.acompletion(model="gpt", messages=[{"role": "user", "content": "hi"}], mock_response="ok"))
|
||||
assert DEPRECATED_ROUTER_MEMBER_USES["get_model_ids"] == before + 1
|
||||
deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning) and "Router." in str(w.message)]
|
||||
assert [str(w.message).split(" ")[0] for w in deprecations] == ["Router.get_model_ids"]
|
||||
assert deprecations[0].filename == __file__
|
||||
|
||||
|
||||
def test_every_deprecated_member_names_a_replacement():
|
||||
import inspect
|
||||
|
||||
from litellm.router_deprecated import _DeprecatedAttribute
|
||||
|
||||
for name, member in vars(DeprecatedRouterMembers).items():
|
||||
if name.startswith("__"):
|
||||
continue
|
||||
if isinstance(member, _DeprecatedAttribute):
|
||||
assert member.replacement
|
||||
continue
|
||||
func = member.__func__ if isinstance(member, staticmethod) else member
|
||||
if callable(func):
|
||||
assert "_note(" in inspect.getsource(func), name
|
||||
|
||||
|
||||
def test_generate_model_id_is_the_module_function():
|
||||
from litellm.router_api import deployment_id
|
||||
|
||||
params = {"model": "openai/gpt-4o", "api_key": "k", "extra": {"a": 1}}
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert Router.generate_model_id("g", params) == deployment_id("g", params)
|
||||
|
||||
|
||||
def test_subclass_override_of_an_old_member_is_still_called():
|
||||
calls = []
|
||||
|
||||
class Sub(litellm.Router):
|
||||
async def async_pre_routing_hook(
|
||||
self, model, request_kwargs, messages=None, input=None, specific_deployment=False
|
||||
):
|
||||
calls.append(model)
|
||||
return await super().async_pre_routing_hook(model, request_kwargs, messages, input, specific_deployment)
|
||||
|
||||
r = Sub(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "m",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x", "mock_response": "ok"},
|
||||
}
|
||||
]
|
||||
)
|
||||
response = asyncio.run(r.acompletion(model="m", messages=[{"role": "user", "content": "hi"}]))
|
||||
assert response.choices[0].message.content == "ok"
|
||||
assert calls == ["m"]
|
||||
|
||||
|
||||
def test_direct_call_of_a_subclass_override_runs_it_once():
|
||||
calls = []
|
||||
|
||||
class Sub(litellm.Router):
|
||||
def get_model_access_groups(self, model_name=None, model_access_group=None, team_id=None):
|
||||
calls.append(model_name)
|
||||
return super().get_model_access_groups(model_name, model_access_group, team_id)
|
||||
|
||||
r = Sub(model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}}])
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert r.get_model_access_groups("m") == {}
|
||||
assert calls == ["m"]
|
||||
assert asyncio.run(asyncio.to_thread(r.registry().access_groups, model_name="m")) == {}
|
||||
assert calls == ["m", "m"]
|
||||
|
||||
|
||||
def test_subclass_staticmethod_override_is_still_consulted():
|
||||
calls = []
|
||||
|
||||
class Sub(litellm.Router):
|
||||
@staticmethod
|
||||
def is_deployment_blocked(deployment):
|
||||
calls.append(deployment.model_info.id)
|
||||
return True
|
||||
|
||||
r = Sub(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "m",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"},
|
||||
"model_info": {"id": "d1"},
|
||||
}
|
||||
]
|
||||
)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert r.get_deployment_credentials(model_id="d1") is None
|
||||
assert calls == ["d1"]
|
||||
|
||||
|
||||
def test_private_alias_skips_a_subclass_override_of_the_public_name():
|
||||
calls = []
|
||||
|
||||
class Sub(litellm.Router):
|
||||
def get_all_deployments(self, *args, **kwargs):
|
||||
calls.append("get_all_deployments")
|
||||
return super().get_all_deployments(*args, **kwargs)
|
||||
|
||||
r = Sub(model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}}])
|
||||
calls.clear()
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert len(r._get_all_deployments(model_name="m")) == 1
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_old_name_writes_and_deletes_warn(router):
|
||||
with pytest.warns(DeprecationWarning, match=r"^Router\.fallbacks is deprecated"):
|
||||
router.fallbacks = []
|
||||
with pytest.warns(DeprecationWarning, match=r"^Router\.fallbacks is deprecated"):
|
||||
del router.fallbacks
|
||||
|
||||
|
||||
def test_alias_attribute_writes_the_same_storage(router):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
replacement = {}
|
||||
router._routing_groups = replacement
|
||||
assert router.routing_groups is replacement
|
||||
assert router.registry().routing_groups() is replacement
|
||||
|
||||
|
||||
def test_core_members_keep_their_old_names_in_errors():
|
||||
from litellm.router import _name_core_members_after_old_names
|
||||
|
||||
_name_core_members_after_old_names()
|
||||
|
||||
assert Router._core_get_deployment_model_info.__qualname__ == "Router.get_deployment_model_info"
|
||||
assert Router._core_get_deployment.__qualname__ == "Router.get_deployment"
|
||||
|
||||
|
||||
def test_wrong_arity_errors_still_name_router(router):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
with pytest.raises(TypeError, match=r"^Router\.get_deployment\(\) takes 2 positional arguments"):
|
||||
router.get_deployment("dep-a", "extra") # pyright: ignore[reportCallIssue]
|
||||
with pytest.raises(TypeError, match=r"^Router\.get_available_guardrail\(\) missing 1 required"):
|
||||
router.get_available_guardrail() # pyright: ignore[reportCallIssue]
|
||||
with pytest.raises(TypeError, match=r"^litellm\.router\.Router\.has_model_id\(\) argument after \*\*"):
|
||||
router.has_model_id(**None) # pyright: ignore[reportCallIssue, reportArgumentType]
|
||||
with pytest.raises(TypeError, match=r"^Router\.is_deployment_blocked\(\) missing 1 required"):
|
||||
router.is_deployment_blocked() # pyright: ignore[reportCallIssue]
|
||||
with pytest.raises(TypeError, match=r"^Router\.get_deployment_model_info\(\) missing 2 required"):
|
||||
router.cached_deployment_model_info() # pyright: ignore[reportCallIssue]
|
||||
|
||||
|
||||
_SENTINEL = object()
|
||||
|
||||
# (entry point call, Router core member it forwards to, the arguments the core member must get)
|
||||
FORWARDS = [
|
||||
(
|
||||
lambda r: r.registry().deployment_model_info("dep-a", "gpt", cached=True),
|
||||
"_core_cached_deployment_model_info",
|
||||
(("dep-a", "gpt"), {}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().deployment_model_info("dep-a", "gpt"),
|
||||
"_core_get_deployment_model_info",
|
||||
((), {"model_id": "dep-a", "model_name": "gpt"}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().router_model_info(None, "gpt", id="dep-a"),
|
||||
"_core_get_router_model_info",
|
||||
((), {"deployment": None, "received_model_name": "gpt", "id": "dep-a"}),
|
||||
),
|
||||
(lambda r: r.registry().discovered_model_info("dep-a"), "_core_get_discovered_model_info", (("dep-a",), {})),
|
||||
(
|
||||
lambda r: r.registry().deployments("gpt", source="routing_groups"),
|
||||
"_core_get_model_list_from_routing_groups",
|
||||
((), {"model_name": "gpt"}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().deployments("gpt", source="routed_group"),
|
||||
"_core_get_model_list_of_routed_group",
|
||||
(("gpt",), {}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().deployments("gpt", source="wildcard", team_id="t"),
|
||||
"_core_wildcard_route_deployments",
|
||||
(("gpt", "t"), {}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().deployments("gpt", source="routable", request={"a": 1}, call_type="aembedding"),
|
||||
"_core_get_routable_deployments",
|
||||
(("gpt", {"a": 1}, "aembedding"), {}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().deployments("gpt", source="for_request"),
|
||||
"_core_deployments_for_request",
|
||||
(("gpt", {}), {}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().includes({"model_name": "gpt"}, model_name="gpt", team_id="t"),
|
||||
"_core_should_include_deployment",
|
||||
((), {"model_name": "gpt", "model": {"model_name": "gpt"}, "team_id": "t"}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().names("gpt", kind="litellm_models", team_id="t"),
|
||||
"_core_resolved_litellm_models",
|
||||
(("gpt",), {"team_id": "t"}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().names("gpt", kind="litellm_base_models"),
|
||||
"_core_resolved_litellm_models",
|
||||
(("gpt",), {"team_id": None, "use_base_model": True}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().names("gpt", kind="wildcard", team_id="t"),
|
||||
"_core_wildcard_routed_models",
|
||||
(("gpt", "t"), {}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().names("gpt", kind="route_candidates"),
|
||||
"_core_get_candidate_model_ids_for_route",
|
||||
(("gpt",), {"team_id": None}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().resolve("dep-a", kind="from_id", custom_llm_provider="openai"),
|
||||
"_core_resolve_model_name_from_model_id",
|
||||
(("dep-a",), {"custom_llm_provider": "openai"}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().resolve("rg", kind="routing_group_alternatives"),
|
||||
"_core_routing_group_has_alternatives",
|
||||
(("rg",), {}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().resolve("gpt", kind="served", served_model="gpt-4o"),
|
||||
"_core_resolve_served_model_name",
|
||||
(("gpt", "gpt-4o"), {}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().resolve("gpt", kind="upstream", team_id="t"),
|
||||
"_core_get_routable_upstream_model",
|
||||
(("gpt",), {"team_id": "t"}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().resolve("gpt", kind="search_tool_support", team_id="t"),
|
||||
"_core_get_search_tool_support",
|
||||
(("gpt",), {"team_id": "t"}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().resolve("grp", kind="wildcard_access_group"),
|
||||
"_core_is_model_access_group_for_wildcard_route",
|
||||
((), {"model_access_group": "grp"}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().resolve("dep-a", kind="team_model_alternatives"),
|
||||
"_core_team_model_has_alternatives",
|
||||
(("dep-a",), {}),
|
||||
),
|
||||
(
|
||||
lambda r: r.registry().group("gpt", include_hidden=True, team_id="t"),
|
||||
"_core_get_model_group_info",
|
||||
(("gpt",), {"include_hidden": True, "team_id": "t"}),
|
||||
),
|
||||
(lambda r: r.registry().group("gpt", field="cached_info"), "_core_cached_model_group_info", (("gpt",), {})),
|
||||
(lambda r: r.registry().group("gpt", field="display_name"), "_core_get_configured_display_name", (("gpt",), {})),
|
||||
(
|
||||
lambda r: r.registry().group("gpt", field="service_tiers", team_id="t"),
|
||||
"_core_get_configured_service_tiers",
|
||||
(("gpt",), {"team_id": "t"}),
|
||||
),
|
||||
(lambda r: r.registry().group("gpt", field="listing_info"), "_core_get_model_listing_info", (("gpt",), {})),
|
||||
(lambda r: r.registry().plugins("budget_limiter"), "_core_get_router_deployment_budget_limiter", ((), {})),
|
||||
(lambda r: r.settings().allowed_fails_for(ValueError("x")), "_core_get_allowed_fails_from_policy", None),
|
||||
(lambda r: r.settings().guardrail("g"), "_core_get_available_guardrail", (("g",), {})),
|
||||
(lambda r: r.health().team_ptu_ceiling("t", "gpt"), "_core_team_ptu_ceiling", (("t", "gpt"), {})),
|
||||
(
|
||||
lambda r: r.health().split_priority("gpt", {"priority": 1}),
|
||||
"_core_split_request_priority",
|
||||
(("gpt", {"priority": 1}), {}),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("call, core, expected", FORWARDS, ids=[f[1] for f in FORWARDS])
|
||||
def test_entry_point_forwards_to_its_core_member(router, monkeypatch, call, core, expected):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
mock = MagicMock(return_value=_SENTINEL)
|
||||
monkeypatch.setattr(router, core, mock)
|
||||
assert call(router) is _SENTINEL
|
||||
if expected is not None:
|
||||
assert mock.call_args == expected
|
||||
else:
|
||||
mock.assert_called_once()
|
||||
|
||||
|
||||
ASYNC_FORWARDS = [
|
||||
(lambda r: r.health().fully_unhealthy_model_names(), "_core_async_get_fully_unhealthy_model_names", ((), {})),
|
||||
(lambda r: r.health().usage("gpt"), "_core_get_model_group_usage", ((), {"model_group": "gpt"})),
|
||||
(
|
||||
lambda r: r.health().usage("gpt", kind="remaining"),
|
||||
"_core_get_remaining_model_group_usage",
|
||||
((), {"model_group": "gpt"}),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("call, core, expected", ASYNC_FORWARDS, ids=[f[1] for f in ASYNC_FORWARDS])
|
||||
def test_async_entry_point_forwards_to_its_core_member(router, monkeypatch, call, core, expected):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
mock = AsyncMock(return_value=_SENTINEL)
|
||||
monkeypatch.setattr(router, core, mock)
|
||||
assert asyncio.run(call(router)) is _SENTINEL
|
||||
assert mock.call_args == expected
|
||||
|
||||
|
||||
def test_registry_reads_without_a_core_forward(router):
|
||||
registry = router.registry()
|
||||
assert registry.zero_cost() is getattr(router, "_zero_cost_cache", None)
|
||||
assert registry.includes(router.registry().deployment("dep-a").model_dump(), team_id=None) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call, error",
|
||||
[
|
||||
(lambda r: r.registry().deployments(source="named"), TypeError),
|
||||
(lambda r: r.registry().deployments("gpt", source="nope"), ValueError),
|
||||
(lambda r: r.registry().index("nope"), ValueError),
|
||||
(lambda r: r.registry().names(kind="litellm_models"), TypeError),
|
||||
(lambda r: r.registry().names("gpt", kind="nope"), ValueError),
|
||||
(lambda r: r.registry().resolve("team-gpt", kind="team_model"), TypeError),
|
||||
(lambda r: r.registry().resolve(None, kind="alias"), TypeError),
|
||||
(lambda r: r.registry().resolve("gpt", kind="nope"), ValueError),
|
||||
(lambda r: r.registry().group("gpt", field="nope"), ValueError),
|
||||
(lambda r: r.registry().credentials("dep-a", form="nope"), ValueError),
|
||||
(lambda r: r.registry().plugins("nope"), ValueError),
|
||||
(lambda r: asyncio.run(r.health().usage("gpt", kind="nope")), ValueError),
|
||||
(lambda r: RouterSettings().as_dict(), AttributeError),
|
||||
],
|
||||
)
|
||||
def test_entry_points_reject_unknown_kinds_and_missing_names(router, call, error):
|
||||
with pytest.raises(error):
|
||||
call(router)
|
||||
|
||||
|
||||
def test_health_is_blocked_and_the_old_static_name_agree():
|
||||
paused = Deployment(
|
||||
model_name="m",
|
||||
litellm_params={"model": "openai/gpt-4o-mini"}, # pyright: ignore[reportArgumentType]
|
||||
model_info={"id": "d1", "blocked": True}, # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
from litellm.router_api import RouterHealth
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
assert Router.is_deployment_blocked(paused) is RouterHealth.is_blocked(paused)
|
||||
|
||||
|
||||
def test_old_members_refuse_objects_that_are_not_routers():
|
||||
from litellm.router_deprecated import _as_router
|
||||
|
||||
with pytest.raises(TypeError, match=r"is not a litellm\.Router"):
|
||||
_as_router(object()) # pyright: ignore[reportArgumentType]
|
||||
|
||||
|
||||
def test_direct_call_of_an_async_subclass_override_runs_it_once():
|
||||
calls = []
|
||||
|
||||
class Sub(litellm.Router):
|
||||
async def async_pre_routing_hook(
|
||||
self, model, request_kwargs, messages=None, input=None, specific_deployment=False
|
||||
):
|
||||
calls.append(model)
|
||||
return await super().async_pre_routing_hook(model, request_kwargs, messages, input, specific_deployment)
|
||||
|
||||
r = Sub(model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}}])
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
asyncio.run(r.async_pre_routing_hook("m", {}))
|
||||
assert calls == ["m"]
|
||||
|
||||
|
||||
def test_routing_context_forwards_to_the_router(router, monkeypatch):
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from litellm.router_api import RoutingContext
|
||||
|
||||
ctx = RoutingContext(router)
|
||||
assert ctx.registry().version == router.registry().version
|
||||
assert ctx.settings() is router.settings()
|
||||
assert isinstance(ctx.health(), type(router.health()))
|
||||
for name in (
|
||||
"_pre_call_checks",
|
||||
"_count_pre_call_check_tokens",
|
||||
"_update_kwargs_before_fallbacks",
|
||||
"_core_arm_routing_read_prefetch",
|
||||
"_core_log_retry",
|
||||
):
|
||||
monkeypatch.setattr(router, name, MagicMock(return_value=_SENTINEL))
|
||||
monkeypatch.setattr(router, "_core_async_pre_routing_hook", AsyncMock(return_value=_SENTINEL))
|
||||
assert ctx.pre_call_checks("gpt", []) is _SENTINEL
|
||||
assert ctx.count_pre_call_check_tokens(None, "hi") is _SENTINEL
|
||||
assert ctx.update_kwargs_before_fallbacks("gpt", {}) is _SENTINEL
|
||||
assert ctx.arm_read_prefetch("gpt") is _SENTINEL
|
||||
assert asyncio.run(ctx.pre_routing_hook("gpt", {})) is _SENTINEL
|
||||
assert ctx.log_retry({}, ValueError("x")) is _SENTINEL
|
||||
|
||||
|
||||
def test_attribute_errors_about_other_objects_pass_through_unchanged(router):
|
||||
from litellm.router_deprecated import _public_attribute_error
|
||||
|
||||
other = AttributeError("'dict' object has no attribute 'x'", name="x", obj={})
|
||||
assert _public_attribute_error(router, "_core_x", "x", other) is other
|
||||
|
|
@ -1158,7 +1158,7 @@ async def test_set_response_headers_passes_through_post_increment_counters(model
|
|||
_hidden_params: dict = {}
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-remaining-tokens": 958,
|
||||
"x-ratelimit-limit-tokens": 1000,
|
||||
|
|
@ -1539,7 +1539,7 @@ async def test_get_remaining_model_group_usage_merges_io_and_tpm_headers(model_l
|
|||
)
|
||||
)
|
||||
router.get_model_group_io_token_usage = AsyncMock(return_value=(100, 40))
|
||||
router.get_model_group_usage = AsyncMock(return_value=(500, 5))
|
||||
router._core_get_model_group_usage = AsyncMock(return_value=(500, 5))
|
||||
|
||||
headers = await router.get_remaining_model_group_usage("gpt-3.5-turbo")
|
||||
|
||||
|
|
@ -1566,7 +1566,7 @@ async def test_set_response_headers_native_input_token_header_does_not_suppress_
|
|||
_hidden_params: dict = {}
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-remaining-tokens": 1000,
|
||||
"x-ratelimit-remaining-requests": 100,
|
||||
|
|
@ -1596,7 +1596,7 @@ async def test_set_response_headers_native_token_header_does_not_suppress_io_hea
|
|||
_hidden_params: dict = {}
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-remaining-tokens": 1000,
|
||||
"x-ratelimit-remaining-requests": 100,
|
||||
|
|
@ -1628,7 +1628,7 @@ async def test_set_response_headers_handles_missing_usage(model_list):
|
|||
_hidden_params: dict = {}
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-remaining-tokens": 1000,
|
||||
"x-ratelimit-remaining-requests": 100,
|
||||
|
|
@ -1648,7 +1648,7 @@ async def test_set_response_headers_handles_missing_usage(model_list):
|
|||
async def test_set_response_headers_dict_anthropic_messages_response(model_list):
|
||||
"""Anthropic /v1/messages returns a dict; IO rate-limit headers must attach."""
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-limit-input-tokens": 25,
|
||||
"x-ratelimit-remaining-input-tokens": 20,
|
||||
|
|
@ -1688,7 +1688,7 @@ async def test_set_response_headers_wraps_bare_async_generator(model_list):
|
|||
yield {"type": "message_stop"}
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router.get_remaining_model_group_usage = AsyncMock(
|
||||
router._core_get_remaining_model_group_usage = AsyncMock(
|
||||
return_value={
|
||||
"x-ratelimit-limit-input-tokens": 25,
|
||||
"x-ratelimit-remaining-input-tokens": 20,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue