From 4b1ab19d1b0b50d6839f9081f7e4c63f04173e5e Mon Sep 17 00:00:00 2001 From: narmstrong-berri Date: Thu, 29 Oct 2026 12:52:24 -0700 Subject: [PATCH 1/2] refactor(router): move litellm/router.py to litellm/router_python_core.py A pure rename with no content change, so git and review tools track the router body as a move. The next commit turns it into PythonCore and adds the Router facade back at litellm/router.py --- litellm/{router.py => router_python_core.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename litellm/{router.py => router_python_core.py} (100%) diff --git a/litellm/router.py b/litellm/router_python_core.py similarity index 100% rename from litellm/router.py rename to litellm/router_python_core.py From 254aae8cf4c59d46b8f17ecf815d3888df8db122 Mon Sep 17 00:00:00 2001 From: narmstrong-berri Date: Thu, 29 Oct 2026 12:52:24 -0700 Subject: [PATCH 2/2] refactor(router): extract PythonCore behind RouterCore and pass the router its config The routing logic moves unchanged from litellm/router.py into PythonCore in litellm/router_python_core.py. Router becomes a facade that builds a RouterConfig, picks a RouterCore with make_core and delegates every member to it. The core no longer reads or writes litellm.* directly: process settings come from RouterDynamicConfig, and inference calls, sleeps, callback lists and the global cache go through PythonHost. A CI check bans litellm.* in the router modules, and the proxy's construction sites build the config outside the router Settings are static: the constructor arguments and changeset values are copied when the router is built or the change applied, and a write to a setting attribute becomes apply(ChangeSet(assign=...)). A Router subclass's core class also subclasses the facade, so an override's super(Sub, self) works, and an instance-level patch of a core method is kept on the facade so nested patches unwind. tests/unit/test_router/core tests the core in isolation: ScriptedHost serves every host operation (scripted inference answers, recorded sleeps, settings, callbacks, cache), and make_core runs each test once per core in CORE_BUILDERS --- .../anthropic/messages/adapters/handler.py | 3 +- litellm/proxy/proxy_server.py | 64 +- litellm/proxy/route_llm_request.py | 6 +- litellm/router.py | 330 +++++++ litellm/router_api.py | 44 +- litellm/router_config.py | 157 ++++ litellm/router_core.py | 56 ++ litellm/router_deprecated.py | 30 +- litellm/router_python_core.py | 788 +++++++++-------- litellm/router_python_host.py | 73 ++ .../code_coverage_tests/recursive_detector.py | 1 + .../router_litellm_globals_check.py | 54 ++ .../router_surface_check.py | 8 +- .../test_router_routing_groups.py | 8 +- .../test_router_tag_routing.py | 4 +- .../router_utils/test_cooldown_handlers.py | 4 +- .../test_kubernetes_pod_discovery.py | 3 +- tests/unit/test_conftest_isolation.py | 6 +- tests/unit/test_router/core/__init__.py | 0 tests/unit/test_router/core/conftest.py | 26 + tests/unit/test_router/core/scripted_host.py | 487 ++++++++++ tests/unit/test_router/core/test_endpoints.py | 445 ++++++++++ tests/unit/test_router/core/test_fallbacks.py | 756 ++++++++++++++++ .../core/test_registry_changesets.py | 657 ++++++++++++++ .../core/test_retries_cooldowns.py | 832 ++++++++++++++++++ tests/unit/test_router/core/test_selection.py | 624 +++++++++++++ tests/unit/test_router/test_router.py | 24 +- .../test_router/test_router_entry_points.py | 6 +- tests/unit/test_router/test_router_facade.py | 52 ++ .../test_router/test_router_helper_utils.py | 8 +- .../test_router_provider_endpoints.py | 2 +- tests/unit/test_router_silent_experiment.py | 4 +- tests/unit/test_router_weighted_failover.py | 12 +- 33 files changed, 5116 insertions(+), 458 deletions(-) create mode 100644 litellm/router.py create mode 100644 litellm/router_config.py create mode 100644 litellm/router_core.py create mode 100644 litellm/router_python_host.py create mode 100644 tests/code_coverage_tests/router_litellm_globals_check.py create mode 100644 tests/unit/test_router/core/__init__.py create mode 100644 tests/unit/test_router/core/conftest.py create mode 100644 tests/unit/test_router/core/scripted_host.py create mode 100644 tests/unit/test_router/core/test_endpoints.py create mode 100644 tests/unit/test_router/core/test_fallbacks.py create mode 100644 tests/unit/test_router/core/test_registry_changesets.py create mode 100644 tests/unit/test_router/core/test_retries_cooldowns.py create mode 100644 tests/unit/test_router/core/test_selection.py create mode 100644 tests/unit/test_router/test_router_facade.py diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index b3b84f721c0..d13fd4f47ff 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -118,9 +118,10 @@ def _extract_proxy_litellm_metadata( def _requested_router(kwargs: Mapping[str, object]) -> "Router | None": from litellm.router import Router + from litellm.router_python_core import PythonCore requested: Final = kwargs.get("litellm_router") - return requested if isinstance(requested, Router) else None + return requested if isinstance(requested, (Router, PythonCore)) else None # pyright: ignore[reportReturnType] # the router passes its core, which carries Router's members def request_safeguards_evaluator( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1f1ab52e70d..ed95399f454 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -844,6 +844,8 @@ from litellm.proxy.telemetry.middleware import TelemetryMiddleware from litellm.proxy.telemetry.runtime import TelemetryRuntime from litellm.proxy.telemetry.settings import describe_errors as describe_telemetry_errors from litellm.proxy.telemetry.settings import load_settings as load_telemetry_settings +from litellm.router import router_arguments +from litellm.router_config import build_router_config try: from litellm.proxy.enterprise_billing.billing_metrics import ( @@ -7374,20 +7376,24 @@ class ProxyConfig: verbose_proxy_logger.warning( "Key '%s' is not a valid argument for Router.__init__(). Ignoring this key.", k ) - router = litellm.Router( - **router_params, - assistants_config=assistants_config, - search_tools=search_tools, - router_general_settings=RouterGeneralSettings( - async_only_mode=True # only init async clients - ), - ignore_invalid_deployments=True, # don't raise an error if a deployment is invalid - fallback_access_check=router_fallback_access_check, - fallback_budget_check=RouterFallbackBudgetCheck( - is_enforced=router_fallback_budget_check.is_enforced, - team_member_model_budget_limiter=model_max_budget_limiter, - ), - auto_router_capability_limit=_license_check.auto_router_capability_limit, + router = litellm.Router.from_config( + build_router_config( + router_arguments( + **router_params, + assistants_config=assistants_config, + search_tools=search_tools, + router_general_settings=RouterGeneralSettings( + async_only_mode=True # only init async clients + ), + ignore_invalid_deployments=True, # don't raise an error if a deployment is invalid + fallback_access_check=router_fallback_access_check, + fallback_budget_check=RouterFallbackBudgetCheck( + is_enforced=router_fallback_budget_check.is_enforced, + team_member_model_budget_limiter=model_max_budget_limiter, + ), + auto_router_capability_limit=_license_check.auto_router_capability_limit, + ) + ) ) if redis_usage_cache is not None and router.health().store.redis_cache is None: @@ -7877,19 +7883,23 @@ class ProxyConfig: # Router can function with model_list=[] if search_tools are configured if len(_model_list) > 0 or search_tools: verbose_proxy_logger.debug("_model_list: %s", _model_list) - llm_router = litellm.Router( - model_list=_model_list, - router_general_settings=RouterGeneralSettings( - async_only_mode=True # only init async clients - ), - search_tools=search_tools, - ignore_invalid_deployments=True, - fallback_access_check=router_fallback_access_check, - fallback_budget_check=RouterFallbackBudgetCheck( - is_enforced=router_fallback_budget_check.is_enforced, - team_member_model_budget_limiter=model_max_budget_limiter, - ), - auto_router_capability_limit=_license_check.auto_router_capability_limit, + llm_router = litellm.Router.from_config( + build_router_config( + router_arguments( + model_list=_model_list, + router_general_settings=RouterGeneralSettings( + async_only_mode=True # only init async clients + ), + search_tools=search_tools, + ignore_invalid_deployments=True, + fallback_access_check=router_fallback_access_check, + fallback_budget_check=RouterFallbackBudgetCheck( + is_enforced=router_fallback_budget_check.is_enforced, + team_member_model_budget_limiter=model_max_budget_limiter, + ), + auto_router_capability_limit=_license_check.auto_router_capability_limit, + ) + ) ) verbose_proxy_logger.debug("updated llm_router: %s", llm_router) else: diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index a17b7fee8c9..ecc31f3b0a7 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -18,7 +18,9 @@ from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.router import router_arguments from litellm.router_api import router_init_args +from litellm.router_config import build_router_config # Client-supplied params that make the router or the call path fabricate a # failure or a delay instead of calling the provider. The ``mock_testing_*`` @@ -98,7 +100,9 @@ async def _route_user_config_request(data: dict, route_type: str): filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args} logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type) - cached_router: Final = user_config_routers.get(filtered_config, lambda: litellm.Router(**filtered_config)) + cached_router: Final = user_config_routers.get( + filtered_config, lambda: litellm.Router.from_config(build_router_config(router_arguments(**filtered_config))) + ) cached_router.observe(logging_obj) return await getattr(cached_router, f"{route_type}")(**{**data, "litellm_logging_obj": logging_obj}) diff --git a/litellm/router.py b/litellm/router.py new file mode 100644 index 00000000000..4ee138fc9fb --- /dev/null +++ b/litellm/router.py @@ -0,0 +1,330 @@ +""" +litellm.Router: load balancing, retries, fallbacks and cooldowns across deployments. + +``Router`` holds no routing logic. It builds its configuration (``litellm/router_config.py``), picks +one ``RouterCore`` at construction (``litellm/router_core.py``) and delegates every member to it: +the endpoint methods (acompletion, aembedding, aresponses, aanthropic_messages, ...) and the entry +points (registry, settings, health, select, apply, observe, register_plugin, retire). The members +outside code used before the entry points live in ``DeprecatedRouterMembers`` until they are removed. +""" + +import inspect +import weakref +from collections.abc import Mapping +from types import CellType, FunctionType, MappingProxyType +from typing import TYPE_CHECKING, Final + +from litellm import router_python_core +from litellm.router_config import RouterConfig, build_router_config +from litellm.router_core import RouterCore, make_core +from litellm.router_deprecated import DeprecatedRouterMembers +from litellm.router_python_core import ( + PythonCore, + as_output_cap, + model_info_is_active_for_environment, + stamped_client_ceiling, +) +from litellm.router_python_host import PythonHost, python_host +from litellm.types.router import ( + AllowedFailsPolicy, + AssistantsTypedDict, + CustomRoutingStrategyBase, + Deployment, + LiteLLM_Params, + ModelGroupInfo, + RetryPolicy, +) + +_CORE_SIGNATURE: Final = inspect.signature(PythonCore.__init__) + + +def router_arguments( + *args: object, + **kwargs: object, # kwargs-ok: Router's constructor arguments, as passed +) -> Mapping[str, object]: + """The constructor arguments as passed, by name (positional ones bound to their parameter).""" + try: + bound: Final = _CORE_SIGNATURE.bind_partial(None, *args, **kwargs).arguments + except TypeError as error: + raise TypeError(f"Router.__init__() {error}") from None + return MappingProxyType({name: value for name, value in bound.items() if name != "self"}) + + +def _class_descriptor(cls: type, name: str) -> bool: + return any(hasattr(vars(base).get(name), "__set__") for base in cls.__mro__ if name in vars(base)) + + +if TYPE_CHECKING: + + class Router(PythonCore, DeprecatedRouterMembers): + @classmethod + def from_config(cls, config: RouterConfig, host: PythonHost = python_host) -> "Router": ... + +else: + _MISSING: Final = object() + _replaced_core_members: Final[ # mutable-ok: originals of patched core members, restored on delete + dict[tuple[type, str], list[object]] + ] = {} + _FACADE_ONLY: Final = frozenset( + { + "__module__", + "__qualname__", + "__doc__", + "__dict__", + "__weakref__", + "__init__", + "__init_subclass__", + "__annotations__", + "__firstlineno__", + "__static_attributes__", + "__classcell__", + "_core_class", + } + ) + + def _with_class_cell(value: object, cell: "CellType") -> object: + """``value`` with its ``__class__`` cell (what zero-argument ``super()`` reads) replaced by ``cell``.""" + if isinstance(value, staticmethod | classmethod): + return type(value)(_function_with_class_cell(value.__func__, cell)) + if isinstance(value, property): + return property( + *( + None if f is None else _function_with_class_cell(f, cell) + for f in (value.fget, value.fset, value.fdel) + ), + value.__doc__, + ) + return _function_with_class_cell(value, cell) + + def _function_with_class_cell(value: object, cell: "CellType") -> object: + """A plain function with its ``__class__`` cell replaced by ``cell``; anything else unchanged.""" + if not isinstance(value, FunctionType) or "__class__" not in value.__code__.co_freevars: + return value + closure: Final = tuple( + cell if name == "__class__" else original + for name, original in zip(value.__code__.co_freevars, value.__closure__ or (), strict=True) + ) + rebound: Final = FunctionType(value.__code__, value.__globals__, value.__name__, value.__defaults__, closure) + rebound.__kwdefaults__ = value.__kwdefaults__ + rebound.__qualname__ = value.__qualname__ + rebound.__dict__.update(value.__dict__) + return rebound + + class _Absent: + """A facade-only member, hidden from the core class: reading it raises the usual AttributeError.""" + + def __set_name__(self, owner: type, name: str) -> None: + self._name = name + + def __get__(self, instance: object, owner: type | None = None) -> object: + if instance is None: + raise AttributeError( + f"type object {getattr(owner, '__name__', owner)!r} has no attribute {self._name!r}" + ) + raise AttributeError(f"{type(instance).__name__!r} object has no attribute {self._name!r}") + + def _core_class_for(facade: type, parent: type) -> type: + """ + A core class carrying a Router subclass's own members, so the core runs its overrides. It + subclasses the facade too, so an override's ``super(Sub, self)`` finds the core it runs on. + """ + cell: Final = CellType() + members: Final = { + name: _with_class_cell(value, cell) for name, value in vars(facade).items() if name not in _FACADE_ONLY + } + namespace: Final = { + # The facade's own plumbing sits in the core class's MRO ahead of the parent core: put + # the parent core's versions back, and hide the facade-only members. + "__init__": parent.__init__, + "__getattribute__": object.__getattribute__, + "__setattr__": parent.__setattr__, + "__delattr__": parent.__delattr__, + "__dir__": parent.__dir__, + **{name: _Absent() for name in ("_attach", "_core", "from_config", "_core_class")}, + **members, + "__module__": facade.__module__, + } + core_class: Final = _CoreType(f"{facade.__name__}Core", (facade, parent), namespace) + cell.cell_contents = core_class + for name in members: + type.__delattr__(facade, name) + return core_class + + class _RouterType(type): + """ + Class-level reads of core members (``Router._helper(...)``) go to the router's core class, and + so do class-level patches, which put the original back when the patch is removed. + ``isinstance`` and ``dir`` see the core class too. + """ + + def __getattr__(cls, name: str) -> object: + if name == "_core_class": + raise AttributeError(name) + return getattr(cls._core_class, name) + + def __setattr__(cls, name: str, value: object) -> None: + core_class: Final = vars(cls).get("_core_class") or getattr(cls, "_core_class", None) + if core_class is None or any(name in vars(base) for base in cls.__mro__) or not hasattr(core_class, name): + super().__setattr__(name, value) + return + _replaced_core_members.setdefault((core_class, name), []).append(vars(core_class).get(name, _MISSING)) + setattr(core_class, name, value) + + def __delattr__(cls, name: str) -> None: + core_class: Final = getattr(cls, "_core_class", None) + replaced: Final = _replaced_core_members.get((core_class, name)) if core_class is not None else None + if not replaced or any(name in vars(base) for base in cls.__mro__): + super().__delattr__(name) + return + original: Final = replaced.pop() + if original is _MISSING: + delattr(core_class, name) + return + setattr(core_class, name, original) + + def __instancecheck__(cls, instance: object) -> bool: + return type.__instancecheck__(cls, instance) or isinstance(instance, cls._core_class) + + def __dir__(cls) -> list[str]: # mutable-ok: dir() returns a list + return sorted({*type.__dir__(cls), *dir(cls._core_class)}) + + class _CoreType(_RouterType): + """The metaclass of a Router subclass's core class: a plain class, none of the facade's redirects.""" + + def __getattr__(cls, name: str) -> object: + raise AttributeError(f"type object {cls.__name__!r} has no attribute {name!r}") + + __setattr__ = type.__setattr__ + __delattr__ = type.__delattr__ + __instancecheck__ = type.__instancecheck__ + __dir__ = type.__dir__ + + def _is_core(instance: object) -> bool: + return isinstance(type(instance), _CoreType) + + def _patchable_method(core: object, name: str) -> bool: + """A method of the core: an instance-level replacement of it is also kept on the facade, so + ``mock.patch.object(router, name)`` finds it there and nested patches unwind in order. A core + that serves its members from another core (``__getattr__``) is asked for the bound method.""" + static: Final = inspect.getattr_static(type(core), name, None) + if static is not None: + return isinstance(static, FunctionType | staticmethod | classmethod) + return hasattr(type(core), "__getattr__") and inspect.ismethod(getattr(core, name, None)) + + class Router(DeprecatedRouterMembers, metaclass=_RouterType): + _core_class: type[PythonCore] = PythonCore + + def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: class keywords, passed on to super() + if isinstance(cls, _CoreType): + # A core class built below: the parent core's hooks wire it, as for any core class. + super().__init_subclass__(**kwargs) + return + # DeprecatedRouterMembers wires a subclass's overrides of old members; here that runs + # on the core class built from it, which is where the router's own calls land. + super(DeprecatedRouterMembers, cls).__init_subclass__(**kwargs) + type.__setattr__(cls, "_core_class", _core_class_for(cls, cls._core_class)) + + def __init__(self, *args: object, **kwargs: object) -> None: # kwargs-ok: Router's constructor arguments + self._attach( + make_core(build_router_config(router_arguments(*args, **kwargs)), python_host, type(self)._core_class) + ) + + @classmethod + def from_config(cls, config: RouterConfig, host: PythonHost = python_host) -> "Router": + router: Final = cls.__new__(cls) + router._attach(make_core(config, host, cls._core_class)) + return router + + def _attach(self, core: RouterCore) -> None: + pending: Final = self.__dict__.pop("_pending", {}) + self.__dict__["_core"] = core + core.__dict__["_core_owner"] = weakref.ref(self) # rebind-ok: the facade records itself on the core it owns + for name, value in pending.items(): + setattr(core, name, value) + + @property + def _core(self) -> RouterCore: + core: Final = self.__dict__.get("_core") + if core is None: + raise AttributeError("_core") + return core + + def __getattribute__(self, name: str) -> object: + try: + return object.__getattribute__(self, name) + except AttributeError: + if _is_core(self): + raise + facade: Final = object.__getattribute__(self, "__dict__") + core: Final = facade.get("_core") + if core is None: + pending: Final = facade.get("_pending", {}) + if name in pending: + return pending[name] + raise AttributeError(name) + try: + return getattr(core, name) + except AttributeError as error: + if error.obj is not core or error.name != name: + raise + # What reading an attribute the router lacks raised before: it names the Router. + raise AttributeError( + f"'{type(self).__name__}' object has no attribute '{name}'", name=name, obj=self + ) from None + + def __setattr__(self, name: str, value: object) -> None: + if _is_core(self): + super().__setattr__(name, value) + return + if _class_descriptor(type(self), name): + object.__setattr__(self, name, value) + return + if "_core" not in self.__dict__: + self.__dict__.setdefault("_pending", {})[name] = value + return + core: Final = self._core + mirrored: Final = name in self.__dict__ or _patchable_method(core, name) + setattr(core, name, value) + if mirrored: + self.__dict__[name] = value + + def __delattr__(self, name: str) -> None: + if _is_core(self): + super().__delattr__(name) + return + if _class_descriptor(type(self), name): + object.__delattr__(self, name) + return + self.__dict__.pop(name, None) + delattr(self._core, name) + + def __dir__(self) -> list[str]: # mutable-ok: dir() returns a list + if _is_core(self): + return super().__dir__() + return sorted({*super().__dir__(), *dir(self.__dict__.get("_core", type(self)._core_class))}) + + Router.__init__.__signature__ = _CORE_SIGNATURE.replace( # pyright: ignore[reportFunctionMemberAccess] # inspect reads it + parameters=[parameter for parameter in _CORE_SIGNATURE.parameters.values() if parameter.name != "_host"] + ) + Router.__init__.__doc__ = PythonCore.__init__.__doc__ + + +def __getattr__(name: str) -> object: + """Module-level helpers that lived in this module before the core moved out.""" + return getattr(router_python_core, name) + + +__all__ = [ + "AllowedFailsPolicy", + "AssistantsTypedDict", + "CustomRoutingStrategyBase", + "Deployment", + "LiteLLM_Params", + "ModelGroupInfo", + "RetryPolicy", + "Router", + "as_output_cap", + "model_info_is_active_for_environment", + "router_arguments", + "stamped_client_ceiling", +] diff --git a/litellm/router_api.py b/litellm/router_api.py index e4939254b81..c013a99573f 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -42,6 +42,8 @@ from dataclasses import dataclass, field from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, overload +from litellm.router_config import detached + if TYPE_CHECKING: from typing import Any # noqa: TID251 # only to spell the cores' untyped (Unknown) containers and one Any type @@ -176,9 +178,9 @@ def deployment_usable_by_team(deployment: "Mapping[str, object] | Deployment", t Whether ``team_id`` may use ``deployment``: a team-scoped deployment (``model_info.team_id`` set) only by that team, any other deployment by everyone. Needs no router instance. """ - from litellm.router import Router + from litellm.router_python_core import PythonCore - return Router._core_deployment_usable_by_team(deployment, team_id) + return PythonCore._core_deployment_usable_by_team(deployment, team_id) def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str: @@ -200,9 +202,9 @@ def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str def router_init_args() -> tuple[str, ...]: """The arguments ``Router.__init__`` accepts (was ``Router.get_valid_args``). Needs no router instance.""" - from litellm.router import Router + from litellm.router_python_core import PythonCore - return tuple(Router.get_valid_args()) + return tuple(PythonCore.get_valid_args()) def mark_advisor_orchestration_failure(exception: BaseException) -> None: @@ -266,9 +268,9 @@ def deployment_metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, ob def _json_default_stable_id(obj: object) -> str: - from litellm.router import Router + from litellm.router_python_core import PythonCore - return Router._json_default_stable_id(obj) + return PythonCore._json_default_stable_id(obj) class RouterRegistry: @@ -529,10 +531,10 @@ class RouterRegistry: ``should_include_deployment``), or, without ``model_name``, whether the team may use it at all (was ``deployment_usable_by_team``). """ - from litellm.router import Router + from litellm.router_python_core import PythonCore if model_name is None: - return Router._core_deployment_usable_by_team(deployment, team_id) + return PythonCore._core_deployment_usable_by_team(deployment, team_id) return self._router._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id) @overload @@ -1053,6 +1055,12 @@ SETTING_FIELDS: Final[tuple[str, ...]] = ( "cache_responses", ) +# Runtime settings kept as plain router attributes rather than ``settings()`` fields. A write +# to one (``router.num_retries = 2``) is a changeset too: ``ChangeSet(assign={...})``. +PLAIN_SETTING_ATTRIBUTES: Final[frozenset[str]] = frozenset({"num_retries", "max_fallbacks", "retry_after", "timeout"}) +# Former attributes whose value lives under another name, assignable through a changeset. +ASSIGNABLE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"model_group_alias": "_core_model_group_alias"}) + _MISSING: Final = object() @@ -1168,16 +1176,16 @@ class RouterHealth: @staticmethod def is_blocked(deployment: "Deployment") -> bool: """Whether an admin paused this deployment (was ``is_deployment_blocked``).""" - from litellm.router import Router + from litellm.router_python_core import PythonCore - return Router._core_is_deployment_blocked(deployment) + return PythonCore._core_is_deployment_blocked(deployment) @staticmethod def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool: """Whether every deployment in the list is paused (was ``are_all_deployments_blocked``).""" - from litellm.router import Router + from litellm.router_python_core import PythonCore - return Router._core_are_all_deployments_blocked(deployments) + return PythonCore._core_are_all_deployments_blocked(deployments) def fully_blocked_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set """Groups whose every deployment is paused (was ``get_fully_blocked_model_names``).""" @@ -1296,13 +1304,17 @@ class RoutingObservers: def apply_changes(router: "Router", changes: ChangeSet) -> ApplyResult: if changes.refresh_model_info: raise TypeError("ChangeSet(refresh_model_info=True) needs aapply()") - unknown: Final = sorted(set(changes.assign) - set(SETTING_FIELDS)) + unknown: Final = sorted(set(changes.assign) - {*SETTING_FIELDS, *PLAIN_SETTING_ATTRIBUTES, *ASSIGNABLE_ALIASES}) if unknown: raise ValueError(f"not assignable router settings: {unknown}") for name, value in changes.assign.items(): - setattr(router, "_core_" + name, value) + setattr( + router, + name if name in PLAIN_SETTING_ATTRIBUTES else ASSIGNABLE_ALIASES.get(name, "_core_" + name), + detached(value), + ) if changes.settings: - router._core_update_settings(**changes.settings) + router._core_update_settings(**{name: detached(value) for name, value in changes.settings.items()}) upserted: Final = [router._core_upsert_deployment(deployment=d) for d in changes.upsert] deleted: Final = [router._core_delete_deployment(id=i) for i in changes.delete] if changes.redis_cache is not None: @@ -1388,7 +1400,7 @@ class RoutingContext: return self._router.observe() def belongs_to(self, router: "Router") -> bool: - return self._router is router + return self._router is router or router.__dict__.get("_core") is self._router @property def explicit_allowed_fails(self) -> int | None: diff --git a/litellm/router_config.py b/litellm/router_config.py new file mode 100644 index 00000000000..75c79068191 --- /dev/null +++ b/litellm/router_config.py @@ -0,0 +1,157 @@ +""" +The router's configuration, built outside the router. + +``RouterConfig`` is static: the constructor arguments, frozen for the router's life. +``RouterDynamicConfig`` is a snapshot of the process settings the router reads while it serves +(``litellm.num_retries``, ``litellm.model_cost``, the fallback lists, ...). A call reads one snapshot +at its start. This module is the only router code that reads ``litellm.*`` settings. +""" + +import itertools +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +import litellm + +if TYPE_CHECKING: + from litellm.caching.caching import Cache + from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager + + +@dataclass(frozen=True, slots=True) +class RouterConfig: + """The arguments the router was built with, exactly as passed (unset ones are absent).""" + + arguments: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) + + def get(self, name: str, default: object = None) -> object: + return self.arguments.get(name, default) + + +@dataclass(frozen=True, slots=True) +class RouterDynamicConfig: + """Process settings the router reads per call. Values are the live objects, not copies.""" + + version: int + num_retries: int | None + max_fallbacks: int | None + router_max_fallbacks: int + allowed_fails: int + request_timeout: float + fallbacks: Sequence[object] | None + default_fallbacks: Sequence[str] | None + context_window_fallbacks: Sequence[object] | None + content_policy_fallbacks: Sequence[object] | None + drop_params: bool | None + provider_list: Sequence[str] + expose_router_debug_in_errors: bool + redact_user_api_key_info: bool | None + model_alias_map: Mapping[str, str] + model_cost: Mapping[str, object] + known_custom_logger_compatible_callbacks: Sequence[str] + exception_types: Sequence[type[BaseException]] + callbacks: Sequence[object] + success_callback: Sequence[object] + failure_callback: Sequence[object] + input_callback: Sequence[object] + callback_manager: "LoggingCallbackManager" + cache: "Cache | None" + + +_versions: Final = itertools.count(1) + + +def detached(value: object) -> object: + """``value`` with its plain containers (dict, list, tuple, set) copied, all the way down, so the + caller's own objects can't change the router afterwards. Other objects (clients, caches, + callbacks, pydantic models) are kept as they are.""" + if isinstance(value, dict): + return {key: detached(item) for key, item in value.items()} + if isinstance(value, list): + return [detached(item) for item in value] + if isinstance(value, tuple) and type(value) is tuple: + return tuple(detached(item) for item in value) + if isinstance(value, set): + return {detached(item) for item in value} + return value + + +def build_router_config(arguments: Mapping[str, object]) -> RouterConfig: + """The constructor arguments, detached from the caller's objects: settings are static.""" + return RouterConfig(arguments=MappingProxyType({name: detached(value) for name, value in arguments.items()})) + + +def current_dynamic_config() -> RouterDynamicConfig: + """A snapshot of the process settings as they are right now.""" + return RouterDynamicConfig( + version=next(_versions), + num_retries=litellm.num_retries, + max_fallbacks=litellm.max_fallbacks, + router_max_fallbacks=litellm.ROUTER_MAX_FALLBACKS, + allowed_fails=litellm.allowed_fails, + request_timeout=litellm.request_timeout, + fallbacks=litellm.fallbacks, + default_fallbacks=litellm.default_fallbacks, + context_window_fallbacks=litellm.context_window_fallbacks, + content_policy_fallbacks=litellm.content_policy_fallbacks, + drop_params=litellm.drop_params, + provider_list=litellm.provider_list, + expose_router_debug_in_errors=litellm.expose_router_debug_in_errors, + redact_user_api_key_info=litellm.redact_user_api_key_info, + model_alias_map=litellm.model_alias_map, + model_cost=litellm.model_cost, + known_custom_logger_compatible_callbacks=litellm._known_custom_logger_compatible_callbacks, + exception_types=litellm.LITELLM_EXCEPTION_TYPES, + callbacks=litellm.callbacks, + success_callback=litellm.success_callback, + failure_callback=litellm.failure_callback, + input_callback=litellm.input_callback, + callback_manager=litellm.logging_callback_manager, + cache=litellm.cache, + ) + + +_LIVE_SOURCES: Final = MappingProxyType( + { + "num_retries": "num_retries", + "max_fallbacks": "max_fallbacks", + "router_max_fallbacks": "ROUTER_MAX_FALLBACKS", + "allowed_fails": "allowed_fails", + "request_timeout": "request_timeout", + "fallbacks": "fallbacks", + "default_fallbacks": "default_fallbacks", + "context_window_fallbacks": "context_window_fallbacks", + "content_policy_fallbacks": "content_policy_fallbacks", + "drop_params": "drop_params", + "provider_list": "provider_list", + "expose_router_debug_in_errors": "expose_router_debug_in_errors", + "redact_user_api_key_info": "redact_user_api_key_info", + "model_alias_map": "model_alias_map", + "model_cost": "model_cost", + "known_custom_logger_compatible_callbacks": "_known_custom_logger_compatible_callbacks", + "exception_types": "LITELLM_EXCEPTION_TYPES", + "callbacks": "callbacks", + "success_callback": "success_callback", + "failure_callback": "failure_callback", + "input_callback": "input_callback", + "callback_manager": "logging_callback_manager", + "cache": "cache", + } +) + + +class LiveDynamicConfig: + """``RouterDynamicConfig``'s fields read from the process settings at access time, with no snapshot.""" + + __slots__ = () + + def __getattr__(self, name: str) -> object: + source: Final = _LIVE_SOURCES.get(name) + if source is None: + raise AttributeError(name) + return getattr(litellm, source) + + +live_dynamic_config: Final = LiveDynamicConfig() diff --git a/litellm/router_core.py b/litellm/router_core.py new file mode 100644 index 00000000000..acbe82a0643 --- /dev/null +++ b/litellm/router_core.py @@ -0,0 +1,56 @@ +""" +``RouterCore``: the interface ``litellm.Router`` delegates to, and ``make_core``, which picks one. + +``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. The Rust core +(litellm-rust/crates/router-core) implements the same operations. The core is picked once, when +the router is built, from its ``RouterConfig``. +""" + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +from litellm.router_config import RouterConfig + +if TYPE_CHECKING: + from litellm.router_api import ( + ApplyResult, + ChangeSet, + RouterHealth, + RouterRegistry, + RouterSettings, + RoutingObservers, + ) + from litellm.router_python_core import PythonCore + from litellm.router_python_host import PythonHost + from litellm.types.router import CustomRoutingStrategyBase + + +@runtime_checkable +class RouterCore(Protocol): + async def call(self, op: str, request: Mapping[str, object]) -> object: ... + + def registry(self) -> "RouterRegistry": ... + + def settings(self) -> "RouterSettings": ... + + def health(self) -> "RouterHealth": ... + + async def select( + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the selection cores take a dict + ) -> object: ... + + def apply(self, changes: "ChangeSet") -> "ApplyResult": ... + + def observe(self) -> "RoutingObservers": ... + + def register_plugin(self, plugin: "CustomRoutingStrategyBase") -> None: ... + + def retire(self) -> None: ... + + +def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore: + from litellm.router_python_core import PythonCore + + return (core_class or PythonCore)(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the arguments were bound against PythonCore's own signature diff --git a/litellm/router_deprecated.py b/litellm/router_deprecated.py index 0b328cc3e6b..83a1251d3c0 100644 --- a/litellm/router_deprecated.py +++ b/litellm/router_deprecated.py @@ -31,7 +31,7 @@ from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload -from litellm.router_api import SETTING_FIELDS, ChangeSet +from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet if TYPE_CHECKING: from typing import Any # noqa: TID251 # only to repeat two old signatures that had Any, unchanged @@ -65,7 +65,7 @@ _INTERNAL_PREFIX: Final = "litellm.router" # Router's own attribute hooks: a write or delete through an old name passes through them first. -_ATTRIBUTE_HOOKS: Final = frozenset({"__setattr__", "__delattr__"}) +_ATTRIBUTE_HOOKS: Final = frozenset({"__getattribute__", "__setattr__", "__delattr__"}) def _note(name: str, replacement: str) -> None: @@ -88,9 +88,10 @@ def _note(name: str, replacement: str) -> None: def _as_router(obj: "DeprecatedRouterMembers") -> "Router": """The Router this mixin is part of (it is mixed into ``litellm.Router`` only).""" from litellm.router import Router + from litellm.router_python_core import PythonCore - if isinstance(obj, Router): - return obj + if isinstance(obj, (Router, PythonCore)): + return obj # pyright: ignore[reportReturnType] # a PythonCore carries every member the mixin reads raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only") @@ -134,7 +135,7 @@ class _DeprecatedAttribute(Generic[_T]): def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None: _note(self.name, self.replacement) - if self.name in SETTING_FIELDS: + if self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES: _as_router(obj).apply(ChangeSet(assign={self.name: value})) else: setattr(obj, "_core_" + self.name, value) @@ -169,6 +170,13 @@ _OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] = ) +def _override_key(router: object, name: str) -> tuple[int, str]: + """The running-override key for ``router``: a Router and its core share one, so an old member + entered on the Router is seen by the override routing that runs on the core.""" + core: Final = getattr(router, "__dict__", {}).get("_core") + return (id(router if core is None else core), name) + + def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., object]: """ A direct call of a subclass override (``router.()``) marks it running, so its @@ -184,7 +192,7 @@ def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., *args: object, **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged ) -> object: - token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)}) + token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)}) try: return await async_override(router, *args, **kwargs) finally: @@ -198,7 +206,7 @@ def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., *args: object, **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged ) -> object: - token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)}) + token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)}) try: return override(router, *args, **kwargs) finally: @@ -236,7 +244,7 @@ def _route_override(cls: type, name: str, override: Callable[..., object]) -> Ca *args: object, **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged ) -> object: - key: Final = (id(router), name) + key: Final = _override_key(router, name) running: Final = _OVERRIDES_RUNNING.get() if key in running: parent_async: Final[Callable[..., Awaitable[object]]] = getattr(super(cls, router), core) @@ -255,7 +263,7 @@ def _route_override(cls: type, name: str, override: Callable[..., object]) -> Ca *args: object, **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged ) -> object: - key: Final = (id(router), name) + key: Final = _override_key(router, name) running: Final = _OVERRIDES_RUNNING.get() if key in running: return parent(router)(*args, **kwargs) @@ -502,9 +510,9 @@ class DeprecatedRouterMembers: @staticmethod def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool: _note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)") - from litellm.router import Router + from litellm.router_python_core import PythonCore - return Router._core_deployment_usable_by_team(model, team_id) + return PythonCore._core_deployment_usable_by_team(model, team_id) _deployment_usable_by_team = deployment_usable_by_team diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 535072d5fcf..94d52d31b21 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -8,26 +8,12 @@ # Thank you ! We ❤️ you! - Krrish & Ishaan """ -litellm.Router: load balancing, retries, fallbacks and cooldowns across deployments. +``PythonCore``: the Python implementation of the router core behind ``litellm.Router``. -Public surface. The router is moving behind a narrow interface; code outside -the router should end up using a Router through two kinds of members only: - -- endpoint methods (acompletion, aembedding, aresponses, aanthropic_messages, - the files, batches, assistants and vector store wrappers, ...), each a thin - wrapper around one call; -- entry points (being added) that replace direct access to internals: registry() for - deployment lookups, settings() for configuration reads, health() for - cooldowns, blocks and usage, select() to pick a deployment without calling - it, apply() for runtime changes, observe() for routing events, - register_plugin() for strategies and sub-routers, and retire() to release - the router. - -Everything else on the class is internal. The members outside code uses today -are listed in tests/code_coverage_tests/router_surface_allowlist.json, and a CI -check (router_surface_check.py next to it) fails on any new outside use, so the -list only shrinks while callers move to the entry points. The same interface -is documented for the Rust core in litellm-rust/crates/router-core. +It holds all routing logic (registry, selection, retries, fallbacks, cooldowns, usage) and reaches +outside itself only through its ``PythonHost``: inference calls, sleeps, callback registration and +the process settings (``litellm/router_config.py``). The Rust core in litellm-rust/crates/router-core +implements the same interface. ``litellm/router.py`` picks one at construction and delegates to it. """ import asyncio @@ -71,12 +57,13 @@ from openai import AsyncOpenAI from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import assert_never, overload -import litellm -import litellm.litellm_core_utils.exception_mapping_utils +from litellm import exceptions as litellm_exceptions from litellm import get_secret_str +from litellm import utils as litellm_utils from litellm._internal_context import in_mirrored_call, mirrored_call, service_target, with_service_target from litellm._logging import verbose_router_logger from litellm._uuid import uuid +from litellm.caching.caching import Cache as LitellmCache from litellm.caching.caching import ( DualCache, InMemoryCache, @@ -106,6 +93,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.routing import routing_decision_attributes from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span +from litellm.litellm_core_utils import exception_mapping_utils from litellm.litellm_core_utils.access_windows import access_windows_config_error, filter_reserved_deployments from litellm.litellm_core_utils.add_retry_fallback_headers import ( add_fallback_headers_to_response, @@ -248,9 +236,11 @@ from litellm.llms.openai_like.model_info import ( MODEL_INFO_REFRESH_SECONDS, get_openai_compatible_model_info, ) +from litellm.main import Chat as LitellmChat from litellm.responses.bridge_check import keep_registered_mode from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults from litellm.router_api import ( + PLAIN_SETTING_ATTRIBUTES, SETTING_FIELDS, ApplyResult, ChangeSet, @@ -266,6 +256,7 @@ from litellm.router_api import RoutingContext as RouterContext from litellm.router_api import deployment_id as _deployment_id from litellm.router_deprecated import DeprecatedRouterMembers from litellm.router_host import router_host +from litellm.router_python_host import PythonHost, python_host from litellm.router_strategy.auto_router.auto_router_model_naming import ( AUTO_ROUTER_MODEL_PREFIX, STRATEGY_ROUTER_PARAM_FIELDS, @@ -766,7 +757,7 @@ def _search_tool_support(deployment: DeploymentTypedDict) -> bool | None: chat_config: Final = ProviderConfigManager.get_provider_chat_config( model=resolved_model, provider=LlmProviders(resolved_provider) ) - except (litellm.BadRequestError, ValueError): + except (litellm_exceptions.BadRequestError, ValueError): return None return True if chat_config_builds_anthropic_request(chat_config) else None @@ -879,7 +870,7 @@ def _stream_fallback_error_for_raised( status_code: Final = _stream_raised_error_status(error) if status_code is None: return _stream_pre_content_error(error, model, llm_provider) - retriable: Final = litellm.should_retry(status_code) + retriable: Final = python_host.invoke("should_retry")(status_code) return _stream_pre_content_error(error, model, llm_provider) if retriable else None @@ -899,16 +890,16 @@ def _stream_pre_content_error( def _context_window_stream_fallback_error( - error: "MidStreamFallbackError | litellm.ContextWindowExceededError", + error: "MidStreamFallbackError | litellm_exceptions.ContextWindowExceededError", model: object, llm_provider: str, has_output: bool, ) -> "MidStreamFallbackError": - if isinstance(error, litellm.ContextWindowExceededError): + if isinstance(error, litellm_exceptions.ContextWindowExceededError): if has_output: raise error return _stream_pre_content_error(error, model if isinstance(model, str) else "", llm_provider) - if isinstance(error.original_exception, litellm.ContextWindowExceededError) and ( + if isinstance(error.original_exception, litellm_exceptions.ContextWindowExceededError) and ( has_output or error.generated_content or error.emitted_disqualifying_content ): raise error.original_exception @@ -1133,7 +1124,7 @@ class FallbackAwareAnthropicMessagesStream: if source_params is None or source_params is self._followed_source_params: return self._followed_source_params = source_params - hidden_params, headers = Router._prepare_fallback_hidden_params(source) # pyright: ignore[reportPrivateUsage] # this wrapper is the Router's own stream type + hidden_params, headers = PythonCore._prepare_fallback_hidden_params(source) # pyright: ignore[reportPrivateUsage] # this wrapper is the Router's own stream type self.merge_fallback_hidden_params(hidden_params, headers) def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream": @@ -1175,8 +1166,8 @@ class RoutingArgs(enum.Enum): # entries their deployments own. Weak so a router nothing references any more, such # as the per-request one built from a caller-supplied user_config, drops out on its # own rather than leaving entries behind that nothing can withdraw. -_live_routers: Final["weakref.WeakSet[Router]"] = weakref.WeakSet() -_routers_holding_global_callbacks: Final["weakref.WeakSet[Router]"] = weakref.WeakSet() +_live_routers: Final["weakref.WeakSet[PythonCore]"] = weakref.WeakSet() +_routers_holding_global_callbacks: Final["weakref.WeakSet[PythonCore]"] = weakref.WeakSet() _routers_holding_global_callbacks_lock: Final = threading.Lock() @@ -1259,7 +1250,7 @@ def stamped_client_ceiling(request_kwargs: Mapping[str, object]) -> Mapping[str, def _exact_base_model_max_input_tokens(base_model: str) -> int | None: try: - model_info: Final = litellm.get_model_info(model=base_model) + model_info: Final = python_host.invoke("get_model_info")(model=base_model) except Exception: # noqa: BLE001 # get_model_info raises ModelNotMappedError or a bare Exception for an unmapped model return None if is_generalized_model_info(model_info): @@ -1303,10 +1294,13 @@ def _base_model_widened_window( _SETTING_ATTRIBUTES: Final = frozenset("_core_" + name for name in SETTING_FIELDS) -class Router(DeprecatedRouterMembers): +class PythonCore(DeprecatedRouterMembers): _core_model_names: set[str] = set() _core_cache_responses: bool | None = False _registry_version: int = 0 + # Bumped by every ``apply``: the static arguments plus ``_core_applied`` are the settings. + _settings_version: int = 0 + _core_applied: Mapping[str, object] = MappingProxyType({}) # -- entry points: everything outside code needs besides the endpoint methods --------------- # See litellm/router_api.py. The members outside code used before these live in @@ -1335,6 +1329,14 @@ class Router(DeprecatedRouterMembers): return snapshot def __setattr__(self, name: str, value: object) -> None: + if ( + (name in PLAIN_SETTING_ATTRIBUTES or name in _SETTING_ATTRIBUTES) + and self.__dict__.get("_core_ready") + and not self.__dict__.get("_core_applying") + ): + # Settings change only through a changeset, so every core serving this router sees it. + self.apply(ChangeSet(assign={name.removeprefix("_core_"): value})) + return if name in _SETTING_ATTRIBUTES: self.__dict__.pop("_settings_snapshot", None) elif name == "_core_model_list" and "_core_model_list" in self.__dict__: @@ -1431,11 +1433,28 @@ class Router(DeprecatedRouterMembers): def apply(self, changes: ChangeSet) -> ApplyResult: """Apply runtime changes: settings, deployments, the routing state's Redis cache.""" - return apply_changes(self, changes) + with self._core_applying_changes(changes): + return apply_changes(self, changes) async def aapply(self, changes: ChangeSet) -> ApplyResult: """``apply``, plus the changes that need I/O (``refresh_model_info``).""" - return await aapply_changes(self, changes) + with self._core_applying_changes(changes): + return await aapply_changes(self, changes) + + @contextlib.contextmanager + def _core_applying_changes(self, changes: ChangeSet) -> Iterator[None]: + """Mark a changeset in progress, then record its settings and bump ``_settings_version``.""" + depth: Final = self.__dict__.get("_core_applying", 0) + self.__dict__["_core_applying"] = depth + 1 + try: + yield + finally: + self.__dict__["_core_applying"] = depth + if changes.assign or changes.settings: + self.__dict__["_core_applied"] = MappingProxyType( + {**self._core_applied, **changes.assign, **changes.settings} + ) + self.__dict__["_settings_version"] = self._settings_version + 1 def observe(self, logging_obj: "LiteLLMLogging | None" = None, *, request_scoped: bool = False) -> RoutingObservers: """ @@ -1456,6 +1475,17 @@ class Router(DeprecatedRouterMembers): """Release the router's global callback registrations and routing state.""" self._core_discard() + def _core_public(self) -> "PythonCore": + """The ``litellm.Router`` this core serves, for objects the router hands out; the core itself if none.""" + owner: Final = self.__dict__.get("_core_owner") + public: Final = owner() if owner is not None else None + return self if public is None else public + + async def call(self, op: str, request: Mapping[str, object]) -> object: + """Run one endpoint (``acompletion``, ``aembedding``, ...) with the request's keyword payload.""" + result: Final = getattr(self, op)(**request) + return await result if inspect.isawaitable(result) else result + default_cache_time_seconds: int = 1 * 60 * 60 # 1 hour tenacity = None leastbusy_logger: LeastBusyLoggingHandler | None = None @@ -1531,6 +1561,7 @@ class Router(DeprecatedRouterMembers): fallback_budget_check: FallbackBudgetCheck | None = None, auto_router_capability_limit: AutoRouterCapabilityLimit | None = None, session_affinity_repin_on_failover: bool = False, + _host: PythonHost = python_host, ) -> None: """ Initialize the Router class with the given parameters for caching, reliability, and routing strategy. @@ -1610,6 +1641,7 @@ class Router(DeprecatedRouterMembers): router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": ["openai-gpt-3.5-turbo"]}]) ``` """ + self._host: Final = _host self._core_ctx: Final = RouterContext(self) self.set_verbose = set_verbose @@ -1625,7 +1657,7 @@ class Router(DeprecatedRouterMembers): from litellm._service_logger import ServiceLogging self.service_logger_obj: ServiceLogging = ServiceLogging() - litellm.suppress_debug_info = True # prevents 'Give Feedback/Get help' message from being emitted on Router - Relevant Issue: https://github.com/BerriAI/litellm/issues/5942 + self._host.suppress_debug_info() # prevents 'Give Feedback/Get help' message from being emitted on Router - Relevant Issue: https://github.com/BerriAI/litellm/issues/5942 if self.set_verbose is True: if debug_level == "INFO": verbose_router_logger.setLevel(logging.INFO) @@ -1671,9 +1703,9 @@ class Router(DeprecatedRouterMembers): redis_cache = self._create_redis_cache(cache_config) if cache_responses: - if litellm.cache is None: + if self._host.global_cache() is None: # the cache can be initialized on the proxy server. We should not overwrite it - litellm.cache = litellm.Cache(type=cache_type, **cache_config) + self._host.set_global_cache(LitellmCache(type=cache_type, **cache_config)) self._core_cache_responses = cache_responses self._core_cache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache() @@ -1779,7 +1811,7 @@ class Router(DeprecatedRouterMembers): if allowed_fails is not None: self._core_allowed_fails = allowed_fails else: - self._core_allowed_fails = litellm.allowed_fails + self._core_allowed_fails = self._host.dynamic().allowed_fails self.explicit_allowed_fails = allowed_fails self._core_cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS self._core_cooldown_cache = CooldownCache( @@ -1805,20 +1837,20 @@ class Router(DeprecatedRouterMembers): if num_retries is not None: self.num_retries = num_retries - elif litellm.num_retries is not None: - self.num_retries = litellm.num_retries + elif self._host.dynamic().num_retries is not None: + self.num_retries = self._host.dynamic().num_retries else: self.num_retries = openai.DEFAULT_MAX_RETRIES if max_fallbacks is not None: self.max_fallbacks = max_fallbacks - elif litellm.max_fallbacks is not None: - self.max_fallbacks = litellm.max_fallbacks + elif self._host.dynamic().max_fallbacks is not None: + self.max_fallbacks = self._host.dynamic().max_fallbacks else: - self.max_fallbacks = litellm.ROUTER_MAX_FALLBACKS + self.max_fallbacks = self._host.dynamic().router_max_fallbacks self._explicit_timeout = timeout # None when user did not pass timeout - self.timeout = timeout or litellm.request_timeout + self.timeout = timeout or self._host.dynamic().request_timeout # Per-attempt request_timeout, independent of router_settings.timeout. # Only stored when a router timeout is also set, since otherwise # request_timeout already flows through self.timeout above. @@ -1831,14 +1863,14 @@ class Router(DeprecatedRouterMembers): ## SETTING FALLBACKS ## ### validate if it's set + in correct format - _fallbacks = fallbacks or litellm.fallbacks + _fallbacks = fallbacks or self._host.dynamic().fallbacks self.validate_fallbacks(fallback_param=_fallbacks) ### set fallbacks self._core_fallbacks = _fallbacks - if default_fallbacks is not None or litellm.default_fallbacks is not None: - _fallbacks = default_fallbacks or litellm.default_fallbacks + if default_fallbacks is not None or self._host.dynamic().default_fallbacks is not None: + _fallbacks = default_fallbacks or self._host.dynamic().default_fallbacks if isinstance(_fallbacks, str): raise TypeError( f"default_fallbacks must be a list of model names, got '{_fallbacks}'. Use ['{_fallbacks}']" @@ -1848,9 +1880,9 @@ class Router(DeprecatedRouterMembers): else: self._core_fallbacks = [{"*": _fallbacks}] - self._core_context_window_fallbacks = context_window_fallbacks or litellm.context_window_fallbacks + self._core_context_window_fallbacks = context_window_fallbacks or self._host.dynamic().context_window_fallbacks - _content_policy_fallbacks: Final = content_policy_fallbacks or litellm.content_policy_fallbacks + _content_policy_fallbacks: Final = content_policy_fallbacks or self._host.dynamic().content_policy_fallbacks self.validate_fallbacks(fallback_param=_content_policy_fallbacks) self._core_content_policy_fallbacks = _content_policy_fallbacks self.total_calls: defaultdict = defaultdict(int) # dict to store total calls made to each model @@ -1859,7 +1891,7 @@ class Router(DeprecatedRouterMembers): # make Router.chat.completions.create compatible for openai.chat.completions.create default_litellm_params = default_litellm_params or {} - self.chat = litellm.Chat(params=default_litellm_params, router_obj=self) + self.chat = LitellmChat(params=default_litellm_params, router_obj=self._core_public()) # default litellm args self._core_default_litellm_params = default_litellm_params @@ -1901,17 +1933,17 @@ class Router(DeprecatedRouterMembers): with _routers_holding_global_callbacks_lock: _routers_holding_global_callbacks.add(self) ## USAGE TRACKING ## - litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success) - if isinstance(litellm.success_callback, list): - litellm.logging_callback_manager.add_litellm_success_callback(self.sync_deployment_callback_on_success) + self._host.callbacks.add_litellm_async_success_callback(self.deployment_callback_on_success) + if isinstance(self._host.callback_list("success_callback"), list): + self._host.callbacks.add_litellm_success_callback(self.sync_deployment_callback_on_success) else: - litellm.success_callback = [self.sync_deployment_callback_on_success] - litellm.logging_callback_manager.add_litellm_async_failure_callback(self.async_deployment_callback_on_failure) + self._host.set_callback_list("success_callback", [self.sync_deployment_callback_on_success]) + self._host.callbacks.add_litellm_async_failure_callback(self.async_deployment_callback_on_failure) ## COOLDOWNS ## - if isinstance(litellm.failure_callback, list): - litellm.logging_callback_manager.add_litellm_failure_callback(self.deployment_callback_on_failure) + if isinstance(self._host.callback_list("failure_callback"), list): + self._host.callbacks.add_litellm_failure_callback(self.deployment_callback_on_failure) else: - litellm.failure_callback = [self.deployment_callback_on_failure] + self._host.set_callback_list("failure_callback", [self.deployment_callback_on_failure]) self.routing_strategy_args = routing_strategy_args self._core_provider_budget_config = provider_budget_config self.router_budget_logger: RouterBudgetLimiting | None = None @@ -1966,17 +1998,16 @@ class Router(DeprecatedRouterMembers): self.initialize_assistants_endpoint() self.initialize_router_endpoints() self.apply_default_settings() + # From here on a settings write is a changeset (``__setattr__``). + self.__dict__["_core_ready"] = True @staticmethod def get_valid_args() -> list[str]: """ Returns a list of valid arguments for the Router.__init__ method. """ - arg_spec: Final = inspect.getfullargspec(Router.__init__) - valid_args: Final = arg_spec.args + arg_spec.kwonlyargs - if "self" in valid_args: - valid_args.remove("self") - return valid_args + arg_spec: Final = inspect.getfullargspec(PythonCore.__init__) + return [arg for arg in (*arg_spec.args, *arg_spec.kwonlyargs) if arg not in ("self", "_host")] def apply_default_settings(self): """ @@ -1989,10 +2020,9 @@ class Router(DeprecatedRouterMembers): if self.enable_mid_stream_fallback_continuation: self._register_continuation_prefill_check() - @staticmethod - def _register_continuation_prefill_check() -> None: + def _register_continuation_prefill_check(self) -> None: # Process-global on purpose: Router._core_discard() must not drop a filter another router still needs. - litellm.logging_callback_manager.add_litellm_callback(ContinuationPrefillDeploymentCheck()) + self._host.callbacks.add_litellm_callback(ContinuationPrefillDeploymentCheck()) def _core_discard(self) -> None: """ @@ -2010,7 +2040,7 @@ class Router(DeprecatedRouterMembers): _routers_holding_global_callbacks.discard(self) slack_alerting_loggers: Final = self._slack_alerting_loggers() adaptive_router_hooks: Final = self._adaptive_router_hooks() - vacated_keys: Final = litellm.logging_callback_manager.drop_callbacks( + vacated_keys: Final = self._host.callbacks.drop_callbacks( ( *self._callback_selectors(), *self._override_selectors.values(), @@ -2023,11 +2053,9 @@ class Router(DeprecatedRouterMembers): with _routers_holding_global_callbacks_lock: survivors: Final = tuple(_routers_holding_global_callbacks) candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors) - for candidate in litellm.logging_callback_manager.with_custom_logger_keys(candidates, vacated_keys): + for candidate in self._host.callbacks.with_custom_logger_keys(candidates, vacated_keys): self._register_router_selector(candidate) - adaptive_keys: Final = frozenset( - map(litellm.logging_callback_manager.get_custom_logger_key, adaptive_router_hooks) - ) + adaptive_keys: Final = frozenset(map(self._host.callbacks.get_custom_logger_key, adaptive_router_hooks)) adaptive_survivor: Final = next((router for router in survivors if router._core_adaptive_routers), None) if adaptive_survivor is not None and vacated_keys & adaptive_keys: adaptive_survivor._sync_adaptive_router_hooks() @@ -2051,14 +2079,14 @@ class Router(DeprecatedRouterMembers): def _request_policies_not_set_globally(self) -> tuple[CustomLogger, ...]: global_keys: Final = frozenset( map( - litellm.logging_callback_manager.get_custom_logger_key, - (callback for callback in litellm.callbacks if isinstance(callback, CustomLogger)), + self._host.callbacks.get_custom_logger_key, + (callback for callback in self._host.callback_list("callbacks") if isinstance(callback, CustomLogger)), ) ) return tuple( policy for policy in self._request_scoped_policies - if litellm.logging_callback_manager.get_custom_logger_key(policy) not in global_keys + if self._host.callbacks.get_custom_logger_key(policy) not in global_keys ) def _request_adaptive_router_hooks(self) -> tuple[CustomLogger, ...]: @@ -2073,12 +2101,12 @@ class Router(DeprecatedRouterMembers): def _routing_callbacks(self) -> Iterable[object]: if not (self._request_scoped_selectors or self._request_scoped_policies): - return litellm.callbacks + return self._host.callback_list("callbacks") replaced_keys: Final = frozenset( - map(litellm.logging_callback_manager.get_custom_logger_key, self._request_scoped_selectors) + map(self._host.callbacks.get_custom_logger_key, self._request_scoped_selectors) ) return chain( - litellm.logging_callback_manager.without_custom_logger_keys(litellm.callbacks, replaced_keys), + self._host.callbacks.without_custom_logger_keys(self._host.callback_list("callbacks"), replaced_keys), self._request_scoped_selectors, self._request_policies_not_set_globally(), ) @@ -2100,7 +2128,7 @@ class Router(DeprecatedRouterMembers): own_strategy_ids: Final = frozenset( id(tagged.strategy) for tagged in chain.from_iterable(self._core_adaptive_routers.values()) ) - hooks: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook) + hooks: Final = self._host.callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook) return tuple( hook for hook in hooks @@ -2214,15 +2242,14 @@ class Router(DeprecatedRouterMembers): return selector - @staticmethod - def _register_router_selector(selector: CustomLogger) -> None: + def _register_router_selector(self, selector: CustomLogger) -> None: if isinstance(selector, LeastBusyLoggingHandler): - if isinstance(litellm.input_callback, list): - litellm.logging_callback_manager.add_litellm_input_callback(selector) + if isinstance(self._host.callback_list("input_callback"), list): + self._host.callbacks.add_litellm_input_callback(selector) else: - litellm.input_callback = [selector] - if isinstance(litellm.callbacks, list): - litellm.logging_callback_manager.add_litellm_callback(selector) + self._host.set_callback_list("input_callback", [selector]) + if isinstance(self._host.callback_list("callbacks"), list): + self._host.callbacks.add_litellm_callback(selector) def _callback_selectors(self) -> tuple[RouterStrategySelector, ...]: return tuple( @@ -2245,7 +2272,7 @@ class Router(DeprecatedRouterMembers): for selector in selectors: if isinstance(selector, BaseRoutingStrategy): selector.retire() - litellm.logging_callback_manager.drop_callbacks(selectors) + self._host.callbacks.drop_callbacks(selectors) def _apply_updated_routing_strategy_args(self) -> None: """ @@ -2741,60 +2768,75 @@ class Router(DeprecatedRouterMembers): def initialize_assistants_endpoint(self): ## INITIALIZE PASS THROUGH ASSISTANTS ENDPOINT ## - self.acreate_assistants = self.factory_function(litellm.acreate_assistants) - self.adelete_assistant = self.factory_function(litellm.adelete_assistant) - self.aget_assistants = self.factory_function(litellm.aget_assistants) - self.acreate_thread = self.factory_function(litellm.acreate_thread) - self.aget_thread = self.factory_function(litellm.aget_thread) - self.a_add_message = self.factory_function(litellm.a_add_message) - self.aget_messages = self.factory_function(litellm.aget_messages) - self.arun_thread = self.factory_function(litellm.arun_thread) + self.acreate_assistants = self.factory_function(self._host.invoke("acreate_assistants")) + self.adelete_assistant = self.factory_function(self._host.invoke("adelete_assistant")) + self.aget_assistants = self.factory_function(self._host.invoke("aget_assistants")) + self.acreate_thread = self.factory_function(self._host.invoke("acreate_thread")) + self.aget_thread = self.factory_function(self._host.invoke("aget_thread")) + self.a_add_message = self.factory_function(self._host.invoke("a_add_message")) + self.aget_messages = self.factory_function(self._host.invoke("aget_messages")) + self.arun_thread = self.factory_function(self._host.invoke("arun_thread")) def _initialize_core_endpoints(self): """Helper to initialize core router endpoints.""" - self.amoderation = self.factory_function(litellm.amoderation, call_type="moderation") - self.aanthropic_messages = self.factory_function(litellm.anthropic_messages, call_type="anthropic_messages") - self.anthropic_messages = self.factory_function(litellm.anthropic_messages, call_type="anthropic_messages") - self.agenerate_content = self.factory_function(litellm.agenerate_content, call_type="agenerate_content") + self.amoderation = self.factory_function(self._host.invoke("amoderation"), call_type="moderation") + self.aanthropic_messages = self.factory_function( + self._host.invoke("anthropic_messages"), call_type="anthropic_messages" + ) + self.anthropic_messages = self.factory_function( + self._host.invoke("anthropic_messages"), call_type="anthropic_messages" + ) + self.agenerate_content = self.factory_function( + self._host.invoke("agenerate_content"), call_type="agenerate_content" + ) self.aadapter_generate_content = self.factory_function( - litellm.aadapter_generate_content, call_type="aadapter_generate_content" + self._host.invoke("aadapter_generate_content"), call_type="aadapter_generate_content" ) - self.aresponses = self.factory_function(litellm.aresponses, call_type="aresponses") - self.afile_delete = self.factory_function(litellm.afile_delete, call_type="afile_delete") - self.afile_content = self.factory_function(litellm.afile_content, call_type="afile_content") - self.responses = self.factory_function(litellm.responses, call_type="responses") - self.aget_responses = self.factory_function(litellm.aget_responses, call_type="aget_responses") - self.acancel_responses = self.factory_function(litellm.acancel_responses, call_type="acancel_responses") - self.acompact_responses = self.factory_function(litellm.acompact_responses, call_type="acompact_responses") - self.adelete_responses = self.factory_function(litellm.adelete_responses, call_type="adelete_responses") - self.alist_input_items = self.factory_function(litellm.alist_input_items, call_type="alist_input_items") - self._arealtime = self.factory_function(litellm._arealtime, call_type="_arealtime") + self.aresponses = self.factory_function(self._host.invoke("aresponses"), call_type="aresponses") + self.afile_delete = self.factory_function(self._host.invoke("afile_delete"), call_type="afile_delete") + self.afile_content = self.factory_function(self._host.invoke("afile_content"), call_type="afile_content") + self.responses = self.factory_function(self._host.invoke("responses"), call_type="responses") + self.aget_responses = self.factory_function(self._host.invoke("aget_responses"), call_type="aget_responses") + self.acancel_responses = self.factory_function( + self._host.invoke("acancel_responses"), call_type="acancel_responses" + ) + self.acompact_responses = self.factory_function( + self._host.invoke("acompact_responses"), call_type="acompact_responses" + ) + self.adelete_responses = self.factory_function( + self._host.invoke("adelete_responses"), call_type="adelete_responses" + ) + self.alist_input_items = self.factory_function( + self._host.invoke("alist_input_items"), call_type="alist_input_items" + ) + self._arealtime = self.factory_function(self._host.invoke("_arealtime"), call_type="_arealtime") self.acreate_realtime_client_secret = self.factory_function( - litellm.acreate_realtime_client_secret, call_type="acreate_realtime_client_secret" + self._host.invoke("acreate_realtime_client_secret"), call_type="acreate_realtime_client_secret" ) - self.arealtime_calls = self.factory_function(litellm.arealtime_calls, call_type="arealtime_calls") + self.arealtime_calls = self.factory_function(self._host.invoke("arealtime_calls"), call_type="arealtime_calls") self.acreate_realtime_transcription_session = self.factory_function( - litellm.acreate_realtime_transcription_session, call_type="acreate_realtime_transcription_session" + self._host.invoke("acreate_realtime_transcription_session"), + call_type="acreate_realtime_transcription_session", ) self._aresponses_websocket = self.factory_function( - litellm._aresponses_websocket, call_type="_aresponses_websocket" + self._host.invoke("_aresponses_websocket"), call_type="_aresponses_websocket" ) self.acreate_fine_tuning_job = self.factory_function( - litellm.acreate_fine_tuning_job, call_type="acreate_fine_tuning_job" + self._host.invoke("acreate_fine_tuning_job"), call_type="acreate_fine_tuning_job" ) self.acancel_fine_tuning_job = self.factory_function( - litellm.acancel_fine_tuning_job, call_type="acancel_fine_tuning_job" + self._host.invoke("acancel_fine_tuning_job"), call_type="acancel_fine_tuning_job" ) self.alist_fine_tuning_jobs = self.factory_function( - litellm.alist_fine_tuning_jobs, call_type="alist_fine_tuning_jobs" + self._host.invoke("alist_fine_tuning_jobs"), call_type="alist_fine_tuning_jobs" ) self.aretrieve_fine_tuning_job = self.factory_function( - litellm.aretrieve_fine_tuning_job, call_type="aretrieve_fine_tuning_job" + self._host.invoke("aretrieve_fine_tuning_job"), call_type="aretrieve_fine_tuning_job" ) - self.afile_list = self.factory_function(litellm.afile_list, call_type="alist_files") - self.aimage_edit = self.factory_function(litellm.aimage_edit, call_type="aimage_edit") + self.afile_list = self.factory_function(self._host.invoke("afile_list"), call_type="alist_files") + self.aimage_edit = self.factory_function(self._host.invoke("aimage_edit"), call_type="aimage_edit") self.allm_passthrough_route = self.factory_function( - litellm.allm_passthrough_route, call_type="allm_passthrough_route" + self._host.invoke("allm_passthrough_route"), call_type="allm_passthrough_route" ) # Note: acancel_batch is defined as a method on the Router class (not using factory_function) # to properly handle model-to-provider mapping like acreate_batch and aretrieve_batch @@ -3019,10 +3061,10 @@ class Router(DeprecatedRouterMembers): def _initialize_skills_endpoints(self): """Initialize Anthropic Skills API endpoints.""" - self.acreate_skill = self.factory_function(litellm.acreate_skill, call_type="acreate_skill") - self.alist_skills = self.factory_function(litellm.alist_skills, call_type="alist_skills") - self.aget_skill = self.factory_function(litellm.aget_skill, call_type="aget_skill") - self.adelete_skill = self.factory_function(litellm.adelete_skill, call_type="adelete_skill") + self.acreate_skill = self.factory_function(self._host.invoke("acreate_skill"), call_type="acreate_skill") + self.alist_skills = self.factory_function(self._host.invoke("alist_skills"), call_type="alist_skills") + self.aget_skill = self.factory_function(self._host.invoke("aget_skill"), call_type="aget_skill") + self.adelete_skill = self.factory_function(self._host.invoke("adelete_skill"), call_type="adelete_skill") def _initialize_interactions_endpoints(self): """Initialize Google Interactions API endpoints.""" @@ -3147,10 +3189,10 @@ class Router(DeprecatedRouterMembers): model_group_affinity_config=self.model_group_affinity_config, ) self.optional_callbacks.append(ec_callback) - litellm.logging_callback_manager.add_litellm_callback(ec_callback) + self._host.callbacks.add_litellm_callback(ec_callback) _move_before_deployment_affinity(self.optional_callbacks, ec_callback) - _move_before_deployment_affinity(litellm.callbacks, ec_callback) + _move_before_deployment_affinity(self._host.callback_list("callbacks"), ec_callback) def _ensure_deployment_affinity_callback(self) -> None: """Register the DeploymentAffinityCheck callback (global flags all False) if absent. @@ -3174,7 +3216,7 @@ class Router(DeprecatedRouterMembers): session_affinity_repin_on_failover=self.session_affinity_repin_on_failover, ) self.optional_callbacks.append(affinity_callback) - litellm.logging_callback_manager.add_litellm_callback(affinity_callback) + self._host.callbacks.add_litellm_callback(affinity_callback) def add_optional_pre_call_checks(self, optional_pre_call_checks: OptionalPreCallChecks | None): if optional_pre_call_checks is None: @@ -3221,7 +3263,7 @@ class Router(DeprecatedRouterMembers): session_affinity_repin_on_failover=self.session_affinity_repin_on_failover, ) self.optional_callbacks.append(affinity_callback) - litellm.logging_callback_manager.add_litellm_callback(affinity_callback) + self._host.callbacks.add_litellm_callback(affinity_callback) # --------------------------------------------------------------------- # Encrypted content affinity @@ -3268,7 +3310,7 @@ class Router(DeprecatedRouterMembers): if self.optional_callbacks is None: self.optional_callbacks = [] self.optional_callbacks.append(_callback) - litellm.logging_callback_manager.add_litellm_callback(_callback) + self._host.callbacks.add_litellm_callback(_callback) def set_optional_pre_call_checks(self, optional_pre_call_checks: OptionalPreCallChecks | None) -> None: if optional_pre_call_checks is None: @@ -3288,10 +3330,10 @@ class Router(DeprecatedRouterMembers): for router in tuple(_live_routers) ): return - for cb in tuple(litellm.callbacks): + for cb in tuple(self._host.callback_list("callbacks")): if type(cb) is callback_cls: - litellm.logging_callback_manager.remove_callback_from_list_by_object( - litellm.callbacks, cb, require_self=False + self._host.callbacks.remove_callback_from_list_by_object( + self._host.callback_list("callbacks"), cb, require_self=False ) def print_deployment(self, deployment: dict): @@ -3304,7 +3346,7 @@ class Router(DeprecatedRouterMembers): _deployment_copy: Final = copy.deepcopy(deployment) litellm_params: Final[dict] = _deployment_copy["litellm_params"] - if litellm.redact_user_api_key_info: + if self._host.dynamic().redact_user_api_key_info: masker: Final = SensitiveDataMasker(visible_prefix=2, visible_suffix=0) _deployment_copy["litellm_params"] = masker.mask_dict(litellm_params) elif "api_key" in litellm_params: @@ -3331,16 +3373,16 @@ class Router(DeprecatedRouterMembers): return sanitized sanitized.pop("thinking", None) - Router._pop_effort_from_nested_carrier(sanitized, "output_config") - Router._pop_effort_from_nested_carrier(sanitized, "reasoning") + PythonCore._pop_effort_from_nested_carrier(sanitized, "output_config") + PythonCore._pop_effort_from_nested_carrier(sanitized, "reasoning") extra_body: Final = sanitized.get("extra_body") if isinstance(extra_body, Mapping): sanitized_extra_body: Final = dict(extra_body) sanitized_extra_body.pop("reasoning_effort", None) sanitized_extra_body.pop("thinking", None) - Router._pop_effort_from_nested_carrier(sanitized_extra_body, "output_config") - Router._pop_effort_from_nested_carrier(sanitized_extra_body, "reasoning") + PythonCore._pop_effort_from_nested_carrier(sanitized_extra_body, "output_config") + PythonCore._pop_effort_from_nested_carrier(sanitized_extra_body, "reasoning") if sanitized_extra_body: sanitized["extra_body"] = sanitized_extra_body else: @@ -3476,14 +3518,14 @@ class Router(DeprecatedRouterMembers): **kwargs, } ) - response: Final = litellm.completion(**input_kwargs) + response: Final = self._host.invoke("completion")(**input_kwargs) verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) ## CHECK CONTENT FILTER ERROR ## if isinstance(response, ModelResponse): _should_raise = self._should_raise_content_policy_error(model=model, response=response, kwargs=kwargs) if _should_raise: - raise litellm.ContentPolicyViolationError( + raise litellm_exceptions.ContentPolicyViolationError( message="Response output was blocked.", model=model, llm_provider="", @@ -3772,7 +3814,7 @@ class Router(DeprecatedRouterMembers): fallback_response: object, ) -> tuple[dict[str, object], dict[str, object]]: """Repoint the wrapper at `fallback_response`, returning its prepared hidden params.""" - prepared: Final = Router._prepare_fallback_hidden_params(fallback_response) + prepared: Final = PythonCore._prepare_fallback_hidden_params(fallback_response) adopting_wrapper: Final = wrapper_ref() if adopting_wrapper is not None: adopting_wrapper.adopt_fallback_response_headers(fallback_response, prepared) @@ -3864,7 +3906,7 @@ class Router(DeprecatedRouterMembers): try: async for item in _achunks_with_first(model_response, first_chunk): yield item - except (MidStreamFallbackError, litellm.ContextWindowExceededError) as stream_error: + except (MidStreamFallbackError, litellm_exceptions.ContextWindowExceededError) as stream_error: e: Final = ( _context_window_stream_fallback_error( stream_error, @@ -3873,8 +3915,8 @@ class Router(DeprecatedRouterMembers): bool(getattr(model_response, "response_uptil_now", "")) or _stream_chunks_have_generated_content(model_response.chunks), ) - if isinstance(stream_error, litellm.ContextWindowExceededError) - or isinstance(stream_error.original_exception, litellm.ContextWindowExceededError) + if isinstance(stream_error, litellm_exceptions.ContextWindowExceededError) + or isinstance(stream_error.original_exception, litellm_exceptions.ContextWindowExceededError) else stream_error ) with anyio.CancelScope(shield=True): @@ -3929,7 +3971,7 @@ class Router(DeprecatedRouterMembers): else None ) cut_partial: Final = ( - Router._price_cut_partial(model_response, complete_response_object) + PythonCore._price_cut_partial(model_response, complete_response_object) if continuation_output is not None else None ) @@ -3949,7 +3991,7 @@ class Router(DeprecatedRouterMembers): self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs) fallback_response = await self.async_function_with_fallbacks_common_utils( e=e.original_exception - if isinstance(e.original_exception, litellm.ContextWindowExceededError) + if isinstance(e.original_exception, litellm_exceptions.ContextWindowExceededError) else e, disable_fallbacks=fallbacks_disabled_for_request(initial_kwargs), fallbacks=fallbacks, @@ -3963,8 +4005,8 @@ class Router(DeprecatedRouterMembers): # If fallback returns a streaming response, iterate over it if hasattr(fallback_response, "__aiter__"): - Router._carry_partial_spend_to_fallback(fallback_response, cut_partial) - prepared_fallback_hidden_params = Router._adopt_fallback_response_headers( + PythonCore._carry_partial_spend_to_fallback(fallback_response, cut_partial) + prepared_fallback_hidden_params = PythonCore._adopt_fallback_response_headers( wrapper_ref, fallback_response ) fallback_headers_are_settled = False @@ -3977,10 +4019,12 @@ class Router(DeprecatedRouterMembers): if not fallback_headers_are_settled: fallback_headers_are_settled = True # a fallback that failed over again only repoints itself once it yields - prepared_fallback_hidden_params = Router._adopt_fallback_response_headers( + prepared_fallback_hidden_params = PythonCore._adopt_fallback_response_headers( wrapper_ref, fallback_response ) - Router._apply_fallback_hidden_params_to_item(fallback_item, prepared_fallback_hidden_params) + PythonCore._apply_fallback_hidden_params_to_item( + fallback_item, prepared_fallback_hidden_params + ) if ( fallback_item and isinstance(fallback_item, ModelResponseStream) @@ -4185,7 +4229,7 @@ class Router(DeprecatedRouterMembers): return False if not e.generated_content or e.emitted_disqualifying_content: return False - if isinstance(e.original_exception, litellm.ContentPolicyViolationError): + if isinstance(e.original_exception, litellm_exceptions.ContentPolicyViolationError): return False cut_deployment: Final = ( self._core_get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None @@ -4210,7 +4254,7 @@ class Router(DeprecatedRouterMembers): and callback._event_hook_is_event_type( # pyright: ignore[reportPrivateUsage] # the only check of a guardrail's hooks GuardrailEventHooks.post_call ) - for callback in litellm.callbacks + for callback in self._host.callback_list("callbacks") ) @staticmethod @@ -4354,7 +4398,7 @@ class Router(DeprecatedRouterMembers): self._hidden_params = dict(getattr(source_iterator, "_hidden_params", None) or {}) def adopt_fallback_headers(self, fallback_response: object) -> tuple[dict[str, object], dict[str, object]]: - prepared: Final = Router._prepare_fallback_hidden_params(fallback_response) + prepared: Final = PythonCore._prepare_fallback_hidden_params(fallback_response) self._billing_source = fallback_response self._hidden_params = {**prepared[0], "additional_headers": prepared[1]} self.fallback_headers_adopted = True @@ -4375,7 +4419,7 @@ class Router(DeprecatedRouterMembers): if current is None or primary is None: return current or primary if primary.response.usage is not None: - Router._combine_responses_fallback_usage(current, primary.response.usage) + PythonCore._combine_responses_fallback_usage(current, primary.response.usage) return current def __aiter__(self): @@ -4439,7 +4483,7 @@ class Router(DeprecatedRouterMembers): yield item for held_event in held_lifecycle_events: yield held_event - except (MidStreamFallbackError, litellm.ContextWindowExceededError) as stream_error: + except (MidStreamFallbackError, litellm_exceptions.ContextWindowExceededError) as stream_error: e: Final = ( _context_window_stream_fallback_error( stream_error, @@ -4447,8 +4491,8 @@ class Router(DeprecatedRouterMembers): source_iterator.custom_llm_provider or "", wrapper.yielded_first_chunk, ) - if isinstance(stream_error, litellm.ContextWindowExceededError) - or isinstance(stream_error.original_exception, litellm.ContextWindowExceededError) + if isinstance(stream_error, litellm_exceptions.ContextWindowExceededError) + or isinstance(stream_error.original_exception, litellm_exceptions.ContextWindowExceededError) else stream_error ) if response_id_exposed: @@ -4502,8 +4546,8 @@ class Router(DeprecatedRouterMembers): original: Final = e.original_exception model_group: Final = initial_kwargs.get("model") if ( - litellm.expose_router_debug_in_errors - and isinstance(original, litellm.ContentPolicyViolationError) + self._host.dynamic().expose_router_debug_in_errors + and isinstance(original, litellm_exceptions.ContentPolicyViolationError) and isinstance(model_group, str) and not fallbacks_disabled_for_request(initial_kwargs) and _mid_stream_error_before_output(e) @@ -4526,7 +4570,7 @@ class Router(DeprecatedRouterMembers): ), isinstance(hop_depth, int) and hop_depth > 0, ) - Router._raise_original_mid_stream_error(e) + PythonCore._raise_original_mid_stream_error(e) def _responses_continuation( self, @@ -4539,14 +4583,16 @@ class Router(DeprecatedRouterMembers): tool call would not parse, and when a content filter cut it, so no other model finishes filtered text.""" if _mid_stream_error_before_output(e): return None - if source_iterator.started_tool_call or isinstance(e.original_exception, litellm.ContentPolicyViolationError): - Router._raise_original_mid_stream_error(e) + if source_iterator.started_tool_call or isinstance( + e.original_exception, litellm_exceptions.ContentPolicyViolationError + ): + PythonCore._raise_original_mid_stream_error(e) cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id") cut_deployment: Final = ( self._core_get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None ) if constrains_responses_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)): - Router._raise_original_mid_stream_error(e) + PythonCore._raise_original_mid_stream_error(e) earlier: Final = request_kwargs.get(RESPONSES_CONTINUATION_KWARG) if isinstance(earlier, ResponsesContinuation): return earlier.extended(e.generated_content) @@ -4561,7 +4607,7 @@ class Router(DeprecatedRouterMembers): emitted_tokens: Final = token_counter(model=e.model or "", text=e.generated_content) reduced: Final = continuation_output_ceilings(request_kwargs, emitted_tokens) if reduced is None: - Router._raise_original_mid_stream_error(e) + PythonCore._raise_original_mid_stream_error(e) return reduced async def _aresponses_fallback_attempt( @@ -4584,10 +4630,10 @@ class Router(DeprecatedRouterMembers): continuation: Final = self._responses_continuation(e, source_iterator, initial_kwargs) reduced_ceilings: Final[Mapping[str, object]] = ( - Router._responses_continuation_ceilings(e, initial_kwargs) if continuation is not None else {} + PythonCore._responses_continuation_ceilings(e, initial_kwargs) if continuation is not None else {} ) seam: Final = ResponsesContinuationSeam(continuation.emitted_text if continuation is not None else "") - partial_usage: Final = Router._extract_partial_responses_usage(source_iterator) + partial_usage: Final = PythonCore._extract_partial_responses_usage(source_iterator) fallback_response = None # rebind-ok: pre-init so finally can close it if a fallback was actually attempted fallback_yielded = False # rebind-ok: flipped on the first fallback item so a fallback that dies before its first event still replays the primary's held announcement try: @@ -4623,7 +4669,8 @@ class Router(DeprecatedRouterMembers): fallback_trigger: Final[Exception] = ( e.original_exception if isinstance( - e.original_exception, (litellm.ContentPolicyViolationError, litellm.ContextWindowExceededError) + e.original_exception, + (litellm_exceptions.ContentPolicyViolationError, litellm_exceptions.ContextWindowExceededError), ) else e ) @@ -4641,9 +4688,9 @@ class Router(DeprecatedRouterMembers): prepared_fallback_hidden_params: Final = adopt_headers(fallback_response) if hasattr(fallback_response, "__aiter__"): async for fallback_item in seam.splice(fallback_response): - Router._apply_fallback_hidden_params_to_item(fallback_item, prepared_fallback_hidden_params) + PythonCore._apply_fallback_hidden_params_to_item(fallback_item, prepared_fallback_hidden_params) if partial_usage is not None: - Router._combine_responses_fallback_usage(fallback_item, partial_usage) + PythonCore._combine_responses_fallback_usage(fallback_item, partial_usage) fallback_yielded = True yield fallback_item else: @@ -4711,7 +4758,7 @@ class Router(DeprecatedRouterMembers): try: for item in _chunks_with_first(model_response, first_chunk): yield item - except (MidStreamFallbackError, litellm.ContextWindowExceededError) as stream_error: + except (MidStreamFallbackError, litellm_exceptions.ContextWindowExceededError) as stream_error: e: Final = ( _context_window_stream_fallback_error( stream_error, @@ -4720,8 +4767,8 @@ class Router(DeprecatedRouterMembers): bool(getattr(model_response, "response_uptil_now", "")) or _stream_chunks_have_generated_content(model_response.chunks), ) - if isinstance(stream_error, litellm.ContextWindowExceededError) - or isinstance(stream_error.original_exception, litellm.ContextWindowExceededError) + if isinstance(stream_error, litellm_exceptions.ContextWindowExceededError) + or isinstance(stream_error.original_exception, litellm_exceptions.ContextWindowExceededError) else stream_error ) if fallbacks_disabled_for_request(initial_kwargs) or ( @@ -4749,7 +4796,7 @@ class Router(DeprecatedRouterMembers): fallback_response = run_async_function( router_self.async_function_with_fallbacks_common_utils, e=e.original_exception - if isinstance(e.original_exception, litellm.ContextWindowExceededError) + if isinstance(e.original_exception, litellm_exceptions.ContextWindowExceededError) else e, disable_fallbacks=fallbacks_disabled_for_request(initial_kwargs), fallbacks=fallbacks, @@ -4762,7 +4809,7 @@ class Router(DeprecatedRouterMembers): ) if hasattr(fallback_response, "__iter__"): - prepared_fallback_hidden_params = Router._adopt_fallback_response_headers( + prepared_fallback_hidden_params = PythonCore._adopt_fallback_response_headers( wrapper_ref, fallback_response ) fallback_headers_are_settled = False @@ -4770,10 +4817,12 @@ class Router(DeprecatedRouterMembers): if not fallback_headers_are_settled: fallback_headers_are_settled = True # a fallback that failed over again only repoints itself once it yields - prepared_fallback_hidden_params = Router._adopt_fallback_response_headers( + prepared_fallback_hidden_params = PythonCore._adopt_fallback_response_headers( wrapper_ref, fallback_response ) - Router._apply_fallback_hidden_params_to_item(fallback_item, prepared_fallback_hidden_params) + PythonCore._apply_fallback_hidden_params_to_item( + fallback_item, prepared_fallback_hidden_params + ) if ( fallback_item and isinstance(fallback_item, ModelResponseStream) @@ -4943,7 +4992,7 @@ class Router(DeprecatedRouterMembers): logging_obj=logging_obj, parent_otel_span=parent_otel_span, ) - response = await litellm.acompletion(**compacted_input) + response = await self._host.invoke("acompletion")(**compacted_input) ## CHECK CONTENT FILTER ERROR ## if isinstance(response, ModelResponse): @@ -4951,7 +5000,7 @@ class Router(DeprecatedRouterMembers): model=model, response=response, kwargs=kwargs ) if _should_raise: - raise litellm.ContentPolicyViolationError( + raise litellm_exceptions.ContentPolicyViolationError( message="Response output was blocked.", model=model, llm_provider="", @@ -4986,11 +5035,11 @@ class Router(DeprecatedRouterMembers): ) return response - except litellm.Timeout as e: + except litellm_exceptions.Timeout as e: deployment_litellm_params: Final[Mapping[str, object]] = _timeout_debug_deployment_dict.get( "litellm_params", {} ) - if litellm.expose_router_debug_in_errors: + if self._host.dynamic().expose_router_debug_in_errors: e.message += ( f"\n\nDeployment Info: request_timeout: {deployment_litellm_params.get('request_timeout')}" f"\ntimeout: {deployment_litellm_params.get('timeout')}" @@ -5090,7 +5139,9 @@ class Router(DeprecatedRouterMembers): return model_info: Final = _MODEL_INFO_ADAPTER.validate_python(effective_model_info) configured_deployment_id: Final = ( - model_info.get("original_model_id") if isinstance(exception, litellm.UnsupportedEndpointError) else None + model_info.get("original_model_id") + if isinstance(exception, litellm_exceptions.UnsupportedEndpointError) + else None ) deployment_id: Final = configured_deployment_id or model_info.get("id") if isinstance(deployment_id, str) and deployment_id: @@ -5149,7 +5200,7 @@ class Router(DeprecatedRouterMembers): litellm_params=LiteLLM_Params.model_validate(dynamic_litellm_params), model_info=model_info, ) - Router._register_deployment_pricing(deployment=deployment_pydantic_obj) + PythonCore._register_deployment_pricing(deployment=deployment_pydantic_obj) return deployment_pydantic_obj @staticmethod @@ -5319,7 +5370,7 @@ class Router(DeprecatedRouterMembers): and isinstance(caller_credential_name, str) and caller_credential_name != configured_per_user_name ): - raise litellm.BadRequestError( + raise litellm_exceptions.BadRequestError( message="litellm_credential_name cannot be overridden on a deployment that uses per-user GitHub OAuth", model=deployment_model_name, llm_provider="", @@ -5804,7 +5855,7 @@ class Router(DeprecatedRouterMembers): return False split_litellm_model: Final = litellm_model.split("/")[0] - return split_litellm_model in litellm._known_custom_logger_compatible_callbacks + return split_litellm_model in self._host.dynamic().known_custom_logger_compatible_callbacks def _pinned_deployment(self, model: str) -> Deployment | None: return self._core_get_deployment(model_id=model) if model not in self._core_model_names else None @@ -5919,7 +5970,7 @@ class Router(DeprecatedRouterMembers): _model_list: Final = self._core_get_model_list(model_name=model) if _model_list is None or len(_model_list) == 0: # if direct call to model kwargs.pop("original_function") - return await litellm.acompletion(**kwargs) + return await self._host.invoke("acompletion")(**kwargs) target_group_timeouts: Final = { key: value @@ -5972,7 +6023,7 @@ class Router(DeprecatedRouterMembers): ### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit) self.routing_strategy_pre_call_checks(deployment=deployment) - response: Final = litellm.image_generation( + response: Final = self._host.invoke("image_generation")( **{ **data, "prompt": prompt, @@ -6061,7 +6112,7 @@ class Router(DeprecatedRouterMembers): self.total_calls[model_name] += 1 async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await litellm.aimage_generation( + response = await self._host.invoke("aimage_generation")( **{ **data, "prompt": prompt, @@ -6144,7 +6195,7 @@ class Router(DeprecatedRouterMembers): self.total_calls[model_name] += 1 async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await litellm.atranscription( + response = await self._host.invoke("atranscription")( **{ **data, "file": file, @@ -6237,7 +6288,7 @@ class Router(DeprecatedRouterMembers): self.total_calls[model_name] += 1 async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await litellm.aspeech( + response = await self._host.invoke("aspeech")( **{ **data, "input": input, @@ -6297,7 +6348,7 @@ class Router(DeprecatedRouterMembers): ) self.total_calls[model_name] += 1 - response: Final = await litellm.arerank( + response: Final = await self._host.invoke("arerank")( **{ **data, "caching": self._core_cache_responses, @@ -6352,7 +6403,7 @@ class Router(DeprecatedRouterMembers): kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults # call via litellm.completion() - return litellm.text_completion( + return self._host.invoke("text_completion")( **{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs} ) except Exception as e: @@ -6416,7 +6467,7 @@ class Router(DeprecatedRouterMembers): self.total_calls[model_name] += 1 async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await litellm.atext_completion( + response = await self._host.invoke("atext_completion")( **{ **data, "prompt": prompt, @@ -6486,7 +6537,7 @@ class Router(DeprecatedRouterMembers): self.total_calls[model_name] += 1 async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await litellm.aadapter_completion( + response = await self._host.invoke("aadapter_completion")( **{ **data, "adapter_id": adapter_id, @@ -7223,7 +7274,7 @@ class Router(DeprecatedRouterMembers): return _StreamRetriesExhausted(last_error) if get_metadata_variable_name_from_kwargs(retry_kwargs) in retry_kwargs: self._core_log_retry(kwargs=retry_kwargs, e=trigger) - await asyncio.sleep( + await self._host.sleep( self._time_to_sleep_before_retry( e=trigger, remaining_retries=budget - attempt, @@ -7267,7 +7318,7 @@ class Router(DeprecatedRouterMembers): anthropic_messages_response_as_sse_events, ) - hidden_params, headers = Router._prepare_fallback_hidden_params(recovered) + hidden_params, headers = PythonCore._prepare_fallback_hidden_params(recovered) wrapper.merge_fallback_hidden_params(hidden_params, headers) wrapper.adopt_fallback_source(recovered) if retry_counters is not None: @@ -7330,7 +7381,8 @@ class Router(DeprecatedRouterMembers): fallback_trigger: Final[Exception] = ( e.original_exception if isinstance( - e.original_exception, (litellm.ContentPolicyViolationError, litellm.ContextWindowExceededError) + e.original_exception, + (litellm_exceptions.ContentPolicyViolationError, litellm_exceptions.ContextWindowExceededError), ) else e ) @@ -7455,7 +7507,9 @@ class Router(DeprecatedRouterMembers): """ deployment_id: Final = _video_call_deployment_id.get() creating: Final = tuple( - deployment for deployment in healthy_deployments if deployment_id in Router._deployment_ids((deployment,)) + deployment + for deployment in healthy_deployments + if deployment_id in PythonCore._deployment_ids((deployment,)) ) return creating or healthy_deployments @@ -7589,7 +7643,7 @@ class Router(DeprecatedRouterMembers): ### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit) self.routing_strategy_pre_call_checks(deployment=deployment) - response: Final = litellm.embedding( + response: Final = self._host.invoke("embedding")( **{ **data, "input": input, @@ -7668,7 +7722,7 @@ class Router(DeprecatedRouterMembers): self.total_calls[model_name] += 1 async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await litellm.aembedding( + response = await self._host.invoke("aembedding")( **{ **data, "input": input, @@ -7790,7 +7844,7 @@ class Router(DeprecatedRouterMembers): async with self._deployment_slot( deployment=deployment, kwargs=kwargs_copy, parent_otel_span=parent_otel_span ): - response = await litellm.acreate_file( + response = await self._host.invoke("acreate_file")( **{ **data, "custom_llm_provider": custom_llm_provider, @@ -7987,7 +8041,7 @@ class Router(DeprecatedRouterMembers): custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await litellm.acreate_batch( + response = await self._host.invoke("acreate_batch")( **{ **data, "custom_llm_provider": custom_llm_provider, @@ -8065,7 +8119,7 @@ class Router(DeprecatedRouterMembers): new_kwargs[metadata_variable_name]["model_group"] = model_group new_kwargs.pop("custom_llm_provider", None) data.pop("custom_llm_provider", None) - return await litellm.aretrieve_batch( + return await self._host.invoke("aretrieve_batch")( **{ **data, "custom_llm_provider": custom_llm_provider, @@ -8192,7 +8246,7 @@ class Router(DeprecatedRouterMembers): custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await litellm.acancel_batch( + response = await self._host.invoke("acancel_batch")( **{ **data, "custom_llm_provider": custom_llm_provider, @@ -8231,7 +8285,7 @@ class Router(DeprecatedRouterMembers): async def try_retrieve_batch(model: DeploymentTypedDict): try: # Update kwargs with the current model name or any other model-specific adjustments - return await litellm.alist_batches(**{**model["litellm_params"], **kwargs}) + return await self._host.invoke("alist_batches")(**{**model["litellm_params"], **kwargs}) except Exception: return None @@ -8474,7 +8528,7 @@ class Router(DeprecatedRouterMembers): { **provider_kwargs, "_direct_vector_store_embedding_executor": RouterVectorStoreEmbeddingExecutor( - router=self, + router=self._core_public(), metadata=self._vector_store_request_metadata(kwargs), ), } @@ -8485,7 +8539,7 @@ class Router(DeprecatedRouterMembers): if search_kwargs.get("model"): return self._generic_api_call_with_fallbacks(original_function=original_function, **search_kwargs) if call_type == "vector_store_search": - return original_function(**MappingProxyType({**search_kwargs, "router": self})) + return original_function(**MappingProxyType({**search_kwargs, "router": self._core_public()})) return original_function(**search_kwargs) return vector_store_sync_wrapper @@ -8666,7 +8720,7 @@ class Router(DeprecatedRouterMembers): { **kwargs, "_direct_vector_store_embedding_executor": RouterVectorStoreEmbeddingExecutor( - router=self, + router=self._core_public(), metadata=self._vector_store_request_metadata(kwargs), ), } @@ -8744,7 +8798,7 @@ class Router(DeprecatedRouterMembers): # router-managed embedding models (e.g. S3 Vectors query embeddings). # The merge also overrides any client-supplied `router` key. if call_type == "avector_store_search": - search_kwargs: Final = MappingProxyType({**kwargs, "router": self}) + search_kwargs: Final = MappingProxyType({**kwargs, "router": self._core_public()}) return await original_function(**search_kwargs) # Otherwise, call the original function directly @@ -9030,7 +9084,7 @@ class Router(DeprecatedRouterMembers): # Skip for error types that have their own dedicated fallback handlers _skip_order_fallback: Final = isinstance( e, - (litellm.ContextWindowExceededError, litellm.ContentPolicyViolationError), + (litellm_exceptions.ContextWindowExceededError, litellm_exceptions.ContentPolicyViolationError), ) _request_team_id: Final[str | None] = (kwargs.get("metadata", {}) or {}).get("user_api_key_team_id") # Use wildcard-aware lookup so order-based fallback also works for model @@ -9038,9 +9092,9 @@ class Router(DeprecatedRouterMembers): order_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group all_deployments: Final = self._core_get_model_list(model_name=order_model_group, team_id=_request_team_id) or () _order_set: Final[set] = { - litellm.utils.get_deployment_order(d) + litellm_utils.get_deployment_order(d) for d in all_deployments - if litellm.utils.get_deployment_order(d) is not None + if litellm_utils.get_deployment_order(d) is not None } order_values: Final[list] = sorted(_order_set) if len(order_values) > 1 and not _skip_order_fallback: @@ -9116,7 +9170,7 @@ class Router(DeprecatedRouterMembers): return response - if isinstance(e, litellm.ContextWindowExceededError): + if isinstance(e, litellm_exceptions.ContextWindowExceededError): if context_window_fallbacks is not None: context_window_fallback_model_group: Final[list[str] | None] = ( self._get_fallback_model_group_for_lookup_groups( @@ -9147,9 +9201,9 @@ class Router(DeprecatedRouterMembers): to fallbacks, if available.{error_message}" ) - if litellm.expose_router_debug_in_errors: + if self._host.dynamic().expose_router_debug_in_errors: e.message += f"\n{error_message}" - elif isinstance(e, litellm.ContentPolicyViolationError): + elif isinstance(e, litellm_exceptions.ContentPolicyViolationError): if content_policy_fallbacks is not None: content_policy_fallback_model_group: Final[list[str] | None] = ( self._get_fallback_model_group_for_lookup_groups( @@ -9181,7 +9235,7 @@ class Router(DeprecatedRouterMembers): to fallbacks, if available.{error_message}" ) - if litellm.expose_router_debug_in_errors: + if self._host.dynamic().expose_router_debug_in_errors: e.message += f"\n{error_message}" if fallbacks is not None and lookup_groups: verbose_router_logger.debug("inside model fallbacks: %s", mask_sensitive_structure(fallbacks)) @@ -9205,7 +9259,7 @@ class Router(DeprecatedRouterMembers): ) if ( hasattr(original_exception, "message") - and litellm.expose_router_debug_in_errors + and self._host.dynamic().expose_router_debug_in_errors and not nested_fallback_hop ): original_exception.message += format_no_fallback_group_message(lookup_groups, fallbacks) @@ -9243,7 +9297,7 @@ class Router(DeprecatedRouterMembers): attempted_fallback_group: Final = input_kwargs.get("fallback_model_group") if ( hasattr(original_exception, "message") - and litellm.expose_router_debug_in_errors + and self._host.dynamic().expose_router_debug_in_errors and not no_fallback_group_explained and not nested_fallback_hop ): @@ -9344,7 +9398,7 @@ class Router(DeprecatedRouterMembers): Raises: litellm.InternalServerError: when `mock_testing_fallbacks=True` passed in request params - litellm.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params + litellm_exceptions.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params litellm.ContentPolicyViolationError: when `mock_testing_content_policy_fallbacks=True` passed in request params """ mock_testing_params: Final = MockRouterTestingParams.from_kwargs(kwargs) @@ -9352,7 +9406,7 @@ class Router(DeprecatedRouterMembers): mock_testing_params.mock_testing_fallbacks is not None and mock_testing_params.mock_testing_fallbacks is True ): - raise litellm.InternalServerError( + raise litellm_exceptions.InternalServerError( model=model_group, llm_provider="", message=f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}", @@ -9361,7 +9415,7 @@ class Router(DeprecatedRouterMembers): mock_testing_params.mock_testing_context_fallbacks is not None and mock_testing_params.mock_testing_context_fallbacks is True ): - raise litellm.ContextWindowExceededError( + raise litellm_exceptions.ContextWindowExceededError( model=model_group, llm_provider="", message=f"This is a mock exception for model={model_group}, to trigger a fallback. \ @@ -9371,7 +9425,7 @@ class Router(DeprecatedRouterMembers): mock_testing_params.mock_testing_content_policy_fallbacks is not None and mock_testing_params.mock_testing_content_policy_fallbacks is True ): - raise litellm.ContentPolicyViolationError( + raise litellm_exceptions.ContentPolicyViolationError( model=model_group, llm_provider="", message=f"This is a mock exception for model={model_group}, to trigger a fallback. \ @@ -9386,7 +9440,7 @@ class Router(DeprecatedRouterMembers): status_code: Final = getattr(exception, "status_code", None) if not failed_deployment_id or not isinstance(status_code, int): return () - if litellm.should_retry(status_code): + if python_host.invoke("should_retry")(status_code): return () already_skipped_ids: Final = as_retry_skipped_deployment_ids(already_skipped) skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id)))) @@ -9502,7 +9556,9 @@ class Router(DeprecatedRouterMembers): ) if first_skipped_ids: kwargs["_retry_skipped_deployment_ids"] = first_skipped_ids # rebind-ok: the next attempt reads it - if isinstance(original_exception, litellm.UnsupportedEndpointError) and _every_deployment_refused( + if isinstance( + original_exception, litellm_exceptions.UnsupportedEndpointError + ) and _every_deployment_refused( _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), first_skipped_ids ): raise @@ -9519,7 +9575,7 @@ class Router(DeprecatedRouterMembers): all_deployments=_all_deployments, ) - await asyncio.sleep(retry_after) + await self._host.sleep(retry_after) for current_attempt in range(num_retries): try: @@ -9585,7 +9641,7 @@ class Router(DeprecatedRouterMembers): ) if skipped_ids: kwargs["_retry_skipped_deployment_ids"] = skipped_ids # rebind-ok: the next attempt reads it - if isinstance(e, litellm.UnsupportedEndpointError) and _every_deployment_refused( + if isinstance(e, litellm_exceptions.UnsupportedEndpointError) and _every_deployment_refused( _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), skipped_ids ): raise @@ -9596,9 +9652,9 @@ class Router(DeprecatedRouterMembers): healthy_deployments=_healthy_deployments, all_deployments=_all_deployments, ) - await asyncio.sleep(_timeout) + await self._host.sleep(_timeout) - if type(original_exception) in litellm.LITELLM_EXCEPTION_TYPES: + if type(original_exception) in self._host.dynamic().exception_types: setattr(original_exception, "max_retries", num_retries) # current_attempt is 0-indexed (0 to num_retries-1), so after loop completion # it represents the last attempt index. The actual number of retries attempted @@ -9631,7 +9687,7 @@ class Router(DeprecatedRouterMembers): except Exception: with contextlib.suppress(Exception): await async_io_token_refund_failure( - self.cache, kwargs, parent_otel_span=get_parent_otel_span_from_kwargs(kwargs) + self._core_cache, kwargs, parent_otel_span=get_parent_otel_span_from_kwargs(kwargs) ) raise await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs) @@ -9661,7 +9717,7 @@ class Router(DeprecatedRouterMembers): verbose_router_logger.info( "litellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=%s", model_group ) - raise litellm.RateLimitError( + raise litellm_exceptions.RateLimitError( model=model_group, llm_provider="", message=f"This is a mock exception for model={model_group}, to trigger a rate limit error.", @@ -9694,19 +9750,19 @@ class Router(DeprecatedRouterMembers): _num_all_deployments = len(all_deployments) ### CHECK IF RATE LIMIT / CONTEXT WINDOW ERROR / CONTENT POLICY VIOLATION ERROR w/ fallbacks available / Bad Request Error - if isinstance(error, litellm.ContextWindowExceededError) and context_window_fallbacks is not None: + if isinstance(error, litellm_exceptions.ContextWindowExceededError) and context_window_fallbacks is not None: raise error - if isinstance(error, litellm.ContentPolicyViolationError) and content_policy_fallbacks is not None: + if isinstance(error, litellm_exceptions.ContentPolicyViolationError) and content_policy_fallbacks is not None: raise error status_code: Final = getattr(error, "status_code", None) - if status_code is not None and not litellm.should_retry(status_code): + if status_code is not None and not self._host.invoke("should_retry")(status_code): # 401/403 are special cases - allow retry if multiple deployments exist (handled below) - if status_code not in (401, 403) and not isinstance(error, litellm.UnsupportedEndpointError): + if status_code not in (401, 403) and not isinstance(error, litellm_exceptions.UnsupportedEndpointError): raise error - if isinstance(error, litellm.NotFoundError): + if isinstance(error, litellm_exceptions.NotFoundError): raise error # Error we should only retry if there are other deployments if isinstance(error, openai.RateLimitError): @@ -9792,7 +9848,7 @@ class Router(DeprecatedRouterMembers): response_headers = e.litellm_response_headers if response_headers is not None: - timeout = litellm.calculate_retry_after( + timeout = self._host.invoke("calculate_retry_after")( remaining_retries=remaining_retries, max_retries=num_retries, response_headers=response_headers, @@ -9800,7 +9856,7 @@ class Router(DeprecatedRouterMembers): ) else: - timeout = litellm.calculate_retry_after( + timeout = self._host.invoke("calculate_retry_after")( remaining_retries=remaining_retries, max_retries=num_retries, min_timeout=self.retry_after, @@ -10109,9 +10165,7 @@ class Router(DeprecatedRouterMembers): ) return False - exception_headers: Final = litellm.litellm_core_utils.exception_mapping_utils._get_response_headers( - original_exception=exception - ) + exception_headers: Final = exception_mapping_utils._get_response_headers(original_exception=exception) # Determine cooldown time with priority: deployment config > response header > router default deployment_cooldown: Final = _first_present( @@ -10120,7 +10174,7 @@ class Router(DeprecatedRouterMembers): header_cooldown = None if exception_headers is not None: - header_cooldown = litellm.utils._get_retry_after_from_exception_header( + header_cooldown = litellm_utils._get_retry_after_from_exception_header( response_headers=exception_headers ) ############################################## @@ -10368,9 +10422,9 @@ class Router(DeprecatedRouterMembers): return tuple( sorted( { - litellm.utils.get_deployment_order(d) + litellm_utils.get_deployment_order(d) for d in all_deployments - if litellm.utils.get_deployment_order(d) is not None + if litellm_utils.get_deployment_order(d) is not None } ) ) @@ -10526,7 +10580,7 @@ class Router(DeprecatedRouterMembers): if isinstance(_callback, CustomLogger): try: await _callback.async_pre_call_check(deployment, parent_otel_span) - except litellm.RateLimitError as e: + except litellm_exceptions.RateLimitError as e: ## LOG FAILURE EVENT if logging_obj is not None: asyncio.create_task( @@ -10589,11 +10643,11 @@ class Router(DeprecatedRouterMembers): request_kwargs=dict(request_kwargs), ) except ValueError as e: - raise litellm.RateLimitError( + raise litellm_exceptions.RateLimitError( message=str(e), llm_provider="", model=model_group, - category=litellm.RateLimitErrorCategory.LITELLM_RATE_LIMIT, + category=litellm_exceptions.RateLimitErrorCategory.LITELLM_RATE_LIMIT, ) from e await self._admit_turn_through_routing_strategy(model_group, deployment, request_kwargs) await self.async_routing_strategy_pre_call_checks( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict @@ -10623,11 +10677,11 @@ class Router(DeprecatedRouterMembers): no_deployment: Final = await async_raise_no_deployment_exception( ctx=self._core_ctx, model=model_group, parent_otel_span=None ) - raise litellm.RateLimitError( + raise litellm_exceptions.RateLimitError( message=str(no_deployment), llm_provider="", model=model_group, - category=litellm.RateLimitErrorCategory.LITELLM_RATE_LIMIT, + category=litellm_exceptions.RateLimitErrorCategory.LITELLM_RATE_LIMIT, ) from no_deployment await self._async_override_selector_pre_call_check( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict strategy, selector, dict(deployment), None @@ -10734,7 +10788,9 @@ class Router(DeprecatedRouterMembers): if not missing_fields: return try: - backend_info: Final = litellm.get_model_info(model=backend_model, custom_llm_provider=custom_llm_provider) + backend_info: Final = python_host.invoke("get_model_info")( + model=backend_model, custom_llm_provider=custom_llm_provider + ) except Exception: return for field in missing_fields: @@ -10788,10 +10844,12 @@ class Router(DeprecatedRouterMembers): if all(model_info.get(field) is None for field in ("input_cost_per_token", "output_cost_per_token")): return try: - backend_info: Final = litellm.get_model_info(model=backend_model, custom_llm_provider=custom_llm_provider) + backend_info: Final = python_host.invoke("get_model_info")( + model=backend_model, custom_llm_provider=custom_llm_provider + ) except Exception: # noqa: BLE001 # get_model_info raises plain Exception for an unmapped backend model return - backend_entry: Final = litellm.model_cost.get(backend_info.get("key") or "") + backend_entry: Final = python_host.dynamic().model_cost.get(backend_info.get("key") or "") if not isinstance(backend_entry, dict): return for field, backend_value in backend_entry.items(): @@ -10836,10 +10894,12 @@ class Router(DeprecatedRouterMembers): ): return try: - backend_info: Final = litellm.get_model_info(model=backend_model, custom_llm_provider=custom_llm_provider) + backend_info: Final = python_host.invoke("get_model_info")( + model=backend_model, custom_llm_provider=custom_llm_provider + ) except Exception: # noqa: BLE001 # get_model_info raises plain Exception for an unmapped backend model return - backend_entry: Final = litellm.model_cost.get(backend_info.get("key") or "") + backend_entry: Final = python_host.dynamic().model_cost.get(backend_info.get("key") or "") if not isinstance(backend_entry, dict): return for field, backend_value in backend_entry.items(): @@ -10873,7 +10933,9 @@ class Router(DeprecatedRouterMembers): if all(isinstance(tier, dict) and "output_cost_per_token" in tier for tier in tiers): return try: - backend_info: Final = litellm.get_model_info(model=backend_model, custom_llm_provider=custom_llm_provider) + backend_info: Final = python_host.invoke("get_model_info")( + model=backend_model, custom_llm_provider=custom_llm_provider + ) except Exception: # noqa: BLE001 # get_model_info raises plain Exception for an unmapped backend model return backend_rate: Final = backend_info.get("output_cost_per_token") @@ -10952,7 +11014,7 @@ class Router(DeprecatedRouterMembers): model_info=_model_info, ) ) - Router._register_deployment_pricing(deployment) + PythonCore._register_deployment_pricing(deployment) ## Check if LLM Deployment is allowed for this deployment if self.deployment_is_active_for_environment(deployment=deployment) is not True: @@ -10982,7 +11044,7 @@ class Router(DeprecatedRouterMembers): return deployment except Exception as e: if self.ignore_invalid_deployments: - if isinstance(e, litellm.BadRequestError): + if isinstance(e, litellm_exceptions.BadRequestError): self._provider_unresolved_deployments = ( *self._provider_unresolved_deployments, partial( @@ -11210,11 +11272,11 @@ class Router(DeprecatedRouterMembers): include=STRATEGY_ROUTER_PARAM_FIELDS | {"model", "tags"}, exclude_none=True, warnings=False ), sort_keys=True, - default=Router._json_default_stable_id, + default=PythonCore._json_default_stable_id, ) except (TypeError, ValueError): return None - return Router._core_generate_model_id(deployment.model_name, {"routing_definition": definition}) + return PythonCore._core_generate_model_id(deployment.model_name, {"routing_definition": definition}) @staticmethod def _unregister_pre_routing_strategy( @@ -11315,15 +11377,13 @@ class Router(DeprecatedRouterMembers): AdaptiveRouterResponseHeaderHook, ) - for callback in litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook): - litellm.logging_callback_manager.remove_callback_from_all_lists(callback) + for callback in self._host.callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook): + self._host.callbacks.remove_callback_from_all_lists(callback) for tagged_adaptive_routers in self._core_adaptive_routers.values(): for tagged in tagged_adaptive_routers: - litellm.logging_callback_manager.add_litellm_callback( - AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy) - ) + self._host.callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)) if self._core_adaptive_routers: - litellm.logging_callback_manager.add_litellm_callback(AdaptiveRouterResponseHeaderHook()) + self._host.callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook()) def init_adaptive_router_deployment(self, deployment: Deployment) -> None: """ @@ -11387,9 +11447,7 @@ class Router(DeprecatedRouterMembers): strategy=adaptive_router, strategy_label="Adaptive-router", ) - litellm.logging_callback_manager.add_litellm_callback( - AdaptiveRouterPostCallHook(adaptive_router=adaptive_router) - ) + self._host.callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=adaptive_router)) verbose_router_logger.info( "AdaptiveRouter[%s] initialized with %d models", deployment.model_name, @@ -11567,7 +11625,7 @@ class Router(DeprecatedRouterMembers): if "/" in litellm_model: split_litellm_model: Final = litellm_model.split("/")[0] - if split_litellm_model in litellm._known_custom_logger_compatible_callbacks: + if split_litellm_model in self._host.dynamic().known_custom_logger_compatible_callbacks: is_prompt_management_model = True if is_prompt_management_model: @@ -11588,7 +11646,7 @@ class Router(DeprecatedRouterMembers): custom_llm_provider, dynamic_api_key, api_base, - ) = litellm.get_llm_provider( + ) = self._host.invoke("get_llm_provider")( model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.get("custom_llm_provider", None), api_base=deployment.litellm_params.api_base, @@ -11597,7 +11655,7 @@ class Router(DeprecatedRouterMembers): # done reading model["litellm_params"] # Check if provider is supported: either in enum or JSON-configured if ( - custom_llm_provider not in litellm.provider_list + custom_llm_provider not in self._host.dynamic().provider_list and not JSONProviderRegistry.exists(custom_llm_provider) and not is_registered_custom_provider(custom_llm_provider) ): @@ -11755,7 +11813,7 @@ class Router(DeprecatedRouterMembers): # initialize client self._add_deployment(deployment=served) - Router._register_deployment_pricing(deployment=served) + PythonCore._register_deployment_pricing(deployment=served) # add to model names self._add_model_to_list_and_index_map(model=_deployment, model_id=served.model_info.id) @@ -12003,7 +12061,7 @@ class Router(DeprecatedRouterMembers): field_value = deployment.litellm_params.get(field) if field_value is not None: model_info[field] = field_value - backend_model: Final = Router._cost_map_backend_model(deployment) + backend_model: Final = PythonCore._cost_map_backend_model(deployment) pricing_provider: Final = ( deployment.model_info.pricing_provider or deployment.litellm_params.custom_llm_provider ) @@ -12023,23 +12081,23 @@ class Router(DeprecatedRouterMembers): model_info.update( {key: value for key, value in priced_info.items() if is_server_derived_pricing_key(key)} ) - Router._inherit_builtin_base_rates_for_off_peak( + PythonCore._inherit_builtin_base_rates_for_off_peak( model_info=model_info, backend_model=backend_model, custom_llm_provider=pricing_provider, ) if model_info.get("input_cost_per_token") is not None: - Router._inherit_builtin_cache_pricing( + PythonCore._inherit_builtin_cache_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider=pricing_provider, ) - Router._inherit_builtin_service_tier_pricing( + PythonCore._inherit_builtin_service_tier_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider=pricing_provider, ) - Router._inherit_builtin_tiered_output_rate( + PythonCore._inherit_builtin_tiered_output_rate( model_info=model_info, backend_model=backend_model, custom_llm_provider=pricing_provider, @@ -12054,9 +12112,9 @@ class Router(DeprecatedRouterMembers): is never added to ``self._core_model_list`` (a per-request client-side-credential deployment) just as readily as one that is. """ - Router._register_deployment_in_model_cost( + PythonCore._register_deployment_in_model_cost( model_id=deployment.model_info.id, - model_info=Router._deployment_model_cost_payload(deployment), + model_info=PythonCore._deployment_model_cost_payload(deployment), model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, pricing_provider=deployment.model_info.pricing_provider, @@ -12089,7 +12147,7 @@ class Router(DeprecatedRouterMembers): model_info = {k: v for k, v in model_info.items() if k not in CustomPricingLiteLLMParams.model_fields} if model_id is not None: - litellm.register_model( + python_host.invoke("register_model")( model_cost={model_id: model_info}, persist_across_reloads=False, warning_display_name=model, @@ -12098,7 +12156,7 @@ class Router(DeprecatedRouterMembers): ) ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes - backend_keys: Final = Router._backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider) + backend_keys: Final = PythonCore._backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider) # For the shared backend key, keep only cost-map schema fields # (minus custom pricing) so that one deployment's pricing overrides @@ -12107,7 +12165,7 @@ class Router(DeprecatedRouterMembers): # name. Each deployment's full model_info is already stored under # its unique model_id above. shared_model_info: Final = shared_backend_model_info(model_info) - litellm.register_model( + python_host.invoke("register_model")( model_cost={_key: keep_registered_mode(_key, shared_model_info) for _key in backend_keys}, persist_across_reloads=False, ) @@ -12130,9 +12188,9 @@ class Router(DeprecatedRouterMembers): "Router: could not rebuild cost-map entry for a deployment during a price data reload" ) continue - Router._register_deployment_in_model_cost( + PythonCore._register_deployment_in_model_cost( model_id=deployment.model_info.id, - model_info=Router._deployment_model_cost_payload(deployment), + model_info=PythonCore._deployment_model_cost_payload(deployment), model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, pricing_provider=deployment.model_info.pricing_provider, @@ -12358,7 +12416,9 @@ class Router(DeprecatedRouterMembers): } ) ) - model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params) + model, provider, dynamic_api_key, api_base = self._host.invoke("get_llm_provider")( + model=params.model, litellm_params=params + ) if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: return if api_base is None or "*" in model or params.get("use_clientside_credentials"): @@ -12768,7 +12828,7 @@ class Router(DeprecatedRouterMembers): f"Got: {type(litellm_params_data).__name__}, " f"deployment_id: {(deployment.get('model_info') or {}).get('id', 'unknown')}" ) - _model, custom_llm_provider, _, _ = litellm.get_llm_provider( + _model, custom_llm_provider, _, _ = self._host.invoke("get_llm_provider")( model=litellm_params.model, litellm_params=litellm_params, ) @@ -12779,7 +12839,7 @@ class Router(DeprecatedRouterMembers): # litellm.model_cost as a zeroed stub, so membership alone can't # tell a resolvable name apart; require usable limits/costs. _azure_fallback_key = _model if _model.startswith("azure/") else f"azure/{_model}" - _fallback_entry = litellm.model_cost.get(_azure_fallback_key) + _fallback_entry = self._host.dynamic().model_cost.get(_azure_fallback_key) _fallback_resolves = _fallback_entry is not None and ( (_fallback_entry.get("max_input_tokens") or 0) > 0 or (_fallback_entry.get("input_cost_per_token") or 0) > 0 @@ -12822,7 +12882,7 @@ class Router(DeprecatedRouterMembers): else: model_info_name = model - model_info: Final = litellm.get_model_info(model=model_info_name) + model_info: Final = self._host.invoke("get_model_info")(model=model_info_name) if model_info is None: return model_info @@ -12894,17 +12954,17 @@ class Router(DeprecatedRouterMembers): try: custom_model_info = ( { - **copy.deepcopy(litellm.model_cost.get(model_id) or MappingProxyType({})), + **copy.deepcopy(self._host.dynamic().model_cost.get(model_id) or MappingProxyType({})), **self._core_get_discovered_model_info(model_id), } - if model_id in litellm.model_cost + if model_id in self._host.dynamic().model_cost else None ) except Exception: pass try: - litellm_model_name_model_info = litellm.get_model_info(model=model_name) + litellm_model_name_model_info = self._host.invoke("get_model_info")(model=model_name) except Exception: pass @@ -12914,7 +12974,7 @@ class Router(DeprecatedRouterMembers): base_model: Final = custom_model_info.get("base_model", None) if base_model is not None: ## update litellm model info with base model info - base_model_info: Final = copy.deepcopy(litellm.get_model_info(model=base_model)) + base_model_info: Final = copy.deepcopy(self._host.invoke("get_model_info")(model=base_model)) if base_model_info is not None: base_model_key = base_model_info.get("key") # Base model provides defaults, custom model info overrides @@ -12962,7 +13022,7 @@ class Router(DeprecatedRouterMembers): model=litellm_params.model, custom_llm_provider=litellm_params.custom_llm_provider, ) - except litellm.exceptions.BadRequestError as e: + except litellm_exceptions.BadRequestError as e: verbose_router_logger.error("litellm.router.py::get_model_group_info() - %s", e) return "", "" @@ -13063,7 +13123,7 @@ class Router(DeprecatedRouterMembers): litellm_model, llm_provider = self._model_group_llm_provider(litellm_params) if model_info is None: - supported_openai_params = litellm.get_supported_openai_params( + supported_openai_params = self._host.invoke("get_supported_openai_params")( model=litellm_model, custom_llm_provider=llm_provider ) if supported_openai_params is None: @@ -14169,13 +14229,13 @@ class Router(DeprecatedRouterMembers): deployment_params: Final = deployment.get("litellm_params") if not deployment_params: return True - if param in Router._declared_param_allowlist(deployment_params): + if param in PythonCore._declared_param_allowlist(deployment_params): return True if declared_authenticating_provider( str(deployment_params.get("model") or ""), deployment_params.get("custom_llm_provider") ): return True - return Router._deployment_supports_param(deployment, group, param) + return PythonCore._deployment_supports_param(deployment, group, param) @staticmethod def _capability_model_and_provider(deployment_params: LiteLLMParamsTypedDict, group: str) -> tuple[str, str]: @@ -14185,7 +14245,7 @@ class Router(DeprecatedRouterMembers): ) if authenticating_provider is not None: return model_label.removeprefix(f"{authenticating_provider}/"), authenticating_provider - model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model, custom_llm_provider, _, _ = python_host.invoke("get_llm_provider")( model=model_label, custom_llm_provider=deployment_params.get("custom_llm_provider"), litellm_params=LiteLLM_Params.model_validate(deployment_params), @@ -14202,8 +14262,8 @@ class Router(DeprecatedRouterMembers): deployment_model_info.get("base_model") if deployment_model_info else None ) or deployment_params.get("base_model") try: - model, custom_llm_provider = Router._capability_model_and_provider(deployment_params, group) - supported: Final = litellm.get_supported_openai_params( + model, custom_llm_provider = PythonCore._capability_model_and_provider(deployment_params, group) + supported: Final = python_host.invoke("get_supported_openai_params")( model=model, custom_llm_provider=custom_llm_provider, base_model=base_model if isinstance(base_model, str) else None, @@ -14702,7 +14762,7 @@ class Router(DeprecatedRouterMembers): system_message: Final = anthropic_system_to_openai_message(extras.get("system")) if messages is not None: counted_messages: Final = (system_message, *messages) if system_message is not None else messages - return litellm.token_counter(model=tokenizer_model, messages=counted_messages, tools=tools) + return self._host.invoke("token_counter")(model=tokenizer_model, messages=counted_messages, tools=tools) if input is not None: from openai.types.responses.response_create_params import ResponseInputParam @@ -14716,7 +14776,7 @@ class Router(DeprecatedRouterMembers): responses_api_request={"instructions": instructions} if instructions is not None else {}, only_since_compaction=True, ) - return litellm.token_counter( + return self._host.invoke("token_counter")( model=tokenizer_model, messages=cast(list, input_messages), # cast-ok: transformed chat messages tools=tools, @@ -14827,19 +14887,19 @@ class Router(DeprecatedRouterMembers): if isinstance(dropped, list) else frozenset() ) - return candidates - Router._declared_param_allowlist(params) - dropped_names + return candidates - PythonCore._declared_param_allowlist(params) - dropped_names @staticmethod def _deployment_lacks_params(deployment: DeploymentTypedDict, group: str, params: frozenset[str]) -> bool: - return any(not Router._deployment_supports_param(deployment, group, param) for param in params) + return any(not PythonCore._deployment_supports_param(deployment, group, param) for param in params) @staticmethod def _deployment_rejects_params( deployment: DeploymentTypedDict, group: str, params: frozenset[str], request_kwargs: Mapping[str, object] ) -> bool: call_settings: Final[Mapping[str, object]] = {**(deployment.get("litellm_params") or {}), **request_kwargs} - return Router._deployment_lacks_params( - deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params) + return PythonCore._deployment_lacks_params( + deployment, group, PythonCore._params_neither_dropped_nor_allowed(call_settings, params) ) @staticmethod @@ -14855,8 +14915,8 @@ class Router(DeprecatedRouterMembers): return False base_model: Final = model_info.get("base_model") or params.get("base_model") try: - model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group) - return litellm.utils.is_vision_explicitly_disabled( + model, custom_llm_provider = PythonCore._capability_model_and_provider(deployment["litellm_params"], group) + return litellm_utils.is_vision_explicitly_disabled( base_model if isinstance(base_model, str) else model, custom_llm_provider ) except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request @@ -14875,7 +14935,7 @@ class Router(DeprecatedRouterMembers): return True try: - model, resolved_custom_llm_provider = Router._capability_model_and_provider( + model, resolved_custom_llm_provider = PythonCore._capability_model_and_provider( deployment["litellm_params"], group ) provider: Final = next( @@ -14901,7 +14961,7 @@ class Router(DeprecatedRouterMembers): filtered_deployments: Final = [ deployment for deployment in healthy_deployments - if Router._deployment_serves_route( + if PythonCore._deployment_serves_route( cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape group, resolver, @@ -14918,7 +14978,7 @@ class Router(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, # mutable-ok: handed to _request_carries_image as-is request_kwargs: Mapping[str, object] | None, ) -> _RouteDeploymentsT: - if isinstance(healthy_deployments, Mapping) or not Router._request_carries_image( + if isinstance(healthy_deployments, Mapping) or not PythonCore._request_carries_image( messages, request_kwargs or {} ): return healthy_deployments @@ -14926,7 +14986,7 @@ class Router(DeprecatedRouterMembers): vision_capable_deployments: Final = [ deployment for deployment in healthy_deployments - if not Router._deployment_declares_no_vision( + if not PythonCore._deployment_declares_no_vision( cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape group, ) @@ -14951,7 +15011,7 @@ class Router(DeprecatedRouterMembers): @staticmethod def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]: - requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs)) + requested: Final = litellm_utils.get_non_default_params(passed_params=dict(request_kwargs)) return frozenset( param for param, value in requested.items() @@ -15008,7 +15068,7 @@ class Router(DeprecatedRouterMembers): request_settings: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else {} requested_params: Final[frozenset[str]] = ( self._pre_call_check_requested_chat_params(request_settings) - if request_kwargs is not None and litellm.drop_params is False + if request_kwargs is not None and self._host.dynamic().drop_params is False else frozenset() ) @@ -15083,7 +15143,7 @@ class Router(DeprecatedRouterMembers): for deployment in group_deployments ) ): - raise litellm.UnsupportedParamsError( + raise litellm_exceptions.UnsupportedParamsError( message=( f"no deployment of model={model} supports {sorted(requested_params)}. " "To drop these, set `litellm.drop_params=True` or for proxy `litellm_settings: drop_params: true`, " @@ -15092,7 +15152,7 @@ class Router(DeprecatedRouterMembers): model=model, ) if _context_window_error is True: - raise litellm.ContextWindowExceededError( + raise litellm_exceptions.ContextWindowExceededError( message=f"litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.\n{_potential_error_str}", model=model, llm_provider="", @@ -15128,7 +15188,7 @@ class Router(DeprecatedRouterMembers): try: with endpoint_selection(call_type, model=model): _, candidates = self._common_checks_available_deployment(model=model, request_kwargs=request_kwargs) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped - except (litellm.BadRequestError, ValueError): + except (litellm_exceptions.BadRequestError, ValueError): return () if isinstance(candidates, dict): return (cast(DeploymentTypedDict, candidates),) # cast-ok: the router returns one deployment as a dict @@ -15187,7 +15247,7 @@ class Router(DeprecatedRouterMembers): if team_id is not None } if len(team_ids) > 1: - raise litellm.BadRequestError( + raise litellm_exceptions.BadRequestError( message=( f"Model name '{model}' matches deployments from multiple teams. " "Specify the deployment ID directly to disambiguate." @@ -15379,14 +15439,14 @@ class Router(DeprecatedRouterMembers): # If still no deployments after checking for fallbacks, raise an error if len(healthy_deployments) == 0: - raise litellm.BadRequestError( + raise litellm_exceptions.BadRequestError( message=f"You passed in model={model}. {RouterErrors.no_healthy_deployments.value}", model=model, llm_provider="", ) - if litellm.model_alias_map and model in litellm.model_alias_map: - model = litellm.model_alias_map[ + if self._host.dynamic().model_alias_map and model in self._host.dynamic().model_alias_map: + model = self._host.dynamic().model_alias_map[ model ] # update the model to the actual value if an alias has been passed in @@ -15398,7 +15458,7 @@ class Router(DeprecatedRouterMembers): """A strategy marker is never a callable deployment, whichever resolution arm produced it.""" selectable: Final = [d for d in deployments if not self._is_strategy_marker_deployment(d)] if deployments and not selectable: - raise litellm.BadRequestError( + raise litellm_exceptions.BadRequestError( message=f"You passed in model={model}. {RouterErrors.only_strategy_marker_deployments.value}", model=model, llm_provider="", @@ -15418,7 +15478,7 @@ class Router(DeprecatedRouterMembers): self._drop_strategy_markers(model, healthy_deployments), request_team_id, now=datetime.now(timezone.utc) ) if result.blocking_window is not None and len(result.deployments) == 0: - raise litellm.BadRequestError( + raise litellm_exceptions.BadRequestError( message=f"Deployment {model} is reserved for another team until " f"{result.blocking_window.end:%H:%M} {result.blocking_window.timezone}", model=model, @@ -15426,7 +15486,7 @@ class Router(DeprecatedRouterMembers): ) shared: Final = filter_ptu_shared_deployments(result.deployments, request_team_id) if shared.withheld and len(shared.deployments) == 0: - raise litellm.BadRequestError( + raise litellm_exceptions.BadRequestError( message=f"Deployment {model} is reserved for the teams holding a PTU share of it", model=model, llm_provider="", @@ -15539,7 +15599,7 @@ class Router(DeprecatedRouterMembers): if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: - raise litellm.ServiceUnavailableError( + raise litellm_exceptions.ServiceUnavailableError( message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", @@ -15663,7 +15723,7 @@ class Router(DeprecatedRouterMembers): _OPAQUE_VALUE_ADAPTER.validate_python(request_kwargs), ) ) - healthy_deployments = litellm.utils.get_order_filtered_deployments( + healthy_deployments = litellm_utils.get_order_filtered_deployments( list( self._narrow_to_video_deployment( without_deployments_that_cannot_batch( @@ -15678,7 +15738,7 @@ class Router(DeprecatedRouterMembers): ## this request via weighted-failover. Always honored, regardless of the ## router-level flag, so a stale exclusion key on kwargs cannot escape. _excluded_deployment_ids: Final = (request_kwargs or {}).pop("_excluded_deployment_ids", None) - healthy_deployments = litellm.utils.get_excluded_filtered_deployments( + healthy_deployments = litellm_utils.get_excluded_filtered_deployments( cast(list[dict], healthy_deployments), excluded_deployment_ids=_excluded_deployment_ids, ) @@ -15690,7 +15750,7 @@ class Router(DeprecatedRouterMembers): request_kwargs.pop("_retry_skipped_deployment_ids", None) if request_kwargs else None ) healthy_deployments = ( - litellm.utils.get_excluded_filtered_deployments( + litellm_utils.get_excluded_filtered_deployments( healthy_deployments, excluded_deployment_ids=_retry_skipped_deployment_ids ) or healthy_deployments @@ -15807,14 +15867,14 @@ class Router(DeprecatedRouterMembers): one setting under three names, and a provider handed two of them either rejects the request or picks one by iteration order.""" if not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(tier_litellm_params): - Router._stamp_client_output_ceiling(request_kwargs) + PythonCore._stamp_client_output_ceiling(request_kwargs) for carrier in OUTPUT_TOKEN_CEILING_PARAMS: request_kwargs.pop(carrier, None) if "reasoning_effort" not in tier_litellm_params: return request_kwargs.pop("thinking", None) - Router._pop_effort_from_nested_carrier(request_kwargs, "output_config") - Router._pop_effort_from_nested_carrier(request_kwargs, "reasoning") + PythonCore._pop_effort_from_nested_carrier(request_kwargs, "output_config") + PythonCore._pop_effort_from_nested_carrier(request_kwargs, "reasoning") async def _async_get_available_deployment_unresolved( # noqa: ANN202 # legacy selector result remains inferred self, @@ -16065,7 +16125,7 @@ class Router(DeprecatedRouterMembers): # 3. If specific deployment returned, verify if it supports pass-through if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: - raise litellm.ServiceUnavailableError( + raise litellm_exceptions.ServiceUnavailableError( message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", @@ -16077,7 +16137,7 @@ class Router(DeprecatedRouterMembers): ) return healthy_deployments else: - raise litellm.BadRequestError( + raise litellm_exceptions.BadRequestError( message=f"Deployment {healthy_deployments.get('model_info', {}).get('id')} does not support pass-through endpoint (use_in_pass_through=False)", model=model, llm_provider="", @@ -16087,7 +16147,7 @@ class Router(DeprecatedRouterMembers): pass_through_deployments = self._filter_pass_through_deployments(healthy_deployments=healthy_deployments) if len(pass_through_deployments) == 0: - raise litellm.BadRequestError( + raise litellm_exceptions.BadRequestError( message=f"Model {model} has no deployments configured with use_in_pass_through=True. Please add use_in_pass_through: true to the deployment configuration", model=model, llm_provider="", @@ -16491,13 +16551,13 @@ class Router(DeprecatedRouterMembers): await router_host().authorize_strategy_inference( deployment=marker, request_kwargs=request_kwargs, - llm_router=self, + llm_router=self._core_public(), ) # Same tag-aware lookup the proxy's pre-call arming used, so an alias with # several tag-scoped markers cannot suppress under one and route under another. compression_policy: Final = router_host().compression_policy( - llm_router=self, + llm_router=self._core_public(), model_alias=registered_model_name, request_kwargs=request_kwargs, request_tags=get_routing_tags_from_request_kwargs(request_kwargs), @@ -16647,7 +16707,7 @@ class Router(DeprecatedRouterMembers): return tuple( key for key in forwarded_keys - if isinstance(key, str) and Router._deployment_sets_litellm_param(deployment_litellm_params, key) + if isinstance(key, str) and PythonCore._deployment_sets_litellm_param(deployment_litellm_params, key) ) @staticmethod @@ -16703,7 +16763,7 @@ class Router(DeprecatedRouterMembers): caller_parameters: Final = ( capture_baseline_parameters(request_kwargs) if router_name and baseline_model else None ) - Router._stamp_or_clear_metadata_key( + PythonCore._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key="_autorouter_baseline_route", value=( @@ -16712,13 +16772,13 @@ class Router(DeprecatedRouterMembers): else None ), ) - Router._stamp_or_clear_metadata_key( + PythonCore._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key="routing_decision", value=( None if routing_decision is None - else Router._redact_prompt_text_if_needed( + else PythonCore._redact_prompt_text_if_needed( request_kwargs=request_kwargs, routing_decision=routing_decision ) ), @@ -16825,7 +16885,7 @@ class Router(DeprecatedRouterMembers): if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: - raise litellm.ServiceUnavailableError( + raise litellm_exceptions.ServiceUnavailableError( message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", @@ -16907,7 +16967,7 @@ class Router(DeprecatedRouterMembers): routing_kwargs, ) ) - healthy_deployments = litellm.utils.get_order_filtered_deployments( + healthy_deployments = litellm_utils.get_order_filtered_deployments( list( self._narrow_to_video_deployment( without_deployments_that_cannot_batch( @@ -16923,7 +16983,7 @@ class Router(DeprecatedRouterMembers): ## this request via weighted-failover. See async counterpart in ## async_get_healthy_deployments for details. _excluded_deployment_ids: Final = (request_kwargs or {}).pop("_excluded_deployment_ids", None) - healthy_deployments = litellm.utils.get_excluded_filtered_deployments( + healthy_deployments = litellm_utils.get_excluded_filtered_deployments( healthy_deployments, excluded_deployment_ids=_excluded_deployment_ids, ) @@ -16933,7 +16993,7 @@ class Router(DeprecatedRouterMembers): request_kwargs.pop("_retry_skipped_deployment_ids", None) if request_kwargs else None ) healthy_deployments = ( - litellm.utils.get_excluded_filtered_deployments( + litellm_utils.get_excluded_filtered_deployments( healthy_deployments, excluded_deployment_ids=_retry_skipped_deployment_ids ) or healthy_deployments @@ -17056,7 +17116,7 @@ class Router(DeprecatedRouterMembers): # 2. If the returned is a specific deployment (Dict), verify and return directly if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: - raise litellm.ServiceUnavailableError( + raise litellm_exceptions.ServiceUnavailableError( message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", @@ -17067,7 +17127,7 @@ class Router(DeprecatedRouterMembers): return healthy_deployments else: # Specific deployment does not support pass-through - raise litellm.BadRequestError( + raise litellm_exceptions.BadRequestError( message=f"Deployment {healthy_deployments.get('model_info', {}).get('id')} does not support pass-through endpoint (use_in_pass_through=False)", model=model, llm_provider="", @@ -17078,7 +17138,7 @@ class Router(DeprecatedRouterMembers): if len(pass_through_deployments) == 0: # No deployments support pass-through - raise litellm.BadRequestError( + raise litellm_exceptions.BadRequestError( message=f"Model {model} has no deployment configured with use_in_pass_through=True. Please add use_in_pass_through: true in the deployment configuration", model=model, llm_provider="", @@ -17117,7 +17177,7 @@ class Router(DeprecatedRouterMembers): ) if self._is_priority_routing_group(model): - pass_through_deployments = litellm.utils.get_order_filtered_deployments( + pass_through_deployments = litellm_utils.get_order_filtered_deployments( pass_through_deployments, target_order=request_kwargs.pop("_target_order", None) if request_kwargs is not None else None, ) @@ -17374,43 +17434,49 @@ class Router(DeprecatedRouterMembers): return None if ( - isinstance(exception, litellm.AuthenticationError) + isinstance(exception, litellm_exceptions.AuthenticationError) and allowed_fails_policy.AuthenticationErrorAllowedFails is not None ): return allowed_fails_policy.AuthenticationErrorAllowedFails - if isinstance(exception, litellm.Timeout) and allowed_fails_policy.TimeoutErrorAllowedFails is not None: + if ( + isinstance(exception, litellm_exceptions.Timeout) + and allowed_fails_policy.TimeoutErrorAllowedFails is not None + ): return allowed_fails_policy.TimeoutErrorAllowedFails if ( - isinstance(exception, litellm.RateLimitError) + isinstance(exception, litellm_exceptions.RateLimitError) and allowed_fails_policy.RateLimitErrorAllowedFails is not None ): return allowed_fails_policy.RateLimitErrorAllowedFails if ( - isinstance(exception, litellm.ContentPolicyViolationError) + isinstance(exception, litellm_exceptions.ContentPolicyViolationError) and allowed_fails_policy.ContentPolicyViolationErrorAllowedFails is not None ): return allowed_fails_policy.ContentPolicyViolationErrorAllowedFails if ( - isinstance(exception, litellm.BadRequestError) + isinstance(exception, litellm_exceptions.BadRequestError) and allowed_fails_policy.BadRequestErrorAllowedFails is not None ): return allowed_fails_policy.BadRequestErrorAllowedFails if ( - isinstance(exception, litellm.InternalServerError) + isinstance(exception, litellm_exceptions.InternalServerError) and allowed_fails_policy.InternalServerErrorAllowedFails is not None ): return allowed_fails_policy.InternalServerErrorAllowedFails if ( - isinstance(exception, litellm.ServiceUnavailableError) + isinstance(exception, litellm_exceptions.ServiceUnavailableError) and allowed_fails_policy.ServiceUnavailableErrorAllowedFails is not None ): return allowed_fails_policy.ServiceUnavailableErrorAllowedFails if ( - isinstance(exception, litellm.BadGatewayError) + isinstance(exception, litellm_exceptions.BadGatewayError) and allowed_fails_policy.BadGatewayErrorAllowedFails is not None ): return allowed_fails_policy.BadGatewayErrorAllowedFails - if isinstance(exception, litellm.NotFoundError) and allowed_fails_policy.NotFoundErrorAllowedFails is not None: + if ( + isinstance(exception, litellm_exceptions.NotFoundError) + and allowed_fails_policy.NotFoundErrorAllowedFails is not None + ): return allowed_fails_policy.NotFoundErrorAllowedFails def _initialize_alerting(self): @@ -17429,10 +17495,8 @@ class Router(DeprecatedRouterMembers): self._core_slack_alerting_logger = _slack_alerting_logger - litellm.logging_callback_manager.add_litellm_callback(_slack_alerting_logger) - litellm.logging_callback_manager.add_litellm_success_callback( - _slack_alerting_logger.response_taking_too_long_callback - ) + self._host.callbacks.add_litellm_callback(_slack_alerting_logger) + self._host.callbacks.add_litellm_success_callback(_slack_alerting_logger.response_taking_too_long_callback) verbose_router_logger.info("\033[94m\nInitialized Alerting for litellm.Router\033[0m\n") def _core_set_custom_routing_strategy(self, CustomRoutingStrategy: CustomRoutingStrategyBase) -> None: @@ -17469,7 +17533,7 @@ class Router(DeprecatedRouterMembers): delattr(self, attr) def flush_cache(self): - litellm.cache = None + self._host.set_global_cache(None) self._core_cache.flush_cache() session_in_memory_cache: Final = self._claude_code_session_router_cache.in_memory_cache if session_in_memory_cache is not None: @@ -17478,10 +17542,10 @@ class Router(DeprecatedRouterMembers): def reset(self): ## clean up on close - litellm.success_callback = [] - litellm._async_success_callback = [] - litellm.failure_callback = [] - litellm._async_failure_callback = [] + self._host.set_callback_list("success_callback", []) + self._host.set_callback_list("_async_success_callback", []) + self._host.set_callback_list("failure_callback", []) + self._host.set_callback_list("_async_failure_callback", []) self.retry_policy = None self.flush_cache() @@ -17490,7 +17554,7 @@ def _name_core_members_after_old_names() -> None: # An old member that moved to ``_core_`` can still be reached as the function itself # (``cached_deployment_model_info`` is an lru_cache over one), so errors that name the method # keep the name it had before. - for name, member in vars(Router).items(): + for name, member in vars(PythonCore).items(): old_name = name.removeprefix("_core_") if old_name != name and inspect.isfunction(member) and hasattr(DeprecatedRouterMembers, old_name): member.__qualname__ = f"Router.{old_name}" diff --git a/litellm/router_python_host.py b/litellm/router_python_host.py new file mode 100644 index 00000000000..145ec83f3dd --- /dev/null +++ b/litellm/router_python_host.py @@ -0,0 +1,73 @@ +""" +``PythonHost``: what the router core reaches outside itself, served from the litellm package. + +The core never touches ``litellm.*`` directly. Inference calls (``invoke``), backoff sleeps, +callback registration and the process-wide cache all go through the host, and the process settings +come from ``dynamic()`` (``litellm/router_config.py``). A Rust core gets the same operations. +""" + +import asyncio +from collections.abc import Awaitable, Callable +from typing import ( + TYPE_CHECKING, + Final, + Literal, + TypeAlias, + cast, # noqa: TID251 # untyped litellm globals: the live config view and the callback lists +) + +import litellm +from litellm.router_config import RouterDynamicConfig, live_dynamic_config + +if TYPE_CHECKING: + from litellm.caching.caching import Cache + from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager + +CallbackList: TypeAlias = Literal[ + "success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback", "input_callback" +] + +Endpoint: TypeAlias = Callable[..., object] + + +class PythonHost: + __slots__ = () + + def invoke(self, op: str) -> Endpoint: + """The litellm function named ``op`` (``acompletion``, ``get_model_info``, ...), looked up at call time.""" + endpoint: Final[object] = getattr(litellm, op, None) + if endpoint is None: + raise AttributeError(f"litellm has no endpoint {op!r}") + if not callable(endpoint): + raise TypeError(f"litellm.{op} is not an endpoint") + return endpoint + + def sleep(self, seconds: float) -> Awaitable[None]: + return asyncio.sleep(seconds) + + def dynamic(self) -> RouterDynamicConfig: + """The process settings, read live on each access (``current_dynamic_config()`` takes a snapshot).""" + return cast(RouterDynamicConfig, live_dynamic_config) # cast-ok: the live view reads the same fields + + @property + def callbacks(self) -> "LoggingCallbackManager": + return litellm.logging_callback_manager + + def callback_list(self, name: CallbackList) -> list[object]: # mutable-ok: the process callback list itself + value: Final = getattr(litellm, name) + return cast(list[object], value) # cast-ok: the process callback list is an untyped litellm global + + def set_callback_list(self, name: CallbackList, value: list[object]) -> None: # mutable-ok: assigned as given + setattr(litellm, name, value) + + def global_cache(self) -> "Cache | None": + return litellm.cache + + def set_global_cache(self, cache: "Cache | None") -> None: + litellm.cache = cache + + def suppress_debug_info(self) -> None: + litellm.suppress_debug_info = True + + +python_host: Final = PythonHost() diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 9b61c1d39ae..a84a40c67c1 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -3,6 +3,7 @@ import os IGNORE_FUNCTIONS = [ "_json_cost", # bounded at depth 32 and consumed under byte/node limits. + "detached", # router_config: copies only the plain containers of the router's own constructor arguments, once per build or changeset "_format_type", "remove_additional_properties", "remove_strict_from_schema", diff --git a/tests/code_coverage_tests/router_litellm_globals_check.py b/tests/code_coverage_tests/router_litellm_globals_check.py new file mode 100644 index 00000000000..ff9fac0e3dc --- /dev/null +++ b/tests/code_coverage_tests/router_litellm_globals_check.py @@ -0,0 +1,54 @@ +""" +The router and its cores never read or write ``litellm.*``. + +Process settings reach the router through ``RouterDynamicConfig`` (litellm/router_config.py) and +everything else outside it (inference calls, callbacks, the global cache) through ``PythonHost`` +(litellm/router_python_host.py). Those two modules are the only router code allowed to touch the +``litellm`` package namespace. This check fails on ``import litellm`` (or ``import litellm.``, +which binds the same name) and on any ``litellm.`` access in the files below. + +Run: python tests/code_coverage_tests/router_litellm_globals_check.py +""" + +import ast +import os +import sys + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +CHECKED = ( + "litellm/router.py", + "litellm/router_api.py", + "litellm/router_core.py", + "litellm/router_deprecated.py", + "litellm/router_python_core.py", +) + + +def violations(rel: str) -> list[str]: + tree = ast.parse(open(os.path.join(ROOT, rel)).read(), rel) + found = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.extend( + f"{rel}:{node.lineno}: import {alias.name}" + for alias in node.names + if alias.asname is None and (alias.name == "litellm" or alias.name.startswith("litellm.")) + ) + elif isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id == "litellm": + found.append(f"{rel}:{node.lineno}: litellm.{node.attr}") + return found + + +def main() -> int: + found = [line for rel in CHECKED for line in violations(rel)] + for line in found: + print(line) + if found: + print(f"{len(found)} litellm.* uses in router modules; read settings from RouterDynamicConfig and use PythonHost") + return 1 + print("router modules use no litellm.* globals") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/code_coverage_tests/router_surface_check.py b/tests/code_coverage_tests/router_surface_check.py index 729ba8ab7a4..a8e0d6fa0d7 100644 --- a/tests/code_coverage_tests/router_surface_check.py +++ b/tests/code_coverage_tests/router_surface_check.py @@ -38,11 +38,15 @@ ALLOWLIST = os.path.join(os.path.dirname(__file__), "router_surface_allowlist.js ROUTER_FILES = { "litellm/router.py", "litellm/router_api.py", + "litellm/router_config.py", + "litellm/router_core.py", + "litellm/router_python_core.py", + "litellm/router_python_host.py", "litellm/router_deprecated.py", "litellm/router_host.py", } -ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_deprecated.py"] -ROUTER_CLASS_NAMES = {"Router", "DeprecatedRouterMembers"} +ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_python_core.py", "litellm/router_deprecated.py"] +ROUTER_CLASS_NAMES = {"Router", "PythonCore", "DeprecatedRouterMembers"} SCAN_ROOTS = ["litellm", "enterprise"] SKIP_DIRS = { diff --git a/tests/unit/router_strategy/test_router_routing_groups.py b/tests/unit/router_strategy/test_router_routing_groups.py index 6b5c052618b..5eeb46c3518 100644 --- a/tests/unit/router_strategy/test_router_routing_groups.py +++ b/tests/unit/router_strategy/test_router_routing_groups.py @@ -236,7 +236,7 @@ async def test_async_dispatch_uses_group_strategy_for_grouped_model(): wraps=group_selector.async_get_available_deployments, ) as latency_spy, patch( - "litellm.router.simple_shuffle", wraps=litellm.router.simple_shuffle + "litellm.router_python_core.simple_shuffle", wraps=litellm.router_python_core.simple_shuffle ) as shuffle_spy, ): await router.async_get_available_deployment( @@ -271,7 +271,7 @@ async def test_async_dispatch_falls_back_to_default_for_ungrouped_models(): wraps=group_selector.async_get_available_deployments, ) as latency_spy, patch( - "litellm.router.simple_shuffle", wraps=litellm.router.simple_shuffle + "litellm.router_python_core.simple_shuffle", wraps=litellm.router_python_core.simple_shuffle ) as shuffle_spy, ): await router.async_get_available_deployment( @@ -331,7 +331,7 @@ async def test_async_dispatch_uses_default_selector_when_constructed_with_enum() wraps=default_selector.async_get_available_deployments, ) as latency_spy, patch( - "litellm.router.simple_shuffle", wraps=litellm.router.simple_shuffle + "litellm.router_python_core.simple_shuffle", wraps=litellm.router_python_core.simple_shuffle ) as shuffle_spy, ): await router.async_get_available_deployment( @@ -1376,7 +1376,7 @@ async def test_group_call_dispatches_via_group_selector(): "async_get_available_deployments", wraps=group_selector.async_get_available_deployments, ) as latency_spy, - patch("litellm.router.simple_shuffle", wraps=litellm.router.simple_shuffle) as shuffle_spy, + patch("litellm.router_python_core.simple_shuffle", wraps=litellm.router_python_core.simple_shuffle) as shuffle_spy, ): deployment = await router.async_get_available_deployment(model="quality", request_kwargs={}) diff --git a/tests/unit/router_strategy/test_router_tag_routing.py b/tests/unit/router_strategy/test_router_tag_routing.py index 28192704bd1..b4e29df8116 100644 --- a/tests/unit/router_strategy/test_router_tag_routing.py +++ b/tests/unit/router_strategy/test_router_tag_routing.py @@ -2686,7 +2686,7 @@ async def test_plain_tag_exhaustion_with_universal_default_tag_raises_by_default router = _quality_high_cost_low_router() with patch( - "litellm.router.async_get_cooldown_deployments", + "litellm.router_python_core.async_get_cooldown_deployments", new=AsyncMock(return_value=["quality-high-1", "quality-high-2"]), ): with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info: @@ -2711,7 +2711,7 @@ async def test_plain_tag_exhaustion_with_universal_default_tag_falls_open_when_a from unittest.mock import AsyncMock, patch with patch( - "litellm.router.async_get_cooldown_deployments", + "litellm.router_python_core.async_get_cooldown_deployments", new=AsyncMock(return_value=["quality-high-1", "quality-high-2"]), ): response = await router.acompletion( diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 7deeb255e6c..71c37b5b1d5 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -1303,7 +1303,7 @@ class TestDeploymentCallbackOnFailureCooldownTimePrecedence: }, } - with patch("litellm.router.set_cooldown_deployments") as mock_set_cooldown: + with patch("litellm.router_python_core.set_cooldown_deployments") as mock_set_cooldown: router.deployment_callback_on_failure( kwargs=kwargs, completion_response=None, @@ -1339,7 +1339,7 @@ class TestDeploymentCallbackOnFailureCooldownTimePrecedence: }, } - with patch("litellm.router.set_cooldown_deployments") as mock_set_cooldown: + with patch("litellm.router_python_core.set_cooldown_deployments") as mock_set_cooldown: router.deployment_callback_on_failure( kwargs=kwargs, completion_response=None, diff --git a/tests/unit/router_utils/test_kubernetes_pod_discovery.py b/tests/unit/router_utils/test_kubernetes_pod_discovery.py index 115e2a49bc5..0b772b09f6f 100644 --- a/tests/unit/router_utils/test_kubernetes_pod_discovery.py +++ b/tests/unit/router_utils/test_kubernetes_pod_discovery.py @@ -17,6 +17,7 @@ import respx from openai import AsyncOpenAI, OpenAI import litellm +import litellm.router_python_core from litellm.constants import ( CALLER_SESSION_ID_METADATA_KEY, KUBERNETES_POD_ROUTING_KEY, @@ -281,7 +282,7 @@ def _patch_router_discovery( *discoveries: KubernetesPodDiscovery, ) -> None: discovery_iter: Final = iter(discoveries) - monkeypatch.setattr(litellm.router, "KubernetesPodDiscovery", lambda: next(discovery_iter)) + monkeypatch.setattr(litellm.router_python_core, "KubernetesPodDiscovery", lambda: next(discovery_iter)) def _stub_sync_dns(monkeypatch: pytest.MonkeyPatch, *ips: str) -> None: diff --git a/tests/unit/test_conftest_isolation.py b/tests/unit/test_conftest_isolation.py index 92457119a82..3fdc2d65da1 100644 --- a/tests/unit/test_conftest_isolation.py +++ b/tests/unit/test_conftest_isolation.py @@ -117,12 +117,12 @@ def test_live_router_membership_is_scoped_to_this_test(): } ] ) - assert _CanaryRouterHolder.router in litellm_router_module._live_routers + assert _CanaryRouterHolder.router._core in litellm_router_module._live_routers def test_live_router_membership_was_rolled_back(): assert _CanaryRouterHolder.router is not None - assert _CanaryRouterHolder.router not in litellm_router_module._live_routers + assert _CanaryRouterHolder.router._core not in litellm_router_module._live_routers def test_aws_cache_reset_sees_new_handlers_and_replaced_caches(monkeypatch: pytest.MonkeyPatch) -> None: @@ -175,7 +175,7 @@ def registered_collection_models() -> Iterator[None]: yield litellm_utils_module._runtime_registered_model_cost.clear() litellm_utils_module._runtime_registered_model_cost.update(original_registrations) - litellm_router_module._live_routers.discard(collection_router) + litellm_router_module._live_routers.discard(collection_router._core) litellm_router_module._live_routers.update(original_routers) diff --git a/tests/unit/test_router/core/__init__.py b/tests/unit/test_router/core/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_router/core/conftest.py b/tests/unit/test_router/core/conftest.py new file mode 100644 index 00000000000..dffa938c441 --- /dev/null +++ b/tests/unit/test_router/core/conftest.py @@ -0,0 +1,26 @@ +from collections.abc import Callable + +import pytest + +from litellm.router_core import RouterCore +from tests.unit.test_router.core.scripted_host import CORE_BUILDERS, CoreDeclined, ScriptedHost + + +@pytest.fixture +def host() -> ScriptedHost: + return ScriptedHost() + + +@pytest.fixture(params=sorted(CORE_BUILDERS)) +def make_core(request: pytest.FixtureRequest, host: ScriptedHost) -> Callable[..., RouterCore]: + """``make_core(**router_arguments)``: the core under test, on the ``host`` fixture. Every test that + uses it runs once per core in ``CORE_BUILDERS`` (``-k python`` or ``-k rust`` picks one).""" + builder = CORE_BUILDERS[request.param] + + def build(**arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments + try: + return builder(host, **arguments) + except CoreDeclined as declined: + pytest.skip(f"{request.param} core declines this configuration: {declined}") + + return build diff --git a/tests/unit/test_router/core/scripted_host.py b/tests/unit/test_router/core/scripted_host.py new file mode 100644 index 00000000000..af28160b620 --- /dev/null +++ b/tests/unit/test_router/core/scripted_host.py @@ -0,0 +1,487 @@ +""" +A host for driving ``PythonCore`` in isolation. + +``PythonCore`` reaches outside itself only through its host (``litellm/router_python_host.py``): +inference calls, backoff sleeps, the process settings, callback registration and the process-wide +cache. ``ScriptedHost`` serves all of those from the test. Inference ops answer from a per-model +script and record every call; sleeps are recorded and return at once; the process settings are a +fixed ``RouterDynamicConfig`` the test can replace; callbacks live in the host's own lists, and an +answered call runs the success or failure callbacks the core registered, as litellm's logging does. +Ops that are not inference (``get_model_info``, ``should_retry``, ...) are litellm's own functions. +""" + +import dataclasses +import itertools +from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping +from types import MethodType +from typing import Final + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager +from litellm.router_config import RouterDynamicConfig, build_router_config +from litellm.router_core import RouterCore +from litellm.router_python_core import PythonCore +from litellm.router_python_host import CallbackList, Endpoint, PythonHost +from litellm.types.utils import EmbeddingResponse, ModelResponse + +INFERENCE_OPS: Final = frozenset( + { + "acompletion", + "completion", + "atext_completion", + "text_completion", + "aembedding", + "embedding", + "aimage_generation", + "image_generation", + "aspeech", + "atranscription", + "arerank", + "rerank", + "amoderation", + "aresponses", + "responses", + "anthropic_messages", + "agenerate_content", + "aadapter_generate_content", + "aocr", + "asearch", + "avideo_generation", + "acreate_batch", + "acreate_file", + "afile_content", + "afile_delete", + "acreate_fine_tuning_job", + "aget_responses", + "adelete_responses", + "acancel_responses", + "alist_input_items", + } +) + +Outcome = object +"""What a scripted call answers: a ``BaseException`` instance is raised, a callable is called with +the request's keyword arguments (and its result used in turn), anything else is returned as is. +``None`` builds a default response for the op.""" + + +@dataclasses.dataclass(frozen=True, slots=True) +class Call: + op: str + model: str + kwargs: Mapping[str, object] + + @property + def deployment_id(self) -> str | None: + model_info: Final = self.kwargs.get("model_info") + return str(model_info["id"]) if isinstance(model_info, Mapping) and "id" in model_info else None + + @property + def model_group(self) -> str | None: + metadata: Final = self.kwargs.get("metadata") or self.kwargs.get("litellm_metadata") + return metadata.get("model_group") if isinstance(metadata, Mapping) else None + + +def completion_response(content: str = "ok", total_tokens: int = 10, model: str = "gpt-4o") -> ModelResponse: + return ModelResponse( + model=model, + choices=[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": content}}], + usage={ + "prompt_tokens": total_tokens // 2, + "completion_tokens": total_tokens - total_tokens // 2, + "total_tokens": total_tokens, + }, + ) + + +def embedding_response(model: str = "text-embedding-3-small") -> EmbeddingResponse: + return EmbeddingResponse( + model=model, + data=[{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + usage={"prompt_tokens": 2, "total_tokens": 2}, + ) + + +def _fresh_error(template: BaseException) -> BaseException: + """A copy of ``template`` with the same attributes (``copy.copy`` rebuilds litellm errors from + their args and drops ``status_code`` and the rest).""" + error: Final = BaseException.__new__(type(template)) + error.args = template.args + error.__dict__.update(template.__dict__) + return error + + +def _default_response(op: str, model: str) -> object: + if op in {"aembedding", "embedding"}: + return embedding_response(model) + return completion_response(model=model) + + +def rate_limit(model: str = "gpt-4o", message: str = "slow down") -> litellm.RateLimitError: + return litellm.RateLimitError(message=message, llm_provider="openai", model=model) + + +def bad_request(model: str = "gpt-4o", message: str = "bad input") -> litellm.BadRequestError: + return litellm.BadRequestError(message=message, llm_provider="openai", model=model) + + +def context_window(model: str = "gpt-4o") -> litellm.ContextWindowExceededError: + return litellm.ContextWindowExceededError(message="too long", llm_provider="openai", model=model) + + +def content_policy(model: str = "gpt-4o") -> litellm.ContentPolicyViolationError: + return litellm.ContentPolicyViolationError(message="blocked", llm_provider="openai", model=model) + + +def server_error(model: str = "gpt-4o") -> litellm.InternalServerError: + return litellm.InternalServerError(message="boom", llm_provider="openai", model=model) + + +def timeout(model: str = "gpt-4o") -> litellm.Timeout: + return litellm.Timeout(message="timed out", llm_provider="openai", model=model) + + +def auth_error(model: str = "gpt-4o") -> litellm.AuthenticationError: + return litellm.AuthenticationError(message="bad key", llm_provider="openai", model=model) + + +def dynamic_config(**changes: object) -> RouterDynamicConfig: # kwargs-ok: RouterDynamicConfig fields + """Process settings as a fresh litellm process has them, with no callbacks, cache or fallbacks.""" + base: Final = RouterDynamicConfig( + version=1, + num_retries=None, + max_fallbacks=None, + router_max_fallbacks=5, + allowed_fails=3, + request_timeout=6000.0, + fallbacks=None, + default_fallbacks=None, + context_window_fallbacks=None, + content_policy_fallbacks=None, + drop_params=None, + provider_list=litellm.provider_list, + expose_router_debug_in_errors=False, + redact_user_api_key_info=None, + model_alias_map={}, + model_cost=litellm.model_cost, + known_custom_logger_compatible_callbacks=litellm._known_custom_logger_compatible_callbacks, + exception_types=litellm.LITELLM_EXCEPTION_TYPES, + callbacks=[], + success_callback=[], + failure_callback=[], + input_callback=[], + callback_manager=litellm.logging_callback_manager, + cache=None, + ) + return dataclasses.replace(base, **changes) + + +class HostCallbacks: + """The callback registration the core asks its host for, over the host's own lists.""" + + def __init__(self, host: "ScriptedHost") -> None: + self._host: Final = host + self._keys: Final = LoggingCallbackManager() + + def _add(self, name: CallbackList, callback: object) -> None: + target: Final = self._host.lists[name] + if callback not in target: + target.append(callback) + + def add_litellm_callback(self, callback: object) -> None: + self._add_to("callbacks", callback) + + def _add_to(self, name: str, callback: object) -> None: + target: Final = self._host.lists.setdefault(name, []) + if callback not in target: + target.append(callback) + + def add_litellm_success_callback(self, callback: object) -> None: + self._add("success_callback", callback) + + def add_litellm_async_success_callback(self, callback: object) -> None: + self._add("_async_success_callback", callback) + + def add_litellm_failure_callback(self, callback: object) -> None: + self._add("failure_callback", callback) + + def add_litellm_async_failure_callback(self, callback: object) -> None: + self._add("_async_failure_callback", callback) + + def add_litellm_input_callback(self, callback: object) -> None: + self._add("input_callback", callback) + + def get_custom_logger_key(self, custom_logger: CustomLogger) -> str: + return self._keys.get_custom_logger_key(custom_logger) + + def with_custom_logger_keys( + self, candidates: Iterable[CustomLogger], keys: frozenset[str] + ) -> tuple[CustomLogger, ...]: + return self._keys.with_custom_logger_keys(candidates, keys) + + def without_custom_logger_keys(self, callbacks: Iterable[object], keys: frozenset[str]) -> Iterator[object]: + return self._keys.without_custom_logger_keys(callbacks, keys) # pyright: ignore[reportArgumentType] # any callback list + + def get_all_callbacks(self) -> list[object]: # mutable-ok: matches LoggingCallbackManager + return [callback for callbacks in self._host.lists.values() for callback in callbacks] + + def get_custom_loggers_for_type( + self, callback_type: type[CustomLogger] + ) -> list[CustomLogger]: # mutable-ok: matches LoggingCallbackManager + found: Final[list[CustomLogger]] = [] + for callback in self.get_all_callbacks(): + if isinstance(callback, callback_type) and callback not in found: + found.append(callback) + return found + + def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]: + dropped_ids: Final = frozenset(map(id, callbacks)) + owner_ids: Final = frozenset(map(id, method_owners)) + + def keep(callback: object) -> bool: + if id(callback) in dropped_ids: + return False + return not (isinstance(callback, MethodType) and id(callback.__self__) in owner_ids) + + vacated: Final = frozenset( + self.get_custom_logger_key(callback) + for callback in self._host.lists.get("callbacks", []) + if isinstance(callback, CustomLogger) and id(callback) in dropped_ids + ) + for name, callbacks_in_list in self._host.lists.items(): + self._host.lists[name] = [callback for callback in callbacks_in_list if keep(callback)] + return vacated + + +class ScriptedHost(PythonHost): + """ + ``script`` maps a deployment's ``litellm_params.model`` (or ``"*"`` for any) to the outcomes its + calls answer in turn; the last outcome repeats. A model with no script answers a default response. + """ + + __slots__ = ("_dynamic", "_scripts", "_turns", "cache", "calls", "lists", "raised", "sleeps", "suppressed") + + def __init__( + self, script: Mapping[str, Iterable[Outcome]] | None = None, **dynamic: object + ) -> None: # kwargs-ok: RouterDynamicConfig fields + self._scripts: dict[str, tuple[Outcome, ...]] = {k: tuple(v) for k, v in (script or {}).items()} + self._turns: dict[str, Iterator[int]] = {} + self._dynamic: RouterDynamicConfig = dynamic_config(**dynamic) + self.calls: Final[list[Call]] = [] + self.sleeps: Final[list[float]] = [] + # The errors the scripted calls raised, in order: each is a fresh copy of its scripted template. + self.raised: Final[list[BaseException]] = [] + self.lists: Final[dict[str, list[object]]] = { + "callbacks": [], + "success_callback": [], + "_async_success_callback": [], + "failure_callback": [], + "_async_failure_callback": [], + "input_callback": [], + } + self.cache: object = None + self.suppressed = False + + # --- what the test controls ------------------------------------------------------------- + + def answer(self, model: str, *outcomes: Outcome) -> None: + """Script the calls to ``model`` from now on.""" + self._scripts[model] = outcomes + self._turns.pop(model, None) + + def set_dynamic(self, **changes: object) -> None: # kwargs-ok: RouterDynamicConfig fields + self._dynamic = dataclasses.replace(self._dynamic, **changes, version=self._dynamic.version + 1) + + def calls_to(self, op: str | None = None) -> list[Call]: # mutable-ok: a fresh list + return [call for call in self.calls if op is None or call.op == op] + + def deployments_called(self, op: str | None = None) -> list[str | None]: # mutable-ok: a fresh list + return [call.deployment_id for call in self.calls_to(op)] + + # --- the host operations ---------------------------------------------------------------- + + def invoke(self, op: str) -> Endpoint: + if op not in INFERENCE_OPS: + return super().invoke(op) + if op.startswith("a") or op in {"anthropic_messages", "_arealtime"}: + + async def endpoint(*args: object, **kwargs: object) -> object: # kwargs-ok: the endpoint's request + return await self._answer_async(op, args, kwargs) + + return endpoint + + def sync_endpoint(*args: object, **kwargs: object) -> object: # kwargs-ok: the endpoint's request + return self._answer_sync(op, args, kwargs) + + return sync_endpoint + + def sleep(self, seconds: float) -> Awaitable[None]: + self.sleeps.append(seconds) + + async def done() -> None: + return None + + return done() + + def dynamic(self) -> RouterDynamicConfig: + return self._dynamic + + @property + def callbacks(self) -> HostCallbacks: # pyright: ignore[reportIncompatibleVariableOverride] # the core uses the same methods + return HostCallbacks(self) + + def callback_list(self, name: CallbackList) -> list[object]: # mutable-ok: the host's own list + return self.lists.setdefault(name, []) + + def set_callback_list(self, name: CallbackList, value: list[object]) -> None: # mutable-ok: assigned as given + self.lists[name] = value + + def global_cache(self) -> object: # pyright: ignore[reportIncompatibleMethodOverride] # any cache object + return self.cache + + def set_global_cache(self, cache: object) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # any cache object + self.cache = cache + + def suppress_debug_info(self) -> None: + self.suppressed = True + + # --- answering ------------------------------------------------------------------------ + + def _outcome(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> tuple[Call, object]: + model: Final = str(kwargs.get("model") or (args[0] if args else "")) + call: Final = Call(op=op, model=model, kwargs=dict(kwargs)) + self.calls.append(call) + key: Final = model if model in self._scripts else "*" if "*" in self._scripts else None + if key is None: + return call, None + outcomes: Final = self._scripts[key] + turn: Final = next(self._turns.setdefault(key, itertools.count())) + outcome: object = outcomes[min(turn, len(outcomes) - 1)] if outcomes else None + while callable(outcome) and not isinstance(outcome, BaseException | type): + outcome = outcome(**kwargs) # rebind-ok: a callable outcome answers with what it returns + if isinstance(outcome, BaseException): + # A fresh error per call: the core stamps attributes (num_retries, ...) on the one it raises. + outcome = _fresh_error(outcome) # rebind-ok: the scripted error is a template + self.raised.append(outcome) + return call, outcome + + def _logging_kwargs( + self, call: Call, error: BaseException | None, response: object + ) -> dict[str, object]: # mutable-ok: litellm's logging kwargs + kwargs: Final = call.kwargs + usage: Final = getattr(response, "usage", None) + metadata: Final = kwargs.get("metadata") or kwargs.get("litellm_metadata") or {} + model_info: Final = kwargs.get("model_info") or {} + return { + "call_type": call.op, + "model": call.model, + "exception": error, + "litellm_params": { + "metadata": metadata, + "model_info": model_info, + "api_base": kwargs.get("api_base"), + **({"cooldown_time": kwargs["cooldown_time"]} if "cooldown_time" in kwargs else {}), + }, + "standard_logging_object": { + "total_tokens": getattr(usage, "total_tokens", 0) or 0, + "model_group": metadata.get("model_group"), + "model_id": model_info.get("id"), + "hidden_params": {"litellm_model_name": call.model}, + }, + } + + def _finish(self, op: str, call: Call, outcome: object) -> tuple[BaseException | None, object]: + if isinstance(outcome, type) and issubclass(outcome, BaseException): + outcome = outcome("scripted failure") # rebind-ok: an exception class raises an instance + if isinstance(outcome, BaseException): + return outcome, None + return None, _default_response(op, call.model) if outcome is None else outcome + + async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: + call, outcome = self._outcome(op, args, kwargs) + error, response = self._finish(op, call, outcome) + logged: Final = self._logging_kwargs(call, error, response) + if error is not None: + for callback in list(self.lists.get("failure_callback", ())): + if callable(callback): + callback(logged, None, 0.0, 0.0) + for callback in list(self.lists.get("_async_failure_callback", ())): + if callable(callback): + await callback(logged, None, 0.0, 0.0) + raise error + for callback in list(self.lists.get("_async_success_callback", ())): + if callable(callback): + await callback(logged, response, 0.0, 0.0) + return response + + def _answer_sync(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: + call, outcome = self._outcome(op, args, kwargs) + error, response = self._finish(op, call, outcome) + logged: Final = self._logging_kwargs(call, error, response) + if error is not None: + for callback in list(self.lists.get("failure_callback", ())): + if callable(callback): + callback(logged, None, 0.0, 0.0) + raise error + for callback in list(self.lists.get("success_callback", ())): + if callable(callback): + callback(logged, response, 0.0, 0.0) + return response + + +def deployment( + model_name: str, + deployment_id: str, + model: str | None = None, + **params: object, # kwargs-ok: extra litellm_params +) -> dict[str, object]: # mutable-ok: a model_list entry + """One ``model_list`` entry; ``model`` defaults to ``openai/`` so each deployment + answers its own script.""" + model_info: Final = params.pop("model_info", None) + return { + "model_name": model_name, + "litellm_params": {"model": model or f"openai/{deployment_id}", "api_key": "sk-test", **params}, + "model_info": {"id": deployment_id, **(model_info if isinstance(model_info, dict) else {})}, + } + + +def python_core(host: ScriptedHost, **arguments: object) -> PythonCore: # kwargs-ok: the router's constructor arguments + """A ``PythonCore`` built the way ``Router`` builds one, on ``host``.""" + config: Final = build_router_config(arguments) + return PythonCore(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the same arguments Router binds + + +CoreBuilder = Callable[..., RouterCore] +"""``builder(host, **router_arguments)``: a router core on ``host``.""" + +CORE_BUILDERS: Final[dict[str, CoreBuilder]] = {"python": python_core} +"""Every core the suite runs on. Another core adds its builder here; a builder that cannot take a +configuration raises ``CoreDeclined``, and the test is skipped for that core.""" + + +class CoreDeclined(Exception): + """The core does not serve this configuration (the router would stay on the Python core).""" + + +__all__ = [ + "Call", + "HostCallbacks", + "INFERENCE_OPS", + "CORE_BUILDERS", + "CoreBuilder", + "CoreDeclined", + "ScriptedHost", + "auth_error", + "bad_request", + "completion_response", + "content_policy", + "context_window", + "deployment", + "dynamic_config", + "embedding_response", + "python_core", + "rate_limit", + "server_error", + "timeout", +] diff --git a/tests/unit/test_router/core/test_endpoints.py b/tests/unit/test_router/core/test_endpoints.py new file mode 100644 index 00000000000..a3cffe8278a --- /dev/null +++ b/tests/unit/test_router/core/test_endpoints.py @@ -0,0 +1,445 @@ +""" +What a router core sends to its host for each endpoint, and what it hands back. + +Every test builds its core through ``make_core`` on the ``host`` fixture, so it runs once per core. +""" + +import datetime +import io +from collections.abc import AsyncIterator, Callable, Coroutine, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.router_core import RouterCore +from litellm.types.utils import EmbeddingResponse, ModelResponse, ModelResponseStream +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + bad_request, + completion_response, + deployment, + embedding_response, +) + +MESSAGES: Final = [{"role": "user", "content": "hi"}] +MakeCore = Callable[..., RouterCore] + +EndpointCall = Callable[[RouterCore], object] + +ENDPOINTS: Final[dict[str, tuple[str, EndpointCall]]] = { + "acompletion": ("acompletion", lambda core: core.acompletion(model="g", messages=MESSAGES)), + "completion": ("completion", lambda core: core.completion(model="g", messages=MESSAGES)), + "aembedding": ("aembedding", lambda core: core.aembedding(model="g", input=["a"])), + "embedding": ("embedding", lambda core: core.embedding(model="g", input=["a"])), + "atext_completion": ("atext_completion", lambda core: core.atext_completion(model="g", prompt="hi")), + "aimage_generation": ("aimage_generation", lambda core: core.aimage_generation(model="g", prompt="a cat")), + "image_generation": ("image_generation", lambda core: core.image_generation(model="g", prompt="a cat")), + "arerank": ("arerank", lambda core: core.arerank(model="g", query="q", documents=["a"])), + "aspeech": ("aspeech", lambda core: core.aspeech(model="g", input="hi", voice="alloy")), + "atranscription": ("atranscription", lambda core: core.atranscription(model="g", file=io.BytesIO(b"x"))), + "aresponses": ("aresponses", lambda core: core.aresponses(model="g", input="hi")), + "aanthropic_messages": ( + "anthropic_messages", + lambda core: core.aanthropic_messages(model="g", messages=MESSAGES, max_tokens=5), + ), + "call-acompletion": ("acompletion", lambda core: core.call("acompletion", {"model": "g", "messages": MESSAGES})), + "call-completion": ("completion", lambda core: core.call("completion", {"model": "g", "messages": MESSAGES})), +} +SYNC_ENDPOINTS: Final = frozenset({"completion", "embedding", "image_generation"}) +LITELLM_METADATA_ENDPOINTS: Final = frozenset({"aresponses", "aanthropic_messages"}) +EMBEDDING_ENDPOINTS: Final = frozenset({"aembedding", "embedding"}) + + +async def _invoke(core: RouterCore, endpoint: str) -> object: + result: Final = ENDPOINTS[endpoint][1](core) + if endpoint in SYNC_ENDPOINTS: + return result + return await result # pyright: ignore[reportGeneralTypeIssues] # the async endpoints return awaitables + + +def _one_group( + make_core: MakeCore, **litellm_params: object +) -> RouterCore: # kwargs-ok: the deployment's litellm_params + return make_core(model_list=[deployment("g", "d1", **litellm_params)]) + + +def _metadata(kwargs: Mapping[str, object], endpoint: str) -> Mapping[str, object]: + bucket: Final = kwargs["litellm_metadata" if endpoint in LITELLM_METADATA_ENDPOINTS else "metadata"] + assert isinstance(bucket, Mapping) + return bucket + + +@pytest.mark.parametrize("endpoint", sorted(ENDPOINTS)) +async def test_each_endpoint_sends_the_deployment_litellm_params_to_the_host( + make_core: MakeCore, host: ScriptedHost, endpoint: str +): + core: Final = _one_group(make_core, api_base="https://d1.example", temperature=0.3, seed=7) + await _invoke(core, endpoint) + sent: Final = host.calls[0].kwargs + assert (sent["model"], sent["api_key"], sent["api_base"], sent["temperature"], sent["seed"]) == ( + "openai/d1", + "sk-test", + "https://d1.example", + 0.3, + 7, + ) + + +@pytest.mark.parametrize("endpoint", sorted(ENDPOINTS)) +async def test_each_endpoint_invokes_its_host_op_once_with_routing_metadata( + make_core: MakeCore, host: ScriptedHost, endpoint: str +): + core: Final = _one_group(make_core, api_base="https://d1.example") + await _invoke(core, endpoint) + assert [call.op for call in host.calls] == [ENDPOINTS[endpoint][0]] + call: Final = host.calls[0] + metadata: Final = _metadata(call.kwargs, endpoint) + assert (call.model_group, call.deployment_id) == ("g", "d1") + assert (metadata["model_group"], metadata["deployment"], metadata["api_base"]) == ( + "g", + "openai/d1", + "https://d1.example", + ) + assert metadata["model_info"]["id"] == "d1" # pyright: ignore[reportIndexIssue] # metadata values are untyped + + +@pytest.mark.parametrize("endpoint", sorted(ENDPOINTS)) +async def test_each_endpoint_returns_the_host_response_object_with_routing_hidden_params( + make_core: MakeCore, host: ScriptedHost, endpoint: str +): + answer: Final[ModelResponse | EmbeddingResponse] = ( + embedding_response() if endpoint in EMBEDDING_ENDPOINTS else completion_response("scripted") + ) + host.answer("openai/d1", answer) + core: Final = _one_group(make_core) + response: Final = await _invoke(core, endpoint) + assert response is answer + assert response._hidden_params["model_id"] == "d1" + assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "g" + + +@pytest.mark.parametrize("endpoint", sorted(SYNC_ENDPOINTS | {"call-completion"})) +async def test_sync_endpoints_return_the_response_not_an_awaitable( + make_core: MakeCore, host: ScriptedHost, endpoint: str +): + answer: Final = embedding_response() if endpoint == "embedding" else completion_response("sync") + host.answer("openai/d1", answer) + core: Final = _one_group(make_core) + if endpoint == "call-completion": + assert await core.call("completion", {"model": "g", "messages": MESSAGES}) is answer + else: + assert ENDPOINTS[endpoint][1](core) is answer + + +@pytest.mark.parametrize( + "endpoint, request_body", + [("acompletion", {"messages": MESSAGES}), ("completion", {"messages": MESSAGES})], +) +async def test_router_only_include_fallback_errors_is_not_sent_to_the_host( + make_core: MakeCore, host: ScriptedHost, endpoint: str, request_body: dict[str, object] +): + core: Final = _one_group(make_core) + await core.call(endpoint, {"model": "g", "include_fallback_errors": True, **request_body}) + assert host.calls[0].op == endpoint + assert "include_fallback_errors" not in host.calls[0].kwargs + + +@pytest.mark.parametrize("router_only_param", ["rpm", "tpm", "max_parallel_requests"]) +async def test_deployment_limit_params_are_forwarded_with_the_rest_of_litellm_params( + make_core: MakeCore, host: ScriptedHost, router_only_param: str +): + core: Final = _one_group(make_core, **{router_only_param: 50}) + await core.acompletion(model="g", messages=MESSAGES) + assert host.calls[0].kwargs[router_only_param] == 50 + + +@pytest.mark.parametrize("endpoint", ["acompletion", "aembedding", "completion", "embedding"]) +async def test_request_metadata_reaches_the_host_beside_the_routing_metadata( + make_core: MakeCore, host: ScriptedHost, endpoint: str +): + core: Final = _one_group(make_core) + request: Final[dict[str, object]] = {"messages": MESSAGES} if "completion" in endpoint else {"input": ["a"]} + await core.call(endpoint, {"model": "g", "metadata": {"trace": "t-1"}, **request}) + metadata: Final = _metadata(host.calls[0].kwargs, endpoint) + assert (metadata["trace"], metadata["model_group"]) == ("t-1", "g") + + +async def test_aresponses_sends_request_metadata_as_metadata_and_routing_metadata_as_litellm_metadata( + make_core: MakeCore, host: ScriptedHost +): + core: Final = _one_group(make_core) + await core.aresponses(model="g", input="hi", metadata={"trace": "t-1"}) + sent: Final = host.calls[0].kwargs + assert sent["metadata"] == {"trace": "t-1"} + assert sent["litellm_metadata"]["model_group"] == "g" # pyright: ignore[reportIndexIssue] # untyped kwargs + + +@pytest.mark.parametrize( + "router_arguments, deployment_params, expected_timeout", + [ + ({}, {}, 6000.0), + ({"timeout": 42}, {}, 42), + ({"timeout": 42}, {"timeout": 11}, 11.0), + ({}, {"timeout": 11}, 11.0), + ], +) +async def test_acompletion_timeout_comes_from_deployment_then_router_then_process_setting( + make_core: MakeCore, + host: ScriptedHost, + router_arguments: dict[str, object], + deployment_params: dict[str, object], + expected_timeout: float, +): + core: Final = make_core(model_list=[deployment("g", "d1", **deployment_params)], **router_arguments) + await core.acompletion(model="g", messages=MESSAGES) + assert host.calls[0].kwargs["timeout"] == expected_timeout + + +@pytest.mark.parametrize("endpoint", ["acompletion", "aembedding", "aresponses", "aanthropic_messages"]) +async def test_request_timeout_overrides_router_and_deployment_timeouts( + make_core: MakeCore, host: ScriptedHost, endpoint: str +): + core: Final = make_core(model_list=[deployment("g", "d1", timeout=11)], timeout=42) + request: Final[dict[str, object]] = ( + {"input": ["a"]} + if endpoint == "aembedding" + else {"input": "hi"} + if endpoint == "aresponses" + else {"messages": MESSAGES, "max_tokens": 5} + ) + await core.call(endpoint, {"model": "g", "timeout": 7, **request}) + assert host.calls[0].kwargs["timeout"] == 7 + + +async def test_aresponses_timeout_falls_back_to_router_timeout(make_core: MakeCore, host: ScriptedHost): + core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42) + await core.aresponses(model="g", input="hi") + assert host.calls[0].kwargs["timeout"] == 42 + + +@pytest.mark.parametrize( + "request_stream_timeout, expected_stream_timeout", + [(None, 3.0), (2, 2)], +) +async def test_streaming_acompletion_sends_deployment_or_request_stream_timeout( + make_core: MakeCore, host: ScriptedHost, request_stream_timeout: int | None, expected_stream_timeout: float +): + core: Final = _one_group(make_core, timeout=11, stream_timeout=3) + extra: Final = {} if request_stream_timeout is None else {"stream_timeout": request_stream_timeout} + await core.acompletion(model="g", messages=MESSAGES, stream=True, **extra) + sent: Final = host.calls[0].kwargs + assert (sent["stream"], sent["timeout"], sent["stream_timeout"]) == (True, 11.0, expected_stream_timeout) + + +@pytest.mark.parametrize( + "request_params, expected", + [ + ({}, ("u1", 0.9)), + ({"temperature": 0.1}, ("u1", 0.1)), + ({"user": "u2"}, ("u2", 0.9)), + ], +) +async def test_default_litellm_params_fill_what_the_request_leaves_unset( + make_core: MakeCore, host: ScriptedHost, request_params: dict[str, object], expected: tuple[str, float] +): + core: Final = make_core( + model_list=[deployment("g", "d1")], default_litellm_params={"user": "u1", "temperature": 0.9} + ) + await core.acompletion(model="g", messages=MESSAGES, **request_params) + sent: Final = host.calls[0].kwargs + assert (sent["user"], sent["temperature"]) == expected + + +@pytest.mark.parametrize( + "request_credentials, expected_api_base", + [ + ({"api_key": "sk-client"}, "https://d1.example"), + ({"api_key": "sk-client", "api_base": "https://client.example"}, "https://client.example"), + ], +) +async def test_request_level_api_key_replaces_the_deployment_key_on_the_host_call( + make_core: MakeCore, host: ScriptedHost, request_credentials: dict[str, str], expected_api_base: str +): + core: Final = _one_group(make_core, api_base="https://d1.example") + await core.acompletion(model="g", messages=MESSAGES, **request_credentials) + call: Final = host.calls[0] + assert (call.kwargs["api_key"], call.kwargs["api_base"], call.model_group) == ( + "sk-client", + expected_api_base, + "g", + ) + + +async def test_construction_registers_one_success_and_one_failure_callback_per_mode( + make_core: MakeCore, host: ScriptedHost +): + _one_group(make_core) + registered: Final = {name: len(callbacks) for name, callbacks in host.lists.items()} + assert registered == { + "callbacks": 0, + "success_callback": 1, + "_async_success_callback": 1, + "failure_callback": 1, + "_async_failure_callback": 1, + "input_callback": 0, + } + assert all(callable(callbacks[0]) for callbacks in host.lists.values() if callbacks) + + +async def test_retire_removes_the_callbacks_the_core_registered(make_core: MakeCore, host: ScriptedHost): + core: Final = _one_group(make_core) + core.retire() + assert {name: callbacks for name, callbacks in host.lists.items() if callbacks} == {} + + +async def test_sync_completion_counts_its_tokens_and_request_in_group_usage(make_core: MakeCore, host: ScriptedHost): + core: Final = _one_group(make_core, tpm=100, rpm=5) + assert await core.health().usage("g") == (None, None) + core.completion(model="g", messages=MESSAGES) + assert await core.health().usage("g") == (10, 1) + assert await core.health().usage("g", kind="remaining") == { + "x-ratelimit-remaining-tokens": 90, + "x-ratelimit-limit-tokens": 100, + "x-ratelimit-remaining-requests": 4, + "x-ratelimit-limit-requests": 5, + } + + +class DeferredLoggingHost(ScriptedHost): + """``ScriptedHost`` that, like litellm, runs async success callbacks after the endpoint returns.""" + + __slots__ = ("pending",) + + def __init__(self) -> None: + super().__init__() + self.pending: list[Coroutine[object, object, None]] = [] + + async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object: + callbacks: Final = self.lists["_async_success_callback"] + self.lists["_async_success_callback"] = [] + try: + response: Final = await super()._answer_async(op, args, kwargs) + finally: + self.lists["_async_success_callback"] = callbacks + call: Final = self.calls[-1] + logged: Final = self._logging_kwargs(call, None, response) + self.pending.extend(callback(logged, response, 0.0, 0.0) for callback in callbacks if callable(callback)) + return response + + async def drain(self) -> None: + while self.pending: + await self.pending.pop(0) + + +class TestUsageWhenTheHostLogsAfterTheResponse: + """litellm runs async success callbacks after the endpoint has returned; this host does the same.""" + + @pytest.fixture + def host(self) -> DeferredLoggingHost: + return DeferredLoggingHost() + + async def test_one_acompletion_counts_its_tokens_and_request_once( + self, make_core: MakeCore, host: DeferredLoggingHost + ): + core: Final = _one_group(make_core, tpm=100, rpm=5) + await core.acompletion(model="g", messages=MESSAGES) + assert await core.health().usage("g") == (10, 1) + await host.drain() + assert await core.health().usage("g") == (10, 1) + + +async def test_usage_based_routing_registers_its_logger_on_the_host(make_core: MakeCore, host: ScriptedHost): + make_core(model_list=[deployment("g", "d1", rpm=1)], routing_strategy="usage-based-routing-v2") + assert [type(callback).__name__ for callback in host.lists["callbacks"]] == ["LowestTPMLoggingHandler_v2"] + + +async def test_a_successful_call_uses_up_the_deployment_rpm_for_the_next_selection( + make_core: MakeCore, host: ScriptedHost +): + core: Final = make_core(model_list=[deployment("g", "d1", rpm=1)], routing_strategy="usage-based-routing-v2") + await core.acompletion(model="g", messages=MESSAGES) + with pytest.raises(litellm.RateLimitError, match="No deployments available"): + await core.acompletion(model="g", messages=MESSAGES) + assert host.deployments_called() == ["d1"] + + +async def test_host_error_reaches_the_caller_as_raised(make_core: MakeCore, host: ScriptedHost): + error: Final = bad_request(message="scripted bad input") + host.answer("openai/d1", error) + core: Final = _one_group(make_core) + with pytest.raises(litellm.BadRequestError, match="scripted bad input"): + await core.acompletion(model="g", messages=MESSAGES) + assert host.deployments_called("acompletion")[0] == "d1" + + +@pytest.mark.parametrize("endpoint", ["acompletion", "completion"]) +async def test_unknown_model_group_raises_without_calling_the_host( + make_core: MakeCore, host: ScriptedHost, endpoint: str +): + core: Final = _one_group(make_core) + with pytest.raises(litellm.BadRequestError, match="model=missing"): + await core.call(endpoint, {"model": "missing", "messages": MESSAGES}) + assert host.calls == [] + + +@pytest.mark.parametrize("op", ["no_such_endpoint", "acompletions", ""]) +async def test_call_with_an_unknown_op_raises_attribute_error_without_calling_the_host( + make_core: MakeCore, host: ScriptedHost, op: str +): + core: Final = _one_group(make_core) + with pytest.raises(AttributeError): + await core.call(op, {"model": "g"}) + assert host.calls == [] + + +async def test_sync_text_completion_invokes_the_sync_host_op_with_deployment_credentials( + make_core: MakeCore, host: ScriptedHost +): + core: Final = _one_group(make_core, api_base="https://d1.example") + core.text_completion(model="g", prompt="hi") + call: Final = host.calls[0] + assert (call.op, call.kwargs["api_base"], call.kwargs["prompt"], call.model_group) == ( + "text_completion", + "https://d1.example", + "hi", + "g", + ) + + +def _scripted_stream(*texts: str) -> Callable[..., CustomStreamWrapper]: + def answer(**_request: object) -> CustomStreamWrapper: # kwargs-ok: the host passes the request kwargs + async def chunks() -> AsyncIterator[ModelResponseStream]: + for text in texts: + yield ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}]) + + logging_obj: Final = LiteLLMLogging( + model="d1", + messages=MESSAGES, + stream=True, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="stream-test", + function_id="stream-test", + ) + logging_obj.update_environment_variables(model="d1", user=None, optional_params={}, litellm_params={}) + return CustomStreamWrapper( + completion_stream=chunks(), model="d1", custom_llm_provider="openai", logging_obj=logging_obj + ) + + return answer + + +async def test_streaming_acompletion_yields_the_host_stream_chunks_with_routing_hidden_params( + make_core: MakeCore, host: ScriptedHost +): + host.answer("openai/d1", _scripted_stream("he", "llo")) + core: Final = _one_group(make_core) + stream: Final = await core.acompletion(model="g", messages=MESSAGES, stream=True) + texts: Final = [chunk.choices[0].delta.content async for chunk in stream] # pyright: ignore[reportUnknownMemberType] # stream chunks are untyped + assert "".join(text for text in texts if text) == "hello" + assert stream._hidden_params["model_id"] == "d1" + assert stream._hidden_params["additional_headers"]["x-litellm-model-group"] == "g" + assert (host.calls[0].op, host.calls[0].kwargs["stream"]) == ("acompletion", True) diff --git a/tests/unit/test_router/core/test_fallbacks.py b/tests/unit/test_router/core/test_fallbacks.py new file mode 100644 index 00000000000..e4e5c3120c8 --- /dev/null +++ b/tests/unit/test_router/core/test_fallbacks.py @@ -0,0 +1,756 @@ +"""Cross-group fallbacks of a router core: where the chain comes from, how it is walked, and what is raised.""" + +from collections.abc import Callable, Mapping + +import pytest + +import litellm +from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + bad_request, + content_policy, + context_window, + deployment, + rate_limit, + server_error, + timeout, +) + +MakeCore = Callable[..., RouterCore] + +MESSAGES = [{"role": "user", "content": "hi"}] +GROUPS_ABC = [deployment("a", "a"), deployment("b", "b"), deployment("c", "c")] +GROUPS_ABCD = [*GROUPS_ABC, deployment("d", "d")] +CHAIN_ABCD = [{"a": ["b"]}, {"b": ["c"]}, {"c": ["d"]}] +TIERED_A = [deployment("a", "a1", order=1), deployment("a", "a2", order=2), deployment("b", "b")] + + +def build(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments + return make_core(**{"model_list": GROUPS_ABC, "num_retries": 0, **arguments}) + + +async def complete(core: RouterCore, model: str = "a", **request: object) -> object: # kwargs-ok: request kwargs + return await core.acompletion(model=model, messages=MESSAGES, **request) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + +def headers(response: object) -> Mapping[str, object]: + return getattr(response, "_hidden_params")["additional_headers"] + + +def show_router_debug(host: ScriptedHost) -> None: + host.set_dynamic(expose_router_debug_in_errors=True) + + +# --- where the fallback chain comes from ------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("constructor", "process", "request_kwargs", "expected"), + [ + ({"fallbacks": [{"a": ["b"]}]}, None, {}, ["a", "b"]), + ({}, [{"a": ["c"]}], {}, ["a", "c"]), + ({}, None, {"fallbacks": [{"a": ["b"]}]}, ["a", "b"]), + ({}, None, {"fallbacks": ["c"]}, ["a", "c"]), + ({"fallbacks": [{"a": ["b"]}]}, [{"a": ["c"]}], {}, ["a", "b"]), + ({"fallbacks": [{"a": ["b"]}]}, None, {"fallbacks": [{"a": ["c"]}]}, ["a", "c"]), + ], + ids=[ + "constructor", + "process-setting", + "request-kwarg", + "request-plain-list", + "constructor-over-process-setting", + "request-over-constructor", + ], +) +async def test_a_failed_group_falls_back_to_the_chain_from_the_winning_source( + make_core: MakeCore, + host: ScriptedHost, + constructor: dict[str, object], + process: list[object] | None, + request_kwargs: dict[str, object], + expected: list[str], +) -> None: + if process is not None: + host.set_dynamic(fallbacks=process) + core = build(make_core, **constructor) + host.answer("openai/a", rate_limit()) + + response = await complete(core, **request_kwargs) + + assert host.deployments_called("acompletion") == expected + assert headers(response)["x-litellm-model-group"] == expected[-1] + + +@pytest.mark.asyncio +async def test_an_empty_request_fallback_list_overrides_the_constructor_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core, fallbacks=[]) + + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["assign", "settings"]) +async def test_fallbacks_applied_by_changeset_take_effect(make_core: MakeCore, host: ScriptedHost, field: str) -> None: + core = build(make_core) + core.apply(ChangeSet(**{field: {"fallbacks": [{"a": ["c"]}]}})) # pyright: ignore[reportArgumentType] # assign or settings + host.answer("openai/a", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "c"] + + +# --- resolving the group's chain ---------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_the_generic_star_chain_catches_a_group_with_no_entry(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"b": ["a"]}, {"*": ["c"]}]) + host.answer("openai/a", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "c"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fallbacks", + [[{"*": ["c"]}, {"a": ["b"]}], [{"a": ["b"]}, {"*": ["c"]}]], + ids=["star-first", "star-last"], +) +async def test_a_group_specific_chain_wins_over_the_generic_star_chain( + make_core: MakeCore, host: ScriptedHost, fallbacks: list[object] +) -> None: + core = build(make_core, fallbacks=fallbacks) + host.answer("openai/a", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("group", "fallback_key"), + [("openai/gpt-x", "gpt-x"), ("gpt-4o", "openai/gpt-4o")], + ids=["provider-stripped-key", "provider-prefixed-key"], +) +async def test_a_chain_keyed_with_or_without_the_provider_prefix_matches_the_group( + make_core: MakeCore, host: ScriptedHost, group: str, fallback_key: str +) -> None: + core = build( + make_core, model_list=[deployment(group, "p"), deployment("b", "b")], fallbacks=[{fallback_key: ["b"]}] + ) + host.answer("openai/p", rate_limit()) + + await complete(core, model=group) + + assert host.deployments_called("acompletion") == ["p", "b"] + + +@pytest.mark.asyncio +async def test_a_prefixed_key_does_not_match_a_group_whose_provider_cannot_be_inferred( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build( + make_core, model_list=[deployment("mygroup", "p"), deployment("b", "b")], fallbacks=[{"openai/mygroup": ["b"]}] + ) + host.answer("openai/p", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core, model="mygroup") + + assert host.deployments_called("acompletion") == ["p"] + + +@pytest.mark.asyncio +async def test_a_dict_target_names_its_group_under_model(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": [{"model": "c"}]}]) + host.answer("openai/a", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "c"] + + +# --- walking the chain ------------------------------------------------------------------------ + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failing", "expected"), + [(["a"], ["a", "b"]), (["a", "b"], ["a", "b", "c"])], + ids=["first-target-answers", "second-target-answers"], +) +async def test_targets_are_tried_in_listed_order_until_one_answers( + make_core: MakeCore, host: ScriptedHost, failing: list[str], expected: list[str] +) -> None: + core = build(make_core, fallbacks=[{"a": ["b", "c"]}]) + for group in failing: + host.answer(f"openai/{group}", rate_limit()) + + response = await complete(core) + + assert host.deployments_called("acompletion") == expected + assert headers(response)["x-litellm-model-group"] == expected[-1] + + +@pytest.mark.asyncio +async def test_a_failed_fallback_group_follows_its_own_chain(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}]) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit()) + + response = await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b", "c"] + assert headers(response)["x-litellm-model-group"] == "c" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("fallbacks", "expected"), + [ + ([{"a": ["b"]}, {"b": ["a", "c"]}, {"c": ["a", "b"]}], ["a", "b", "c"]), + ([{"a": ["a", "b"]}], ["a", "b"]), + ([{"a": ["b", "b", "c"]}], ["a", "b", "c"]), + ], + ids=["cyclic-graph", "group-lists-itself", "target-listed-twice"], +) +async def test_no_group_is_tried_twice_in_one_request( + make_core: MakeCore, host: ScriptedHost, fallbacks: list[object], expected: list[str] +) -> None: + core = build(make_core, fallbacks=fallbacks) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core) + + assert host.deployments_called("acompletion") == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("constructor", "process", "expected"), + [ + ({"max_fallbacks": 0}, {}, ["a"]), + ({"max_fallbacks": 1}, {}, ["a", "b"]), + ({"max_fallbacks": 2}, {}, ["a", "b", "c"]), + ({}, {"max_fallbacks": 2}, ["a", "b", "c"]), + ({}, {"router_max_fallbacks": 2}, ["a", "b", "c"]), + ({}, {}, ["a", "b", "c", "d"]), + ({"max_fallbacks": 3}, {"max_fallbacks": 1}, ["a", "b", "c", "d"]), + ({}, {"max_fallbacks": 1, "router_max_fallbacks": 3}, ["a", "b"]), + ], + ids=[ + "constructor-0", + "constructor-1", + "constructor-2", + "process-max_fallbacks-2", + "process-router_max_fallbacks-2", + "default", + "constructor-over-process", + "process-max_fallbacks-over-router_max_fallbacks", + ], +) +async def test_max_fallbacks_bounds_the_depth_of_a_chained_walk( + make_core: MakeCore, + host: ScriptedHost, + constructor: dict[str, object], + process: dict[str, object], + expected: list[str], +) -> None: + if process: + host.set_dynamic(**process) + core = build(make_core, model_list=GROUPS_ABCD, fallbacks=CHAIN_ABCD, **constructor) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core) + + assert host.deployments_called("acompletion") == expected + + +@pytest.mark.asyncio +async def test_max_fallbacks_applied_by_changeset_bounds_the_walk(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, model_list=GROUPS_ABCD, fallbacks=CHAIN_ABCD) + core.apply(ChangeSet(assign={"max_fallbacks": 1})) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + + +@pytest.mark.asyncio +async def test_a_fallback_group_with_no_deployments_is_passed_over(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["missing", "c"]}]) + host.answer("openai/a", rate_limit()) + + response = await complete(core) + + assert host.deployments_called("acompletion") == ["a", "c"] + assert headers(response)["x-litellm-model-group"] == "c" + + +@pytest.mark.asyncio +async def test_a_chain_of_only_empty_groups_raises_the_primary_error(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["missing"]}]) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core) + + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +async def test_a_fallback_group_runs_after_the_primary_group_spends_its_retries( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}], num_retries=2) + host.answer("openai/a", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "a", "a", "b"] + + +# --- error-specific chains --------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error", "field"), + [(context_window, "context_window_fallbacks"), (content_policy, "content_policy_fallbacks")], + ids=["context-window", "content-policy"], +) +@pytest.mark.parametrize("source", ["constructor", "process-setting", "request-kwarg"]) +async def test_an_error_specific_chain_wins_over_the_general_chain( + make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, source: str +) -> None: + chain = [{"a": ["c"]}] + if source == "process-setting": + host.set_dynamic(**{field: chain}) + core = build(make_core, fallbacks=[{"a": ["b"]}], **({field: chain} if source == "constructor" else {})) + host.answer("openai/a", error()) + + response = await complete(core, **({field: chain} if source == "request-kwarg" else {})) + + assert host.deployments_called("acompletion") == ["a", "c"] + assert headers(response)["x-litellm-model-group"] == "c" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error", "field", "raised"), + [ + (context_window, "context_window_fallbacks", litellm.ContextWindowExceededError), + (content_policy, "content_policy_fallbacks", litellm.ContentPolicyViolationError), + ], + ids=["context-window", "content-policy"], +) +async def test_an_error_specific_chain_without_the_group_raises_without_trying_the_general_chain( + make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, raised: type[Exception] +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: [{"other": ["c"]}]}) + host.answer("openai/a", error()) + + with pytest.raises(raised): + await complete(core) + + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [context_window, content_policy], ids=["context-window", "content-policy"]) +async def test_without_an_error_specific_chain_the_general_chain_applies( + make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception] +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", error()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + + +@pytest.mark.asyncio +async def test_a_context_window_fallback_that_fails_otherwise_follows_the_general_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, context_window_fallbacks=[{"a": ["b"]}], fallbacks=[{"b": ["c"]}]) + host.answer("openai/a", context_window()) + host.answer("openai/b", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b", "c"] + + +# --- default fallbacks ------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("source", ["constructor", "process-setting"]) +async def test_default_fallbacks_catch_any_group(make_core: MakeCore, host: ScriptedHost, source: str) -> None: + if source == "process-setting": + host.set_dynamic(default_fallbacks=["c"]) + core = build(make_core, **({"default_fallbacks": ["c"]} if source == "constructor" else {})) + host.answer("openai/a", rate_limit()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "c"] + + +@pytest.mark.asyncio +async def test_default_fallbacks_join_the_configured_chains_as_the_star_entry( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"b": ["a"]}], default_fallbacks=["c"]) + host.answer("openai/a", rate_limit()) + + await complete(core) + + assert core.settings().fallbacks == [{"b": ["a"]}, {"*": ["c"]}] + assert host.deployments_called("acompletion") == ["a", "c"] + + +def test_default_fallbacks_given_as_a_string_are_refused(make_core: MakeCore) -> None: + with pytest.raises(TypeError, match=r"default_fallbacks must be a list of model names, got 'c'"): + build(make_core, default_fallbacks="c") + + +# --- order tiers inside one group ---------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failing", "expected", "model_group"), + [ + (["a1"], ["a1", "a2"], "a"), + (["a1", "a2"], ["a1", "a2", "b"], "b"), + ], + ids=["next-tier-answers", "external-fallback-after-every-tier"], +) +async def test_order_tiers_are_tried_before_the_external_fallbacks( + make_core: MakeCore, host: ScriptedHost, failing: list[str], expected: list[str], model_group: str +) -> None: + core = build(make_core, model_list=TIERED_A, fallbacks=[{"a": ["b"]}]) + for deployment_id in failing: + host.answer(f"openai/{deployment_id}", rate_limit()) + + response = await complete(core) + + assert host.deployments_called("acompletion") == expected + assert headers(response)["x-litellm-model-group"] == model_group + + +@pytest.mark.asyncio +async def test_order_tiers_are_walked_without_any_configured_fallbacks(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, model_list=[*TIERED_A, deployment("a", "a3", order=3)]) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core) + + assert host.deployments_called("acompletion") == ["a1", "a2", "a3"] + + +@pytest.mark.asyncio +async def test_a_context_window_error_skips_the_order_tiers_for_its_own_chain( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, model_list=TIERED_A, context_window_fallbacks=[{"a": ["b"]}]) + host.answer("openai/a1", context_window()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a1", "b"] + + +# --- when no fallback runs ----------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_disable_fallbacks_on_the_request_raises_the_primary_error( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError): + await complete(core, disable_fallbacks=True) + + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "error", + [ + GuardrailRaisedException(guardrail_name="g", message="no"), + BlockedPiiEntityError(entity_type="EMAIL", guardrail_name="g"), + ], + ids=["guardrail-raised", "blocked-pii"], +) +async def test_a_guardrail_intervention_never_falls_back( + make_core: MakeCore, host: ScriptedHost, error: Exception +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", error) + + with pytest.raises(type(error)) as raised: + await complete(core) + + assert raised.value is host.raised[-1] + assert host.deployments_called("acompletion") == ["a"] + + +def unsupported_params() -> litellm.UnsupportedParamsError: + return litellm.UnsupportedParamsError(message="nope", llm_provider="openai", model="a") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [bad_request, unsupported_params], ids=["bad-request", "unsupported-params"]) +async def test_a_bad_request_does_not_fall_back_when_fallback_on_unsupported_params_is_off( + make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception] +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}], fallback_on_unsupported_params=False) + host.answer("openai/a", error()) + + with pytest.raises(type(error())): + await complete(core) + + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error", "fallback_on_unsupported_params"), + [ + (bad_request, True), + (unsupported_params, True), + (context_window, False), + (content_policy, False), + (rate_limit, False), + (server_error, False), + ], + ids=[ + "bad-request-flag-on", + "unsupported-params-flag-on", + "context-window-flag-off", + "content-policy-flag-off", + "rate-limit-flag-off", + "server-error-flag-off", + ], +) +async def test_errors_outside_the_unsupported_params_rule_fall_back( + make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], fallback_on_unsupported_params: bool +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}], fallback_on_unsupported_params=fallback_on_unsupported_params) + host.answer("openai/a", error()) + + await complete(core) + + assert host.deployments_called("acompletion") == ["a", "b"] + + +# --- the error when every group fails -------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_when_every_fallback_fails_the_primary_groups_error_is_raised( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["b", "c"]}], max_fallbacks=1) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", timeout()) + host.answer("openai/c", server_error()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert str(raised.value) == "litellm.RateLimitError: slow down" + assert host.deployments_called("acompletion") == ["a", "b", "c"] + + +@pytest.mark.asyncio +async def test_without_router_debug_the_error_message_is_not_extended(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert str(raised.value) == "litellm.RateLimitError: slow down" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("arguments", "suffix"), + [ + ({}, "\n\nLiteLLM: model group 'a' failed with the error above. No fallback was attempted."), + ( + {"fallbacks": [{"a": ["b", "c"]}]}, + "\n\nLiteLLM: model group 'a' failed with the error above. " + "Fallback to b, c also failed: litellm.RateLimitError: slow down", + ), + ( + {"fallbacks": [{"a": ["b"]}, {"b": ["c"]}]}, + "\n\nLiteLLM: model group 'a' failed with the error above. " + "Fallback to b also failed: litellm.RateLimitError: slow down", + ), + ( + {"fallbacks": [{"b": ["c"]}]}, + "\n\nLiteLLM: model group 'a' failed with the error above and no fallback model group was found for it, " + "so the request was not retried on another model. Fallbacks are configured for: b. " + "Add a fallbacks entry for that model group (Router fallbacks or proxy router_settings.fallbacks) " + "to retry on another model.", + ), + ], + ids=["no-fallbacks", "fallback-list-failed", "chained-fallback-failed", "no-chain-for-group"], +) +async def test_router_debug_explains_the_fallback_outcome_once( + make_core: MakeCore, host: ScriptedHost, arguments: dict[str, object], suffix: str +) -> None: + core = build(make_core, **arguments) + show_router_debug(host) + host.answer("*", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert str(raised.value) == f"litellm.RateLimitError: slow down{suffix}" + + +@pytest.mark.asyncio +async def test_router_debug_names_the_first_fallback_groups_error(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", server_error()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert str(raised.value).endswith( + "LiteLLM: model group 'a' failed with the error above. Fallback to b also failed: litellm.InternalServerError: boom" + ) + + +@pytest.mark.asyncio +async def test_router_debug_reports_a_fallback_group_with_no_deployments( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, fallbacks=[{"a": ["missing"]}]) + show_router_debug(host) + host.answer("openai/a", rate_limit()) + + with pytest.raises(litellm.RateLimitError) as raised: + await complete(core) + + assert ( + "LiteLLM: model group 'a' failed with the error above. Fallback to missing also failed: " + "litellm.BadRequestError: You passed in model=missing. There are no healthy deployments for this model" + ) in str(raised.value) + + +@pytest.mark.asyncio +async def test_router_debug_on_an_unmatched_context_window_chain_says_no_fallback_was_attempted( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core, context_window_fallbacks=[{"other": ["c"]}]) + show_router_debug(host) + host.answer("openai/a", context_window()) + + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await complete(core) + + assert str(raised.value).endswith( + "\n\nLiteLLM: model group 'a' failed with the error above. No fallback was attempted." + ) + + +@pytest.mark.asyncio +async def test_router_debug_on_content_policy_without_any_chain_hints_at_content_policy_fallbacks( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = build(make_core) + show_router_debug(host) + host.answer("openai/a", content_policy()) + + with pytest.raises(litellm.ContentPolicyViolationError) as raised: + await complete(core) + + message = str(raised.value) + assert "model=a. content_policy_fallback=None. fallbacks=None." in message + assert message.endswith("LiteLLM: model group 'a' failed with the error above. No fallback was attempted.") + + +# --- response headers ------------------------------------------------------------------------------ + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failing", "attempted", "model_group"), + [([], 0, "a"), (["a"], 1, "b")], + ids=["primary-answers", "one-fallback"], +) +async def test_success_headers_name_the_answering_group_and_the_fallbacks_attempted( + make_core: MakeCore, host: ScriptedHost, failing: list[str], attempted: int, model_group: str +) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + for group in failing: + host.answer(f"openai/{group}", rate_limit()) + + response = await complete(core) + + assert headers(response)["x-litellm-attempted-fallbacks"] == attempted + assert headers(response)["x-litellm-model-group"] == model_group + + +@pytest.mark.asyncio +async def test_each_attempt_carries_its_own_model_group_to_the_host(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + await core.call("acompletion", {"model": "a", "messages": MESSAGES}) + + assert [call.model_group for call in host.calls_to("acompletion")] == ["a", "b"] + assert [call.model for call in host.calls_to("acompletion")] == ["openai/a", "openai/b"] + + +@pytest.mark.asyncio +async def test_embedding_falls_back_like_completion(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, model_list=[deployment("e", "e"), deployment("f", "f")], fallbacks=[{"e": ["f"]}]) + host.answer("openai/e", rate_limit()) + + response = await core.aembedding(model="e", input=["x"]) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + assert host.deployments_called("aembedding") == ["e", "f"] + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 + + +def test_sync_completion_falls_back_like_async(make_core: MakeCore, host: ScriptedHost) -> None: + core = build(make_core, fallbacks=[{"a": ["b"]}]) + host.answer("openai/a", rate_limit()) + + response = core.completion(model="a", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + assert host.deployments_called("completion") == ["a", "b"] + assert headers(response)["x-litellm-attempted-fallbacks"] == 1 diff --git a/tests/unit/test_router/core/test_registry_changesets.py b/tests/unit/test_router/core/test_registry_changesets.py new file mode 100644 index 00000000000..d4aa4d14a75 --- /dev/null +++ b/tests/unit/test_router/core/test_registry_changesets.py @@ -0,0 +1,657 @@ +""" +The core's registry and its settings, which are static after construction: ``apply(ChangeSet)`` +(and ``aapply``) is the only way to change them, and a direct attribute write is routed into one. +""" + +import copy +from collections.abc import Callable, Mapping + +import pytest + +import litellm +from litellm.router_api import ApplyResult, ChangeSet +from litellm.router_core import RouterCore +from litellm.types.router import Deployment +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + context_window, + deployment, + server_error, +) + +MakeCore = Callable[..., RouterCore] + +MESSAGES = [{"role": "user", "content": "hi"}] + + +def _model_list() -> list[dict[str, object]]: # mutable-ok: a fresh model_list + return [deployment("gpt", "a"), deployment("gpt", "b"), deployment("emb", "e")] + + +def _fresh(error: Callable[[], BaseException]) -> Callable[..., BaseException]: + """A scripted outcome raising a new exception per call: the core stamps num_retries on the instance it saw.""" + return lambda **_: error() + + +def _ids(deployments: object) -> list[str]: # mutable-ok: a fresh list + return [entry["model_info"]["id"] for entry in deployments] # pyright: ignore[reportGeneralTypeIssues, reportIndexIssue] # deployment dicts + + +def _deployment(model_name: str, deployment_id: str, **params: object) -> Deployment: # kwargs-ok: litellm_params + return Deployment(**deployment(model_name, deployment_id, **params)) # pyright: ignore[reportArgumentType] # a model_list entry + + +# -- registry lookups after construction -------------------------------------------------------- + + +@pytest.mark.parametrize(("deployment_id", "group"), [("b", "gpt"), ("e", "emb")]) +def test_registry_finds_each_configured_deployment_by_id(make_core: MakeCore, deployment_id: str, group: str) -> None: + registry = make_core(model_list=_model_list()).registry() + + found = registry.deployment(deployment_id) + + assert found is not None + assert (found.model_name, found.model_info.id, found.litellm_params.model) == ( + group, + deployment_id, + f"openai/{deployment_id}", + ) + assert registry.has_deployment(deployment_id) + + +def test_registry_has_no_deployment_for_an_unknown_id(make_core: MakeCore) -> None: + registry = make_core(model_list=_model_list()).registry() + + assert registry.deployment("missing") is None + assert not registry.has_deployment("missing") + + +@pytest.mark.parametrize(("group", "ids"), [("gpt", ["a", "b"]), ("unknown", [])]) +def test_registry_lists_a_groups_deployments_in_configured_order( + make_core: MakeCore, group: str, ids: list[str] +) -> None: + registry = make_core(model_list=_model_list()).registry() + + assert _ids(registry.deployments(group)) == ids + assert registry.names(group, kind="ids") == ids + + +def test_registry_names_every_group_and_deployment_id(make_core: MakeCore) -> None: + registry = make_core(model_list=_model_list()).registry() + + assert registry.names(kind="group_set") == {"gpt", "emb"} + assert registry.names(kind="ids") == ["a", "b", "e"] + assert _ids(registry.deployments()) == ["a", "b", "e"] + + +@pytest.mark.parametrize( + ("name", "kind", "resolved"), + [ + ("fast", "group", "gpt"), + ("fast", "alias", "gpt"), + ("gpt", "alias", None), + ("e", "from_id", "emb"), + ("fast", "recognized", True), + ("nope", "recognized", False), + ], +) +def test_registry_resolves_names_through_groups_aliases_and_ids( + make_core: MakeCore, name: str, kind: str, resolved: object +) -> None: + registry = make_core(model_list=_model_list(), model_group_alias={"fast": "gpt"}).registry() + + assert registry.resolve(name, kind=kind) == resolved # pyright: ignore[reportCallIssue, reportArgumentType] # kind is one of the literals + + +def test_registry_aliases_are_the_configured_model_group_alias(make_core: MakeCore) -> None: + registry = make_core(model_list=_model_list(), model_group_alias={"fast": "gpt"}).registry() + + assert registry.aliases() == {"fast": "gpt"} + assert _ids(registry.deployments("fast")) == ["a", "b"] + + +def test_registry_version_is_unchanged_by_reads(make_core: MakeCore) -> None: + core = make_core(model_list=_model_list()) + before = core.registry().version + + core.registry().deployments("gpt") + core.registry().names() + core.registry().resolve("gpt") + core.settings() + + assert core.registry().version == before + + +# -- registry().version on every write path ----------------------------------------------------- + + +async def _apply_upsert_new(core: RouterCore) -> None: + core.apply(ChangeSet(upsert=[_deployment("gpt", "c")])) + + +async def _apply_upsert_changed(core: RouterCore) -> None: + core.apply(ChangeSet(upsert=[_deployment("gpt", "a", api_base="https://changed.example")])) + + +async def _apply_delete(core: RouterCore) -> None: + core.apply(ChangeSet(delete=["a"])) + + +async def _aapply_upsert_new(core: RouterCore) -> None: + await core.aapply(ChangeSet(upsert=[_deployment("gpt", "c")])) # pyright: ignore[reportAttributeAccessIssue] # aapply is on every core + + +async def _add_deployment(core: RouterCore) -> None: + core.add_deployment(_deployment("gpt", "c")) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + +async def _set_model_list(core: RouterCore) -> None: + core.set_model_list([deployment("gpt", "c")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + +async def _upsert_deployment(core: RouterCore) -> None: + with pytest.deprecated_call(): + core.upsert_deployment(_deployment("gpt", "c")) # pyright: ignore[reportAttributeAccessIssue] # deprecated, still public + + +async def _delete_deployment(core: RouterCore) -> None: + with pytest.deprecated_call(): + core.delete_deployment("a") # pyright: ignore[reportAttributeAccessIssue] # deprecated, still public + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("write", "ids_after"), + [ + (_apply_upsert_new, ["a", "b", "e", "c"]), + (_apply_upsert_changed, ["a", "b", "e"]), + (_apply_delete, ["b", "e"]), + (_aapply_upsert_new, ["a", "b", "e", "c"]), + (_add_deployment, ["a", "b", "e", "c"]), + (_set_model_list, ["c"]), + (_upsert_deployment, ["a", "b", "e", "c"]), + (_delete_deployment, ["b", "e"]), + ], + ids=lambda value: getattr(value, "__name__", None), +) +async def test_every_deployment_write_advances_the_registry_version( + make_core: MakeCore, write: Callable[[RouterCore], object], ids_after: list[str] +) -> None: + core = make_core(model_list=_model_list()) + before = core.registry().version + + await write(core) # pyright: ignore[reportGeneralTypeIssues] # each write is a coroutine function + + assert core.registry().version > before + assert sorted(core.registry().names(kind="ids")) == sorted(ids_after) + + +@pytest.mark.asyncio +async def test_a_changed_upsert_is_what_the_registry_returns_afterwards(make_core: MakeCore) -> None: + core = make_core(model_list=_model_list()) + + await _apply_upsert_changed(core) + + changed = core.registry().deployment("a") + assert changed is not None + assert changed.litellm_params.api_base == "https://changed.example" + + +@pytest.mark.parametrize( + "changes", + [ + ChangeSet(), + ChangeSet(upsert=[_deployment("gpt", "a")]), + ChangeSet(delete=["missing"]), + ChangeSet(assign={"num_retries": 4, "cooldown_time": 9}), + ChangeSet(settings={"num_retries": 4, "allowed_fails": 2}), + ], + ids=["empty", "identical_upsert", "unknown_delete", "assign", "settings"], +) +def test_a_changeset_that_changes_no_deployment_keeps_the_registry_version( + make_core: MakeCore, changes: ChangeSet +) -> None: + core = make_core(model_list=_model_list()) + before = core.registry().version + + result = core.apply(changes) + + assert core.registry().version == before + assert result.version == before + + +def test_adding_a_deployment_whose_id_exists_changes_nothing(make_core: MakeCore) -> None: + core = make_core(model_list=_model_list()) + before = core.registry().version + + added = core.add_deployment(_deployment("emb", "a")) # pyright: ignore[reportAttributeAccessIssue] # public on the core + + assert added is None + assert core.registry().version == before + assert core.registry().resolve("a", kind="from_id") == "gpt" + + +# -- ApplyResult -------------------------------------------------------------------------------- + + +def test_apply_result_carries_the_new_version_and_the_upserted_deployment(make_core: MakeCore) -> None: + core = make_core(model_list=_model_list()) + + result = core.apply(ChangeSet(upsert=[_deployment("gpt", "c")])) + + assert isinstance(result, ApplyResult) + assert result.version == core.registry().version + assert [None if d is None else d.model_info.id for d in result.upserted] == ["c"] + assert list(result.deleted) == [] + + +def test_apply_result_reports_none_for_an_upsert_that_changed_nothing(make_core: MakeCore) -> None: + core = make_core(model_list=_model_list()) + + result = core.apply(ChangeSet(upsert=[_deployment("gpt", "a"), _deployment("gpt", "c")])) + + assert [None if d is None else d.model_info.id for d in result.upserted] == [None, "c"] + + +def test_apply_result_lists_each_delete_in_order_with_none_for_an_unknown_id(make_core: MakeCore) -> None: + core = make_core(model_list=_model_list()) + + result = core.apply(ChangeSet(delete=["e", "missing", "a"])) + + assert [None if d is None else d["model_info"]["id"] for d in result.deleted] == ["e", None, "a"] # pyright: ignore[reportIndexIssue] # the core returns the model_list dict + assert core.registry().names(kind="ids") == ["b"] + + +def test_apply_upserts_before_it_deletes(make_core: MakeCore) -> None: + core = make_core(model_list=_model_list()) + + result = core.apply(ChangeSet(upsert=[_deployment("gpt", "c")], delete=["c"])) + + assert [d is not None for d in result.upserted] == [True] + assert [d is not None for d in result.deleted] == [True] + assert not core.registry().has_deployment("c") + + +def test_settings_and_deployments_change_together_in_one_apply(make_core: MakeCore) -> None: + core = make_core(model_list=_model_list()) + + result = core.apply(ChangeSet(assign={"cooldown_time": 17}, upsert=[_deployment("emb", "f")], delete=["b"])) + + assert result.version == core.registry().version + assert core.settings().cooldown_time == 17 + assert core.registry().names("emb", kind="ids") == ["e", "f"] + assert core.registry().names("gpt", kind="ids") == ["a"] + + +# -- settings changed by a changeset change later calls ----------------------------------------- + + +def _assign(**values: object) -> ChangeSet: # kwargs-ok: settings by name + return ChangeSet(assign=values) + + +def _settings(**values: object) -> ChangeSet: # kwargs-ok: settings by name + return ChangeSet(settings=values) + + +CHANGESET_KINDS = pytest.mark.parametrize("changeset", [_assign, _settings], ids=["assign", "settings"]) + + +@pytest.mark.asyncio +@CHANGESET_KINDS +@pytest.mark.parametrize("num_retries", [0, 3]) +async def test_num_retries_from_a_changeset_governs_the_next_failing_call( + make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet], num_retries: int +) -> None: + core = make_core(model_list=[deployment("gpt", "a")], num_retries=2) + host.answer("openai/a", _fresh(server_error)) + with pytest.raises(litellm.InternalServerError): + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + host.calls.clear() + + core.apply(changeset(num_retries=num_retries)) + with pytest.raises(litellm.InternalServerError): + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert host.deployments_called("acompletion") == ["a"] * (num_retries + 1) + assert core.settings().as_dict()["num_retries"] == num_retries + + +@pytest.mark.asyncio +@CHANGESET_KINDS +async def test_fallbacks_from_a_changeset_are_used_by_the_next_call( + make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet] +) -> None: + core = make_core(model_list=[deployment("gpt", "a"), deployment("backup", "f")], num_retries=0) + host.answer("openai/a", _fresh(server_error)) + with pytest.raises(litellm.InternalServerError): + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + host.calls.clear() + + core.apply(changeset(fallbacks=[{"gpt": ["backup"]}])) + response = await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert host.deployments_called("acompletion") == ["a", "f"] + assert response._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1 + assert core.settings().fallbacks == [{"gpt": ["backup"]}] + + +@pytest.mark.asyncio +@CHANGESET_KINDS +async def test_context_window_fallbacks_from_a_changeset_are_used_by_the_next_call( + make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet] +) -> None: + core = make_core(model_list=[deployment("gpt", "a"), deployment("long", "l")], num_retries=0) + host.answer("openai/a", _fresh(context_window)) + with pytest.raises(litellm.ContextWindowExceededError): + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + host.calls.clear() + + core.apply(changeset(context_window_fallbacks=[{"gpt": ["long"]}])) + response = await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert host.deployments_called("acompletion") == ["a", "l"] + assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "long" + assert core.settings().context_window_fallbacks == [{"gpt": ["long"]}] + + +@pytest.mark.asyncio +@CHANGESET_KINDS +async def test_timeout_from_a_changeset_is_passed_on_the_next_call( + make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet] +) -> None: + core = make_core(model_list=[deployment("gpt", "a")], timeout=30) + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + core.apply(changeset(timeout=12)) + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert [call.kwargs["timeout"] for call in host.calls_to("acompletion")] == [30, 12] + assert core.settings().as_dict()["timeout"] == 12 + + +@pytest.mark.asyncio +@CHANGESET_KINDS +async def test_model_group_alias_from_a_changeset_routes_the_next_call( + make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet] +) -> None: + core = make_core(model_list=[deployment("gpt", "a"), deployment("emb", "e")], model_group_alias={"fast": "gpt"}) + + core.apply(changeset(model_group_alias={"fast": "emb"})) + await core.acompletion(model="fast", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert host.deployments_called("acompletion") == ["e"] + assert core.registry().aliases() == {"fast": "emb"} + assert core.registry().resolve("fast") == "emb" + + +@pytest.mark.asyncio +@CHANGESET_KINDS +async def test_allowed_fails_and_cooldown_time_from_a_changeset_govern_the_next_cooldown( + make_core: MakeCore, changeset: Callable[..., ChangeSet] +) -> None: + core = make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")]) + assert core.health().record_failure("a", server_error(), 500) is False + + core.apply(changeset(allowed_fails=0, cooldown_time=42)) + + assert core.health().record_failure("b", server_error(), 500) is True + assert [ + (i, entry["cooldown_time"]) for i, entry in core.health().cooldowns.get_active_cooldowns(["a", "b"], None) + ] == [("b", 42)] + assert (core.settings().allowed_fails, core.settings().cooldown_time) == (0, 42) + + +@pytest.mark.asyncio +async def test_a_deployment_cooled_down_under_applied_allowed_fails_is_not_picked_again( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")], num_retries=0) + core.apply(ChangeSet(assign={"allowed_fails": 0, "cooldown_time": 60})) + host.answer("*", _fresh(server_error)) + + for _ in range(2): + with pytest.raises(litellm.InternalServerError): + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert sorted(host.deployments_called("acompletion")) == ["a", "b"] + + +@pytest.mark.asyncio +@CHANGESET_KINDS +async def test_tag_filtering_switched_on_by_a_changeset_restricts_the_next_pick( + make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet] +) -> None: + core = make_core(model_list=[deployment("gpt", "a", tags=["paid"]), deployment("gpt", "b", tags=["free"])]) + tagged = {"metadata": {"tags": ["paid"]}} + assert sorted(_ids(await core.select("gpt", dict(tagged), kind="healthy"))) == ["a", "b"] # pyright: ignore[reportCallIssue] # select takes kind + + core.apply(changeset(enable_tag_filtering=True)) + + assert _ids(await core.select("gpt", dict(tagged), kind="healthy")) == ["a"] # pyright: ignore[reportCallIssue] # select takes kind + for _ in range(3): + await core.acompletion(model="gpt", messages=MESSAGES, metadata={"tags": ["paid"]}) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + assert host.deployments_called("acompletion") == ["a", "a", "a"] + assert core.settings().enable_tag_filtering is True + + +@pytest.mark.asyncio +async def test_routing_strategy_from_settings_is_reported_and_still_routes( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = make_core(model_list=[deployment("gpt", "a")]) + strategy = "least-busy" + + core.apply(ChangeSet(settings={"routing_strategy": strategy})) + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert core.settings().routing_strategy == strategy + assert core.settings().as_dict()["routing_strategy"] == strategy + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.asyncio +async def test_default_litellm_params_from_a_changeset_are_sent_on_the_next_call( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = make_core(model_list=[deployment("gpt", "a")], default_litellm_params={"temperature": 0.2}) + + core.apply(ChangeSet(assign={"default_litellm_params": {"temperature": 0.7}})) + await core.acompletion(model="gpt", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert [call.kwargs.get("temperature") for call in host.calls_to("acompletion")] == [0.7] + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("cooldown_time", 7.5), + ("allowed_fails", 9), + ("disable_cooldowns", True), + ("content_policy_fallbacks", [{"gpt": ["emb"]}]), + ], +) +def test_settings_reflect_an_assigned_field(make_core: MakeCore, name: str, value: object) -> None: + core = make_core(model_list=_model_list()) + + core.apply(ChangeSet(assign={name: value})) + + assert getattr(core.settings(), name) == value + assert core.settings().is_set(name) + + +# -- what apply refuses ------------------------------------------------------------------------- + + +@pytest.mark.parametrize("name", ["bogus", "model_list", "_core_num_retries"]) +def test_assigning_an_unknown_setting_raises_and_applies_nothing(make_core: MakeCore, name: str) -> None: + core = make_core(model_list=_model_list(), num_retries=1) + settings_before = core.settings() + + with pytest.raises(ValueError, match=name): + core.apply(ChangeSet(assign={"num_retries": 5, "cooldown_time": 3, name: 1}, upsert=[_deployment("gpt", "c")])) + + assert core.settings().as_dict()["num_retries"] == 1 + assert core.settings() is settings_before + assert not core.registry().has_deployment("c") + + +def test_apply_refuses_refresh_model_info_and_applies_nothing(make_core: MakeCore) -> None: + core = make_core(model_list=_model_list(), num_retries=1) + version_before = core.registry().version + + with pytest.raises(TypeError, match="aapply"): + core.apply(ChangeSet(refresh_model_info=True, assign={"num_retries": 5}, delete=["a"])) + + assert core.settings().as_dict()["num_retries"] == 1 + assert core.registry().version == version_before + assert core.registry().has_deployment("a") + + +# -- the settings() snapshot -------------------------------------------------------------------- + + +def test_settings_snapshot_is_reused_between_reads(make_core: MakeCore) -> None: + core = make_core(model_list=_model_list()) + + assert core.settings() is core.settings() + + +@CHANGESET_KINDS +def test_applying_a_setting_replaces_the_snapshot_and_leaves_the_old_one_as_it_was( + make_core: MakeCore, changeset: Callable[..., ChangeSet] +) -> None: + core = make_core(model_list=_model_list(), cooldown_time=5) + before = core.settings() + + core.apply(changeset(cooldown_time=11)) + + after = core.settings() + assert after is not before + assert (before.cooldown_time, after.cooldown_time) == (5, 11) + assert core.settings() is after + + +@pytest.mark.parametrize( + "changes", + [ChangeSet(upsert=[_deployment("gpt", "c")]), ChangeSet(delete=["a"])], + ids=["upsert", "delete"], +) +def test_a_changeset_without_settings_keeps_the_snapshot(make_core: MakeCore, changes: ChangeSet) -> None: + core = make_core(model_list=_model_list()) + before = core.settings() + + core.apply(changes) + + assert core.settings() is before + + +# -- a direct attribute write is a changeset ---------------------------------------------------- + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("num_retries", 4), + ("timeout", 9.0), + ("retry_after", 3), + ("allowed_fails", 1), + ("cooldown_time", 11.0), + ("fallbacks", [{"gpt": ["emb"]}]), + ("context_window_fallbacks", [{"gpt": ["emb"]}]), + ("enable_tag_filtering", True), + ("model_group_alias", {"fast": "emb"}), + ], +) +def test_writing_a_setting_attribute_is_the_same_as_assigning_it_in_a_changeset( + make_core: MakeCore, name: str, value: object +) -> None: + written = make_core(model_list=_model_list()) + applied = make_core(model_list=_model_list()) + snapshot_before = written.settings() + version_before = written.registry().version + + setattr(written, name, copy.deepcopy(value)) + applied.apply(ChangeSet(assign={name: copy.deepcopy(value)})) + + assert written.settings().as_dict() == applied.settings().as_dict() + assert written.registry().aliases() == applied.registry().aliases() + if written.settings().is_set(name): + assert getattr(written.settings(), name) == getattr(applied.settings(), name) == value + assert written.settings() is not snapshot_before + assert written.registry().version == version_before + + +# -- constructor arguments ---------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("model_list", _model_list()), + ("fallbacks", [{"gpt": ["emb"]}]), + ("model_group_alias", {"fast": "gpt"}), + ("model_group_retry_policy", {"gpt": {"RateLimitErrorRetries": 1}}), + ("allowed_fails_policy", {"RateLimitErrorAllowedFails": 2}), + ], +) +def test_construction_leaves_the_arguments_it_was_given_unchanged( + make_core: MakeCore, name: str, value: object +) -> None: + given = copy.deepcopy(value) + arguments: dict[str, object] = {"model_list": _model_list(), name: given} + + make_core(**arguments) + + assert given == value + + +@pytest.mark.asyncio +async def test_mutating_the_model_list_after_construction_changes_nothing( + make_core: MakeCore, host: ScriptedHost +) -> None: + model_list = _model_list() + core = make_core(model_list=model_list) + version_before = core.registry().version + + model_list.append(deployment("gpt", "late")) + model_list[2]["model_name"] = "gpt" + model_list[0]["litellm_params"]["model"] = "openai/elsewhere" # pyright: ignore[reportIndexIssue] # a model_list entry + model_list[0]["model_info"]["id"] = "renamed" # pyright: ignore[reportIndexIssue] # a model_list entry + await core.aembedding(model="emb", input=["x"]) # pyright: ignore[reportAttributeAccessIssue] # an endpoint method + + assert core.registry().names(kind="ids") == ["a", "b", "e"] + assert core.registry().names(kind="group_set") == {"gpt", "emb"} + assert core.registry().version == version_before + assert [(call.model, call.deployment_id) for call in host.calls_to("aembedding")] == [("openai/e", "e")] + + +# -- retire -------------------------------------------------------------------------------------- + + +def _registered(host: ScriptedHost) -> Mapping[str, int]: + return {name: len(callbacks) for name, callbacks in host.lists.items() if callbacks} + + +def test_retire_releases_every_callback_the_core_registered_on_its_host( + make_core: MakeCore, host: ScriptedHost +) -> None: + core = make_core(model_list=_model_list()) + assert _registered(host) != {} + + core.retire() + + assert _registered(host) == {} + + +def test_retire_leaves_another_cores_callbacks_registered(make_core: MakeCore, host: ScriptedHost) -> None: + kept = make_core(model_list=_model_list()) + retired = make_core(model_list=[deployment("other", "o")]) + registered_by_both = _registered(host) + + retired.retire() + + assert _registered(host) != {} + assert sum(_registered(host).values()) < sum(registered_by_both.values()) + kept.retire() + assert _registered(host) == {} diff --git a/tests/unit/test_router/core/test_retries_cooldowns.py b/tests/unit/test_router/core/test_retries_cooldowns.py new file mode 100644 index 00000000000..004b1762240 --- /dev/null +++ b/tests/unit/test_router/core/test_retries_cooldowns.py @@ -0,0 +1,832 @@ +""" +Retries and cooldowns of a router core, driven through its public surface on a ``ScriptedHost``. + +Backoff values carry litellm's random jitter (up to ``JITTER`` seconds), so sleeps are asserted +inside ``[base, base + JITTER)`` rather than exactly. +""" + +from collections.abc import Callable, Mapping +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.constants import DEFAULT_COOLDOWN_TIME_SECONDS, INITIAL_RETRY_DELAY, JITTER +from litellm.router_api import ChangeSet +from litellm.router_core import RouterCore +from litellm.types.router import AllowedFailsPolicy, RetryPolicy +from tests.unit.test_router.core.scripted_host import ( + Call, + ScriptedHost, + auth_error, + bad_request, + content_policy, + context_window, + deployment, + rate_limit, + server_error, + timeout, +) + +MESSAGES: Final = [{"role": "user", "content": "hi"}] + +MakeCore = Callable[..., RouterCore] +ErrorBuilder = Callable[[], Exception] + + +class LitellmParamsLoggingHost(ScriptedHost): + """``ScriptedHost`` whose failure logging carries the call's ``cooldown_time`` in ``litellm_params``, as + litellm's own logging does (``get_litellm_params``).""" + + def _logging_kwargs( + self, call: Call, error: BaseException | None, response: object + ) -> dict[str, object]: # mutable-ok: litellm's logging kwargs + logged: Final = super()._logging_kwargs(call, error, response) + litellm_params: Final = logged["litellm_params"] + if isinstance(litellm_params, dict) and "cooldown_time" in call.kwargs: + litellm_params["cooldown_time"] = call.kwargs["cooldown_time"] + return logged + + +@pytest.fixture +def host() -> ScriptedHost: + return LitellmParamsLoggingHost() + + +def fresh(builder: ErrorBuilder) -> Callable[..., Exception]: + """A scripted outcome that raises a new error per call, so stamps from one call never leak into the next.""" + return lambda **_: builder() + + +def rate_limited(retry_after: str | None = None) -> litellm.RateLimitError: + headers: Final = {"retry-after": retry_after} if retry_after is not None else {} + return litellm.RateLimitError( + message="slow down", llm_provider="openai", model="gpt-4o", response=httpx.Response(429, headers=headers) + ) + + +async def attempt(core: RouterCore, model: str = "g", **kwargs: object) -> object: # kwargs-ok: request kwargs + """The response, or the error the core raised.""" + try: + return await core.acompletion(model=model, messages=MESSAGES, **kwargs) # pyright: ignore[reportAttributeAccessIssue] # every core serves acompletion + except Exception as error: + return error + + +def cooled_down(core: RouterCore, ids: list[str]) -> dict[str, Mapping[str, object]]: # mutable-ok: a fresh dict + return dict(core.health().cooldowns.get_active_cooldowns(ids, None)) + + +def headers_of(response: object) -> Mapping[str, object]: + hidden: Final = getattr(response, "_hidden_params", None) or {} + return hidden.get("additional_headers") or {} + + +def two_deployments( + **params: object, +) -> list[dict[str, object]]: # kwargs-ok: litellm_params # mutable-ok: a model_list + return [deployment("g", "a", **params), deployment("g", "b", **params)] + + +# --- how many retries --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("constructor", "request_value", "process", "expected_retries"), + [ + (None, None, None, 2), + (3, None, None, 3), + (None, None, 4, 4), + (1, None, 4, 1), + (3, 1, None, 1), + (3, 0, None, 0), + (None, 2, 5, 2), + (0, 2, None, 2), + ], +) +@pytest.mark.asyncio +async def test_num_retries_comes_from_request_then_constructor_then_process_then_default( + make_core: MakeCore, + host: ScriptedHost, + constructor: int | None, + request_value: int | None, + process: int | None, + expected_retries: int, +) -> None: + if process is not None: + host.set_dynamic(num_retries=process) + arguments: Final = {"num_retries": constructor} if constructor is not None else {} + core: Final = make_core(model_list=[deployment("g", "a")], **arguments) + host.answer("openai/a", fresh(server_error)) + request: Final = {"num_retries": request_value} if request_value is not None else {} + + error: Final = await attempt(core, **request) + + assert isinstance(error, litellm.InternalServerError) + assert host.deployments_called("acompletion") == ["a"] * (expected_retries + 1) + + +@pytest.mark.asyncio +async def test_process_num_retries_is_read_once_when_the_core_is_built(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + host.set_dynamic(num_retries=5) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + assert len(host.calls_to("acompletion")) == 3 + + +@pytest.mark.asyncio +async def test_num_retries_applied_through_settings_changes_later_requests( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3) + core.apply(ChangeSet(settings={"num_retries": 1})) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + assert len(host.calls_to("acompletion")) == 2 + + +# --- which errors retry ------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("builder", "expected_calls"), + [ + (bad_request, 1), + (auth_error, 1), + (context_window, 1), + (content_policy, 1), + (rate_limit, 3), + (server_error, 3), + (timeout, 3), + ], +) +@pytest.mark.asyncio +async def test_lone_deployment_retries_only_retryable_errors( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder, expected_calls: int +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2) + host.answer("openai/a", fresh(builder)) + + error: Final = await attempt(core) + + assert type(error) is type(builder()) + assert len(host.calls_to("acompletion")) == expected_calls + + +@pytest.mark.parametrize( + ("builder", "expected_calls"), + [(bad_request, 1), (context_window, 1), (content_policy, 1), (server_error, 3), (timeout, 3)], +) +@pytest.mark.asyncio +async def test_two_deployment_group_retries_only_retryable_errors( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder, expected_calls: int +) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=2) + host.answer("*", fresh(builder)) + + error: Final = await attempt(core) + + assert type(error) is type(builder()) + assert len(host.calls_to("acompletion")) == expected_calls + + +@pytest.mark.asyncio +async def test_auth_error_retries_on_the_other_deployment_of_the_group(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=2) + host.answer("*", fresh(auth_error)) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.AuthenticationError) + assert sorted(host.deployments_called("acompletion")) == ["a", "b"] + + +@pytest.mark.asyncio +async def test_rate_limits_stop_retrying_once_every_deployment_is_cooling_down( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=5) + host.answer("*", fresh(rate_limit)) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RateLimitError) + assert sorted(host.deployments_called("acompletion")) == ["a", "b"] + assert set(cooled_down(core, ["a", "b"])) == {"a", "b"} + + +@pytest.mark.parametrize("builder", [bad_request, auth_error, context_window, content_policy]) +@pytest.mark.asyncio +async def test_an_error_raised_without_retrying_carries_no_retry_counts( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2) + host.answer("openai/a", fresh(builder)) + + error: Final = await attempt(core) + + assert getattr(error, "num_retries", None) is None + assert getattr(error, "max_retries", None) is None + + +@pytest.mark.parametrize("builder", [rate_limit, server_error, timeout]) +@pytest.mark.parametrize("num_retries", [1, 3]) +@pytest.mark.asyncio +async def test_exhausted_retries_stamp_num_retries_and_max_retries_on_the_final_error( + make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder, num_retries: int +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=num_retries) + host.answer("openai/a", fresh(builder)) + + error: Final = await attempt(core) + + assert getattr(error, "num_retries", None) == num_retries + assert getattr(error, "max_retries", None) == num_retries + assert f"LiteLLM Retried: {num_retries} times" in str(error) + + +@pytest.mark.asyncio +async def test_the_final_error_is_the_latest_one_not_the_first(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2) + first: Final = server_error() + last: Final = timeout() + host.answer("openai/a", first, fresh(server_error), last) + + error: Final = await attempt(core) + + assert error is host.raised[-1] + assert type(error) is type(last) + + +# --- retry policies ----------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("field", "builder", "retries"), + [ + ("BadRequestErrorRetries", bad_request, 2), + ("BadRequestErrorRetries", context_window, 1), + ("ContentPolicyViolationErrorRetries", content_policy, 1), + ("RateLimitErrorRetries", rate_limit, 0), + ("TimeoutErrorRetries", timeout, 1), + ("InternalServerErrorRetries", server_error, 4), + ("DefaultRetries", server_error, 1), + ], +) +@pytest.mark.asyncio +async def test_retry_policy_sets_retries_per_error_type( + make_core: MakeCore, host: ScriptedHost, field: str, builder: ErrorBuilder, retries: int +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], num_retries=3, retry_policy=RetryPolicy(**{field: retries}) + ) + host.answer("openai/a", fresh(builder)) + + error: Final = await attempt(core) + + assert type(error) is type(builder()) + assert len(host.calls_to("acompletion")) == retries + 1 + + +@pytest.mark.asyncio +async def test_retry_policy_given_as_a_dict_works_like_a_retry_policy(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], num_retries=3, retry_policy={"BadRequestErrorRetries": 1} + ) + host.answer("openai/a", fresh(bad_request)) + + await attempt(core) + + assert len(host.calls_to("acompletion")) == 2 + + +@pytest.mark.asyncio +async def test_retry_policy_for_one_error_type_leaves_others_to_num_retries( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], num_retries=1, retry_policy=RetryPolicy(RateLimitErrorRetries=4) + ) + host.answer("openai/a", fresh(timeout)) + + await attempt(core) + + assert len(host.calls_to("acompletion")) == 2 + + +@pytest.mark.asyncio +async def test_model_group_retry_policy_applies_only_to_its_group(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("h", "c")], + num_retries=2, + model_group_retry_policy={"g": RetryPolicy(InternalServerErrorRetries=0)}, + ) + host.answer("*", fresh(server_error)) + + await attempt(core, model="g") + await attempt(core, model="h") + + assert host.deployments_called("acompletion") == ["a", "c", "c", "c"] + + +@pytest.mark.asyncio +async def test_model_group_retry_policy_wins_over_retry_policy(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], + retry_policy=RetryPolicy(InternalServerErrorRetries=4), + model_group_retry_policy={"g": RetryPolicy(InternalServerErrorRetries=1)}, + ) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + assert len(host.calls_to("acompletion")) == 2 + + +@pytest.mark.parametrize(("request_value", "expected_calls"), [(4, 2), (0, 1)]) +@pytest.mark.asyncio +async def test_retry_policy_wins_over_request_num_retries_unless_the_request_opts_out( + make_core: MakeCore, host: ScriptedHost, request_value: int, expected_calls: int +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], retry_policy=RetryPolicy(InternalServerErrorRetries=1)) + host.answer("openai/a", fresh(server_error)) + + await attempt(core, num_retries=request_value) + + assert len(host.calls_to("acompletion")) == expected_calls + + +# --- per-deployment num_retries ----------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("deployment_value", "request_kwargs", "expected_calls"), + [ + (1, {}, 2), + ("1", {}, 2), + (0, {}, 1), + (4, {}, 5), + (4, {"num_retries": 0}, 1), + (0, {"num_retries": 2}, 3), + ], +) +@pytest.mark.asyncio +async def test_deployment_num_retries_beats_the_router_but_not_the_request( + make_core: MakeCore, + host: ScriptedHost, + deployment_value: object, + request_kwargs: dict[str, object], + expected_calls: int, +) -> None: + core: Final = make_core(model_list=[deployment("g", "a", num_retries=deployment_value)], num_retries=3) + host.answer("openai/a", fresh(server_error)) + + error: Final = await attempt(core, **request_kwargs) + + assert isinstance(error, litellm.InternalServerError) + assert len(host.calls_to("acompletion")) == expected_calls + + +# --- backoff ------------------------------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_lone_deployment_backs_off_exponentially(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + bases: Final = [INITIAL_RETRY_DELAY * 2**n for n in range(3)] + assert len(host.sleeps) >= 3 + assert all(base <= slept < base + JITTER for base, slept in zip(bases, host.sleeps)) + + +@pytest.mark.asyncio +async def test_retry_after_setting_is_the_minimum_backoff(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2, retry_after=3) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + assert len(host.sleeps) >= 2 + assert all(3 <= slept < 3 + JITTER for slept in host.sleeps[:2]) + + +@pytest.mark.asyncio +async def test_lone_deployment_waits_the_retry_after_header(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2) + host.answer("openai/a", lambda **_: rate_limited(retry_after="7")) + + await attempt(core) + + assert len(host.sleeps) >= 2 + assert all(7 <= slept < 7 + JITTER for slept in host.sleeps[:2]) + + +@pytest.mark.asyncio +async def test_lone_deployment_ignores_a_retry_after_header_over_a_minute( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2) + host.answer("openai/a", lambda **_: rate_limited(retry_after="120")) + + await attempt(core) + + assert INITIAL_RETRY_DELAY <= host.sleeps[0] < INITIAL_RETRY_DELAY + JITTER + + +@pytest.mark.parametrize("failure", [fresh(server_error), fresh(timeout), lambda **_: rate_limited(retry_after="7")]) +@pytest.mark.asyncio +async def test_a_group_with_siblings_retries_immediately( + make_core: MakeCore, host: ScriptedHost, failure: Callable[..., Exception] +) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=2) + host.answer("*", failure, None) + + response: Final = await attempt(core) + + assert not isinstance(response, Exception) + assert host.sleeps == [0] + + +@pytest.mark.asyncio +async def test_no_sleep_when_the_error_is_not_retried(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3) + host.answer("openai/a", fresh(bad_request)) + + await attempt(core) + + assert host.sleeps == [] + + +# --- retry headers on success ------------------------------------------------------------------- + + +@pytest.mark.parametrize("failures_first", [1, 2, 3]) +@pytest.mark.asyncio +async def test_success_after_retries_reports_attempted_and_max_retries( + make_core: MakeCore, host: ScriptedHost, failures_first: int +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3) + host.answer("openai/a", *[fresh(server_error)] * failures_first, None) + + response: Final = await attempt(core) + + headers: Final = headers_of(response) + assert headers["x-litellm-attempted-retries"] == failures_first + assert headers["x-litellm-max-retries"] == 3 + assert headers["x-litellm-model-group"] == "g" + + +@pytest.mark.asyncio +async def test_first_try_success_reports_zero_retries_and_no_max(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3) + + response: Final = await attempt(core) + + headers: Final = headers_of(response) + assert headers["x-litellm-attempted-retries"] == 0 + assert "x-litellm-max-retries" not in headers + + +@pytest.mark.asyncio +async def test_success_headers_report_the_retry_policy_budget(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], num_retries=1, retry_policy=RetryPolicy(InternalServerErrorRetries=4) + ) + host.answer("openai/a", fresh(server_error), fresh(server_error), None) + + response: Final = await attempt(core) + + assert headers_of(response)["x-litellm-attempted-retries"] == 2 + assert headers_of(response)["x-litellm-max-retries"] == 4 + + +# --- allowed fails and cooldowns ---------------------------------------------------------------- + + +@pytest.mark.parametrize("allowed_fails", [0, 1, 2]) +@pytest.mark.asyncio +async def test_a_deployment_cools_down_once_its_failures_pass_allowed_fails( + make_core: MakeCore, host: ScriptedHost, allowed_fails: int +) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=allowed_fails) + host.answer("*", fresh(server_error)) + + for _ in range(2 * allowed_fails + 2): + await attempt(core) + + called: Final = host.deployments_called("acompletion") + cooled: Final = cooled_down(core, ["a", "b"]) + assert cooled + assert {d for d in ("a", "b") if called.count(d) > allowed_fails} == set(cooled) + + +@pytest.mark.asyncio +async def test_process_allowed_fails_is_the_default(make_core: MakeCore, host: ScriptedHost) -> None: + host.set_dynamic(allowed_fails=0) + core: Final = make_core(model_list=two_deployments(), num_retries=0) + host.answer("*", fresh(server_error)) + + await attempt(core) + + assert core.settings().allowed_fails == 0 + assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion") + + +@pytest.mark.asyncio +async def test_a_cooled_deployment_is_skipped_by_later_picks(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0) + host.answer("*", fresh(server_error)) + + await attempt(core) + first: Final = host.deployments_called("acompletion")[0] + healthy: Final = await core.select("g", {}, kind="healthy") # pyright: ignore[reportAttributeAccessIssue] # every core serves select + await attempt(core) + + assert [d["model_info"]["id"] for d in healthy] == [{"a": "b", "b": "a"}[first]] + assert host.deployments_called("acompletion")[1] != first + + +@pytest.mark.asyncio +async def test_a_group_with_every_deployment_cooling_down_raises_without_calling( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0) + host.answer("*", fresh(server_error)) + await attempt(core) + await attempt(core) + + error: Final = await attempt(core) + + assert isinstance(error, litellm.RouterRateLimitError) + assert len(host.calls_to("acompletion")) == 2 + + +@pytest.mark.asyncio +async def test_rate_limit_cools_down_a_deployment_with_siblings_at_once( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=0) + host.answer("*", fresh(rate_limit)) + + await attempt(core) + + assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion") + + +@pytest.mark.parametrize( + "policy", + [AllowedFailsPolicy(RateLimitErrorAllowedFails=2), {"RateLimitErrorAllowedFails": 2}], + ids=["AllowedFailsPolicy", "dict"], +) +@pytest.mark.asyncio +async def test_allowed_fails_policy_lets_rate_limits_through_until_its_count( + make_core: MakeCore, host: ScriptedHost, policy: object +) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails_policy=policy) + host.answer("*", fresh(rate_limit)) + + await attempt(core) + + assert cooled_down(core, ["a", "b"]) == {} + + +@pytest.mark.asyncio +async def test_allowed_fails_policy_of_zero_cools_down_on_the_first_failure_of_that_type( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=two_deployments(), + num_retries=0, + allowed_fails=5, + allowed_fails_policy=AllowedFailsPolicy(TimeoutErrorAllowedFails=0), + ) + host.answer("*", fresh(server_error), fresh(timeout)) + + await attempt(core) + after_server_error: Final = cooled_down(core, ["a", "b"]) + await attempt(core) + + assert after_server_error == {} + assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion")[1:] + + +@pytest.mark.asyncio +async def test_deployment_allowed_fails_policy_in_model_info_cools_down_on_a_bad_request( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=two_deployments(model_info={"allowed_fails_policy": {"BadRequestErrorAllowedFails": 0}}), + num_retries=0, + ) + host.answer("*", fresh(bad_request)) + + await attempt(core) + + assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion") + + +@pytest.mark.asyncio +async def test_bad_request_does_not_cool_down_without_a_deployment_policy( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0) + host.answer("*", fresh(bad_request)) + + await attempt(core) + + assert cooled_down(core, ["a", "b"]) == {} + + +# --- cooldown time ------------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + ("arguments", "params", "failure", "expected_seconds"), + [ + ({}, {}, fresh(rate_limit), DEFAULT_COOLDOWN_TIME_SECONDS), + ({"cooldown_time": 42}, {}, fresh(rate_limit), 42), + ({}, {"cooldown_time": 30}, fresh(rate_limit), 30), + ({}, {"model_info": {"cooldown_time": 30}}, fresh(rate_limit), 30), + ({}, {}, lambda **_: rate_limited(retry_after="9"), 9), + ({"cooldown_time": 42}, {}, lambda **_: rate_limited(retry_after="9"), 9), + ({}, {"cooldown_time": 30}, lambda **_: rate_limited(retry_after="9"), 30), + ], + ids=[ + "default", + "constructor", + "litellm_params", + "model_info", + "retry-after", + "header-over-router", + "deployment-over-header", + ], +) +@pytest.mark.asyncio +async def test_cooldown_time_comes_from_deployment_then_retry_after_then_router( + make_core: MakeCore, + host: ScriptedHost, + arguments: dict[str, object], + params: dict[str, object], + failure: Callable[..., Exception], + expected_seconds: float, +) -> None: + core: Final = make_core(model_list=two_deployments(**params), num_retries=0, **arguments) + host.answer("*", failure) + + await attempt(core) + + (entry,) = cooled_down(core, ["a", "b"]).values() + assert entry["cooldown_time"] == expected_seconds + + +@pytest.mark.parametrize("change", [{"settings": {"cooldown_time": 20}}, {"assign": {"cooldown_time": 20}}]) +@pytest.mark.asyncio +async def test_cooldown_time_applied_at_runtime_is_used_for_later_cooldowns( + make_core: MakeCore, host: ScriptedHost, change: dict[str, Mapping[str, object]] +) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=0) + core.apply(ChangeSet(**change)) # pyright: ignore[reportArgumentType] # ChangeSet's mapping fields + host.answer("*", fresh(rate_limit)) + + await attempt(core) + + assert core.settings().cooldown_time == 20 + (entry,) = cooled_down(core, ["a", "b"]).values() + assert entry["cooldown_time"] == 20 + + +@pytest.mark.asyncio +async def test_a_deployment_cooldown_time_of_zero_skips_the_cooldown(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=two_deployments(model_info={"cooldown_time": 0}), num_retries=0) + host.answer("*", fresh(rate_limit)) + + await attempt(core) + + assert cooled_down(core, ["a", "b"]) == {} + + +# --- disable_cooldowns -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("failure", [fresh(rate_limit), fresh(server_error)]) +@pytest.mark.asyncio +async def test_disable_cooldowns_keeps_failing_deployments_routable( + make_core: MakeCore, host: ScriptedHost, failure: Callable[..., Exception] +) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0, disable_cooldowns=True) + host.answer("*", failure) + + for _ in range(4): + await attempt(core) + + assert cooled_down(core, ["a", "b"]) == {} + assert len(host.calls_to("acompletion")) == 4 + + +@pytest.mark.asyncio +async def test_disable_cooldowns_assigned_at_runtime_stops_cooldowns(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=0) + core.apply(ChangeSet(assign={"disable_cooldowns": True})) + host.answer("*", fresh(rate_limit)) + + await attempt(core) + + assert core.settings().disable_cooldowns is True + assert cooled_down(core, ["a", "b"]) == {} + + +# --- single-deployment groups ------------------------------------------------------------------- + + +@pytest.mark.parametrize("arguments", [{}, {"allowed_fails": 0}], ids=["default", "allowed_fails=0"]) +@pytest.mark.parametrize("builder", [rate_limit, server_error, timeout]) +@pytest.mark.asyncio +async def test_a_single_deployment_group_is_not_cooled_down( + make_core: MakeCore, host: ScriptedHost, arguments: dict[str, object], builder: ErrorBuilder +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=0, **arguments) + host.answer("openai/a", fresh(builder)) + + for _ in range(3): + await attempt(core) + + assert cooled_down(core, ["a"]) == {} + assert host.deployments_called("acompletion") == ["a", "a", "a"] + + +@pytest.mark.asyncio +async def test_an_allowed_fails_policy_naming_the_error_cools_down_a_single_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a")], + num_retries=0, + allowed_fails_policy=AllowedFailsPolicy(InternalServerErrorAllowedFails=0), + ) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + error: Final = await attempt(core) + + assert set(cooled_down(core, ["a"])) == {"a"} + assert isinstance(error, litellm.RouterRateLimitError) + assert host.deployments_called("acompletion") == ["a"] + + +# --- health() ----------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_health_cooldown_entry_describes_the_failure(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=two_deployments(), num_retries=0) + host.answer("*", fresh(rate_limit)) + + await attempt(core) + + (entry,) = cooled_down(core, ["a", "b"]).values() + assert entry["status_code"] == "429" + assert "slow down" in str(entry["exception_received"]) + + +@pytest.mark.parametrize("failures", [1, 3]) +@pytest.mark.asyncio +async def test_health_counts_failures_this_minute(make_core: MakeCore, host: ScriptedHost, failures: int) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=0) + host.answer("openai/a", fresh(server_error)) + + for _ in range(failures): + await attempt(core) + + assert core.health().failures_this_minute("a") == failures + assert core.health().failures_this_minute("missing") == 0 + + +@pytest.mark.asyncio +async def test_health_counts_every_retried_attempt_as_a_failure(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2) + host.answer("openai/a", fresh(server_error)) + + await attempt(core) + + assert core.health().failures_this_minute("a") == 3 + + +@pytest.mark.asyncio +async def test_health_record_failure_cools_down_past_allowed_fails(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=two_deployments(), allowed_fails=1) + + first: Final = core.health().record_failure("a", server_error(), 500) + second: Final = core.health().record_failure("a", server_error(), 500) + + assert (first, second) == (False, True) + assert set(cooled_down(core, ["a", "b"])) == {"a"} + assert core.health().failures_this_minute("a") == 2 diff --git a/tests/unit/test_router/core/test_selection.py b/tests/unit/test_router/core/test_selection.py new file mode 100644 index 00000000000..4b4c320c428 --- /dev/null +++ b/tests/unit/test_router/core/test_selection.py @@ -0,0 +1,624 @@ +""" +Deployment selection on a router core: which deployment ``select`` picks and which one an endpoint +call reaches, across groups, aliases, wildcard routes, deployment ids, teams, weights, order, +routing strategies and cooldowns. +""" + +import re +from collections.abc import Callable, Mapping +from typing import Final + +import openai +import pytest + +import litellm +from litellm.router_core import RouterCore +from litellm.types.router import RouterRateLimitError +from tests.unit.test_router.core.scripted_host import ( + ScriptedHost, + auth_error, + bad_request, + context_window, + deployment, + rate_limit, + server_error, + timeout, +) + +MESSAGES: Final = [{"role": "user", "content": "hello there, router"}] +PICKS: Final = 20 + +MakeCore = Callable[..., RouterCore] + + +async def _picked_ids( + core: RouterCore, model: str, times: int = PICKS, **select_arguments: object +) -> set[str]: # kwargs-ok: select's keyword arguments + picked: Final[set[str]] = set() + for _ in range(times): + chosen = await core.select(model, **select_arguments) # pyright: ignore[reportCallIssue] # PythonCore's select takes keyword options + assert isinstance(chosen, Mapping) + picked.add(str(chosen["model_info"]["id"])) + return picked + + +async def _fail_through(core: RouterCore, deployment_id: str, times: int = 1) -> None: + for _ in range(times): + with pytest.raises(openai.APIError): + await core.acompletion(model=deployment_id, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method + + +# --- picking within a model group ---------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("model", "expected"), [("g1", "a"), ("g2", "b"), ("g3", "c")]) +async def test_select_returns_the_only_deployment_of_the_requested_group( + make_core: MakeCore, model: str, expected: str +) -> None: + core: Final = make_core(model_list=[deployment("g1", "a"), deployment("g2", "b"), deployment("g3", "c")]) + chosen: Final = await core.select(model) + assert chosen["model_info"]["id"] == expected # pyright: ignore[reportIndexIssue] + assert chosen["model_name"] == model # pyright: ignore[reportIndexIssue] + assert chosen["litellm_params"]["model"] == f"openai/{expected}" # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_select_only_picks_deployments_of_the_requested_group(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b"), deployment("other", "c"), deployment("other", "d")] + ) + assert await _picked_ids(core, "g") <= {"a", "b"} + assert await _picked_ids(core, "other") <= {"c", "d"} + + +@pytest.mark.asyncio +async def test_select_does_not_call_the_provider(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + await core.select("g") + assert host.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("op", "request_payload"), + [ + ("acompletion", {"messages": MESSAGES}), + ("atext_completion", {"prompt": "hi"}), + ("aembedding", {"input": "hi"}), + ], +) +async def test_async_endpoint_calls_the_deployment_of_the_requested_group( + make_core: MakeCore, host: ScriptedHost, op: str, request_payload: dict[str, object] +) -> None: + core: Final = make_core(model_list=[deployment("g1", "a"), deployment("g2", "b")]) + await core.call(op, {"model": "g2", **request_payload}) + assert host.deployments_called(op) == ["b"] + assert host.calls[-1].model == "openai/b" + + +@pytest.mark.parametrize( + ("op", "request_payload"), [("completion", {"messages": MESSAGES}), ("embedding", {"input": "hi"})] +) +def test_sync_endpoint_calls_the_deployment_of_the_requested_group( + make_core: MakeCore, host: ScriptedHost, op: str, request_payload: dict[str, object] +) -> None: + core: Final = make_core(model_list=[deployment("g1", "a"), deployment("g2", "b")]) + getattr(core, op)(model="g1", **request_payload) + assert host.deployments_called(op) == ["a"] + + +@pytest.mark.asyncio +async def test_completion_reports_the_requested_group_in_its_headers(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g1", "a"), deployment("g2", "b")]) + response: Final = await core.acompletion(model="g2", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "g2" + assert host.calls[-1].model_group == "g2" + + +# --- model_group_alias --------------------------------------------------------------------- + +ALIASES: Final = { + "plain": "g", + "visible": {"model": "g", "hidden": False}, + "hidden": {"model": "g", "hidden": True}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("alias", sorted(ALIASES)) +async def test_select_resolves_an_alias_to_its_target_group(make_core: MakeCore, alias: str) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("other", "b")], model_group_alias=dict(ALIASES) + ) + chosen: Final = await core.select(alias) + assert chosen["model_info"]["id"] == "a" # pyright: ignore[reportIndexIssue] + assert chosen["model_name"] == "g" # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("alias", sorted(ALIASES)) +async def test_completion_through_an_alias_calls_the_target_group( + make_core: MakeCore, host: ScriptedHost, alias: str +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("other", "b")], model_group_alias=dict(ALIASES) + ) + await core.acompletion(model=alias, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("acompletion") == ["a"] + + +@pytest.mark.parametrize(("alias", "target"), [("plain", "g"), ("visible", "g"), ("hidden", "g"), ("g", None)]) +def test_registry_resolves_alias_targets(make_core: MakeCore, alias: str, target: str | None) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], model_group_alias=dict(ALIASES)) + assert core.registry().resolve(alias, kind="alias") == target + + +def test_hidden_alias_is_left_out_of_the_routable_names(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], model_group_alias=dict(ALIASES)) + assert sorted(core.registry().names(kind="groups")) == ["g", "plain", "visible"] + + +@pytest.mark.asyncio +async def test_alias_to_an_unknown_group_fails_naming_the_target(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], model_group_alias={"dangling": "missing"}) + with pytest.raises( + litellm.BadRequestError, match=re.escape("You passed in model=missing. There are no healthy deployments") + ): + await core.select("dangling") + + +# --- wildcard / pattern groups --------------------------------------------------------------- + +WILDCARD_MODEL_LIST: Final = [ + deployment("*", "catch_all", model="openai/*"), + deployment("anthropic/*", "anthropic_any", model="anthropic/*"), + deployment("openai/gpt-4o", "exact"), + deployment("openai/gpt-*", "gpt_any", model="openai/gpt-*"), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("requested", "expected_id", "expected_group", "expected_model"), + [ + ("openai/gpt-4o", "exact", "openai/gpt-4o", "openai/exact"), + ("openai/gpt-4o-mini", "gpt_any", "openai/gpt-*", "openai/gpt-4o-mini"), + ("anthropic/claude-x", "anthropic_any", "anthropic/*", "anthropic/claude-x"), + ("gpt-5", "catch_all", "*", "openai/gpt-5"), + ("whatever/foo", "catch_all", "*", "openai/whatever/foo"), + ], +) +async def test_select_prefers_an_exact_group_then_the_most_specific_pattern( + make_core: MakeCore, requested: str, expected_id: str, expected_group: str, expected_model: str +) -> None: + core: Final = make_core(model_list=list(WILDCARD_MODEL_LIST)) + chosen: Final = await core.select(requested) + assert chosen["model_info"]["id"] == expected_id # pyright: ignore[reportIndexIssue] + assert chosen["model_name"] == expected_group # pyright: ignore[reportIndexIssue] + assert chosen["litellm_params"]["model"] == expected_model # pyright: ignore[reportIndexIssue] + + +@pytest.mark.asyncio +async def test_completion_through_a_wildcard_calls_the_requested_model(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("openai/*", "wild", model="openai/*")]) + response: Final = await core.acompletion(model="openai/gpt-x", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("acompletion") == ["wild"] + assert host.calls[-1].model == "openai/gpt-x" + assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "openai/gpt-x" + + +@pytest.mark.asyncio +async def test_a_pattern_that_does_not_match_is_an_unknown_model(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("anthropic/*", "anthropic_any", model="anthropic/*")]) + with pytest.raises(litellm.BadRequestError, match="model=openai/gpt-4o"): + await core.select("openai/gpt-4o") + + +# --- selecting by deployment id or litellm model -------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deployment_id", ["a", "b", "c"]) +async def test_select_by_deployment_id_returns_that_deployment(make_core: MakeCore, deployment_id: str) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b"), deployment("h", "c")]) + assert await _picked_ids(core, deployment_id) == {deployment_id} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deployment_id", ["a", "b"]) +async def test_completion_by_deployment_id_calls_that_deployment( + make_core: MakeCore, host: ScriptedHost, deployment_id: str +) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")]) + for _ in range(5): + await core.acompletion(model=deployment_id, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("acompletion") == [deployment_id] * 5 + + +@pytest.mark.asyncio +async def test_select_by_litellm_model_name_finds_the_deployment(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")]) + assert await _picked_ids(core, "openai/b") == {"b"} + + +@pytest.mark.asyncio +async def test_specific_deployment_picks_by_litellm_model_across_groups(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g1", "c", model="openai/shared"), + deployment("g2", "d", model="openai/shared"), + deployment("g3", "e"), + ] + ) + assert await _picked_ids(core, "openai/shared", specific_deployment=True) <= {"c", "d"} + assert await _picked_ids(core, "openai/e", specific_deployment=True) == {"e"} + + +@pytest.mark.asyncio +async def test_completion_with_specific_deployment_calls_the_named_litellm_model( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")]) + for _ in range(5): + await core.acompletion(model="openai/b", messages=MESSAGES, specific_deployment=True) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("acompletion") == ["b"] * 5 + + +# --- unknown models --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("op", "request_payload"), + [ + ("acompletion", {"messages": MESSAGES}), + ("aembedding", {"input": "hi"}), + ("atext_completion", {"prompt": "hi"}), + ("completion", {"messages": MESSAGES}), + ("embedding", {"input": "hi"}), + ], +) +async def test_unknown_model_raises_bad_request_naming_the_model( + make_core: MakeCore, host: ScriptedHost, op: str, request_payload: dict[str, object] +) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + with pytest.raises(litellm.BadRequestError) as raised: + await core.call(op, {"model": "nope", **request_payload}) + assert ( + str(raised.value) + == "litellm.BadRequestError: You passed in model=nope. There are no healthy deployments for this model" + ) + assert raised.value.status_code == 400 + assert host.calls == [] + + +@pytest.mark.asyncio +async def test_select_of_an_unknown_model_raises_bad_request(make_core: MakeCore) -> None: + core: Final = make_core(model_list=[deployment("g", "a")]) + with pytest.raises(litellm.BadRequestError) as raised: + await core.select("nope") + assert raised.value.status_code == 400 + assert raised.value.model == "nope" + assert "There are no healthy deployments for this model" in str(raised.value) + + +# --- team-scoped deployments -------------------------------------------------------------------- + +TEAM_MODEL_LIST: Final = [ + deployment("g", "public"), + deployment("team-x-group", "team_x", model_info={"team_id": "tx", "team_public_model_name": "g"}), + deployment("team-y-group", "team_y", model_info={"team_id": "ty", "team_public_model_name": "g"}), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata", "expected"), + [ + ({}, "public"), + ({"user_api_key_team_id": "tx"}, "team_x"), + ({"user_api_key_team_id": "ty"}, "team_y"), + ({"user_api_key_team_id": "unknown-team"}, "public"), + ], +) +async def test_select_serves_a_team_its_own_deployment_under_the_public_name( + make_core: MakeCore, metadata: dict[str, str], expected: str +) -> None: + core: Final = make_core(model_list=list(TEAM_MODEL_LIST)) + assert await _picked_ids(core, "g", request_kwargs={"metadata": metadata}) == {expected} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("team_id", "expected"), [("tx", "team_x"), ("ty", "team_y"), ("tz", "public")]) +async def test_completion_for_a_team_calls_its_deployment_under_the_public_group( + make_core: MakeCore, host: ScriptedHost, team_id: str, expected: str +) -> None: + core: Final = make_core(model_list=list(TEAM_MODEL_LIST)) + await core.acompletion(model="g", messages=MESSAGES, metadata={"user_api_key_team_id": team_id}) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("acompletion") == [expected] + assert host.calls[-1].model_group == "g" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata", [{}, {"user_api_key_team_id": "other"}]) +async def test_team_only_public_name_is_unknown_to_other_callers(make_core: MakeCore, metadata: dict[str, str]) -> None: + core: Final = make_core( + model_list=[deployment("team-x-group", "team_x", model_info={"team_id": "tx", "team_public_model_name": "pub"})] + ) + with pytest.raises(litellm.BadRequestError, match="model=pub"): + await core.select("pub", {"metadata": metadata}) + assert await _picked_ids(core, "pub", request_kwargs={"metadata": {"user_api_key_team_id": "tx"}}) == {"team_x"} + + +@pytest.mark.parametrize(("team_id", "names"), [(None, []), ("tx", ["pub"])]) +def test_team_public_names_are_routable_only_for_their_team( + make_core: MakeCore, team_id: str | None, names: list[str] +) -> None: + core: Final = make_core( + model_list=[deployment("team-x-group", "team_x", model_info={"team_id": "tx", "team_public_model_name": "pub"})] + ) + assert list(core.registry().names(kind="groups", team_id=team_id)) == names + + +# --- weights, rpm and tpm under simple-shuffle ----------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("metric", ["weight", "rpm", "tpm"]) +async def test_simple_shuffle_never_picks_a_zero_weighted_deployment_beside_a_positive_one( + make_core: MakeCore, metric: str +) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "unset"), + deployment("g", "positive", **{metric: 5}), + deployment("g", "zero", **{metric: 0}), + ] + ) + assert await _picked_ids(core, "g") == {"positive"} + + +@pytest.mark.asyncio +async def test_weight_takes_precedence_over_rpm_and_tpm(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "weighted", weight=1, rpm=0, tpm=0), + deployment("g", "by_limits", weight=0, rpm=100, tpm=100), + ] + ) + assert await _picked_ids(core, "g") == {"weighted"} + + +@pytest.mark.asyncio +async def test_completion_calls_only_the_weighted_deployment(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a", weight=0), deployment("g", "b", weight=3)]) + for _ in range(10): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert set(host.deployments_called("acompletion")) == {"b"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("favoured", ["a", "b"]) +async def test_request_router_weights_override_deployment_weights(make_core: MakeCore, favoured: str) -> None: + core: Final = make_core(model_list=[deployment("g", "a", weight=1), deployment("g", "b", weight=1)]) + picked: Final = await _picked_ids(core, "g", request_kwargs={"_router_weights": {"g": {favoured: 1.0}}}) + assert picked == {favoured} + + +# --- order -------------------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_select_picks_only_the_lowest_order(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[ + deployment("g", "second", order=2), + deployment("g", "first_a", order=1), + deployment("g", "first_b", order=1), + deployment("g", "third", order=3), + ] + ) + assert await _picked_ids(core, "g") <= {"first_a", "first_b"} + + +@pytest.mark.asyncio +async def test_completion_calls_only_the_lowest_order(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "second", order=2), deployment("g", "first", order=1)]) + for _ in range(5): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("acompletion") == ["first"] * 5 + + +@pytest.mark.asyncio +async def test_next_order_serves_while_the_lowest_order_cools_down(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "second", order=2), deployment("g", "first", order=1)], + allowed_fails=0, + num_retries=0, + cooldown_time=60, + ) + host.answer("openai/first", rate_limit()) + await _fail_through(core, "first") + assert await _picked_ids(core, "g") == {"second"} + + +# --- routing strategies ---------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("routing_strategy", ["least-busy", "usage-based-routing"]) +@pytest.mark.parametrize("ids", [("a", "b"), ("b", "a")]) +async def test_strategy_without_traffic_picks_the_first_configured_deployment( + make_core: MakeCore, routing_strategy: str, ids: tuple[str, str] +) -> None: + core: Final = make_core( + model_list=[deployment("g", deployment_id) for deployment_id in ids], routing_strategy=routing_strategy + ) + assert await _picked_ids(core, "g", messages=MESSAGES) == {ids[0]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cheap_first", [True, False]) +async def test_cost_based_routing_picks_the_cheapest_deployment(make_core: MakeCore, cheap_first: bool) -> None: + cheap: Final = deployment("g", "cheap", input_cost_per_token=0.000001, output_cost_per_token=0.000002) + pricey: Final = deployment("g", "pricey", input_cost_per_token=0.00001, output_cost_per_token=0.00002) + core: Final = make_core( + model_list=[cheap, pricey] if cheap_first else [pricey, cheap], routing_strategy="cost-based-routing" + ) + assert await _picked_ids(core, "g", messages=MESSAGES) == {"cheap"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ids", [("tight", "roomy"), ("roomy", "tight")]) +async def test_usage_based_v2_skips_a_deployment_whose_tpm_the_request_exceeds( + make_core: MakeCore, ids: tuple[str, str] +) -> None: + limits: Final = {"tight": 1, "roomy": 100_000} + core: Final = make_core( + model_list=[deployment("g", deployment_id, tpm=limits[deployment_id]) for deployment_id in ids], + routing_strategy="usage-based-routing-v2", + ) + assert await _picked_ids(core, "g", messages=MESSAGES) == {"roomy"} + + +@pytest.mark.asyncio +async def test_usage_based_v2_rate_limits_when_the_request_exceeds_every_tpm(make_core: MakeCore) -> None: + core: Final = make_core( + model_list=[deployment("g", "a", tpm=1), deployment("g", "b", tpm=1)], + routing_strategy="usage-based-routing-v2", + ) + with pytest.raises( + litellm.RateLimitError, match=re.escape("No deployments available for selected model. Passed model=g") + ): + await core.select("g", messages=MESSAGES) # pyright: ignore[reportCallIssue] + + +# --- cooldowns ---------------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [rate_limit, auth_error, server_error, timeout], ids=lambda e: e.__name__) +async def test_a_deployment_that_failed_past_allowed_fails_is_not_picked( + make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60 + ) + host.answer("openai/a", error()) + await _fail_through(core, "a") + assert await _picked_ids(core, "g") == {"b"} + + +@pytest.mark.asyncio +async def test_completion_avoids_a_cooling_deployment(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60 + ) + host.answer("openai/a", rate_limit()) + await _fail_through(core, "a") + for _ in range(5): + await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + assert host.deployments_called("acompletion") == ["a", "b", "b", "b", "b", "b"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [bad_request, context_window], ids=lambda e: e.__name__) +async def test_client_errors_do_not_cool_a_deployment_down( + make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception] +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60 + ) + host.answer("openai/a", error()) + await _fail_through(core, "a", times=3) + assert await core.select("a") is not None + assert "a" in await _picked_ids(core, "g", times=60) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("allowed_fails", [1, 2, 3]) +async def test_a_deployment_stays_pickable_until_it_fails_more_than_allowed_fails( + make_core: MakeCore, host: ScriptedHost, allowed_fails: int +) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], + allowed_fails=allowed_fails, + num_retries=0, + cooldown_time=60, + ) + host.answer("openai/a", rate_limit()) + await _fail_through(core, "a", times=allowed_fails) + assert "a" in await _picked_ids(core, "g", times=60) + await _fail_through(core, "a") + assert await _picked_ids(core, "g") == {"b"} + + +@pytest.mark.asyncio +async def test_a_single_deployment_group_is_never_cooled_down(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=[deployment("g", "a")], allowed_fails=0, num_retries=0, cooldown_time=60) + host.answer("openai/a", rate_limit()) + await _fail_through(core, "a", times=3) + assert await _picked_ids(core, "g") == {"a"} + + +@pytest.mark.asyncio +async def test_selecting_a_cooling_deployment_by_id_still_returns_it(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60 + ) + host.answer("openai/a", rate_limit()) + await _fail_through(core, "a") + assert await _picked_ids(core, "g") == {"b"} + assert await _picked_ids(core, "a") == {"a"} + + +async def _all_cooling_core(make_core: MakeCore, host: ScriptedHost) -> RouterCore: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60 + ) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit()) + await _fail_through(core, "a") + await _fail_through(core, "b") + return core + + +ALL_COOLING_MESSAGE: Final = ( + "No deployments available for selected model, Try again in 60 seconds. All deployments for selected " + "model are in cooldown. Passed model=g. pre-call-checks=False, cooldown_list=['a', 'b']" +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("entry", ["select", "acompletion", "completion"]) +async def test_every_deployment_cooling_raises_router_rate_limit_error( + make_core: MakeCore, host: ScriptedHost, entry: str +) -> None: + core: Final = await _all_cooling_core(make_core, host) + calls_before: Final = len(host.calls) + call: Final = core.select("g") if entry == "select" else core.call(entry, {"model": "g", "messages": MESSAGES}) + with pytest.raises(RouterRateLimitError) as raised: + await call + assert str(raised.value) == ALL_COOLING_MESSAGE + assert raised.value.model == "g" + assert raised.value.cooldown_time == 60 + assert sorted(raised.value.cooldown_list) == ["a", "b"] + assert len(host.calls) == calls_before + + +@pytest.mark.asyncio +async def test_cooling_down_one_group_leaves_other_groups_pickable(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core( + model_list=[deployment("g", "a"), deployment("g", "b"), deployment("h", "c"), deployment("h", "d")], + allowed_fails=0, + num_retries=0, + cooldown_time=60, + ) + host.answer("openai/a", rate_limit()) + host.answer("openai/b", rate_limit()) + await _fail_through(core, "a") + await _fail_through(core, "b") + with pytest.raises(RouterRateLimitError): + await core.select("g") + assert await _picked_ids(core, "h") <= {"c", "d"} diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 7ac27e22a93..0dc75c0d38f 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2893,11 +2893,11 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto }, ) assert router.routing_strategy == "lar1" - assert "_core_async_get_available_deployment" in router.__dict__ + assert "_core_async_get_available_deployment" in router._core.__dict__ router._switch_routing_strategy("usage-based-routing-v2", {}) assert router.lowesttpm_logger_v2 is not None - assert "async_get_available_deployment" not in router.__dict__ + assert "_core_async_get_available_deployment" not in router._core.__dict__ @pytest.mark.parametrize( @@ -24271,7 +24271,7 @@ def _record_phase_events(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, dic def record(name: str, attributes: dict[str, str | int]) -> None: events.append((name, dict(attributes))) - monkeypatch.setattr(litellm.router, "phase_event", record) + monkeypatch.setattr(litellm.router_python_core, "phase_event", record) return events @@ -25666,7 +25666,9 @@ def test_get_model_list_from_model_alias_should_not_iterate_for_non_alias_lookup ], model_group_alias={"alias-1": "gpt-5.5"}, ) - router.model_group_alias = NoItemsAliasDict({f"alias-{idx}": "gpt-5.5" for idx in range(200)}) + # Assigning model_group_alias copies the value (a changeset detaches it), so the tripwire goes + # straight onto the core, where the lookup reads it. + router._core._core_model_group_alias = NoItemsAliasDict({f"alias-{idx}": "gpt-5.5" for idx in range(200)}) model_alias_list = router.get_model_list_from_model_alias(model_name="gpt-5-mini") assert model_alias_list == [] @@ -25686,7 +25688,9 @@ def test_map_team_model_should_not_iterate_aliases_for_non_alias_team_model_name ], model_group_alias={"alias-1": "gpt-5.5"}, ) - router.model_group_alias = NoItemsAliasDict({f"alias-{idx}": "gpt-5.5" for idx in range(200)}) + # Assigning model_group_alias copies the value (a changeset detaches it), so the tripwire goes + # straight onto the core, where the lookup reads it. + router._core._core_model_group_alias = NoItemsAliasDict({f"alias-{idx}": "gpt-5.5" for idx in range(200)}) # map_team_model should return the public name unchanged (not the internal UUID name) # so the router can find all sibling deployments via team_id filtering @@ -29139,13 +29143,13 @@ def test_reset_custom_routing_strategy(): router = _create_custom_routing_router() router.set_custom_routing_strategy(SpecialEndpointRoutingStrategy(router)) - assert "get_available_deployment" in router.__dict__ - assert "_core_async_get_available_deployment" in router.__dict__ + assert "get_available_deployment" in router._core.__dict__ + assert "_core_async_get_available_deployment" in router._core.__dict__ router._reset_custom_routing_strategy() - assert "get_available_deployment" not in router.__dict__ - assert "async_get_available_deployment" not in router.__dict__ + assert "get_available_deployment" not in router._core.__dict__ + assert "_core_async_get_available_deployment" not in router._core.__dict__ assert router.async_get_available_deployment.__func__ is Router.async_get_available_deployment router._reset_custom_routing_strategy() @@ -30985,7 +30989,7 @@ async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_s _SYNC_SECONDS: Final = 10 -_ROUTER_FILE: Final = Router.__init__.__code__.co_filename +_ROUTER_FILE: Final = Router._update_deployment_indices_after_removal.__code__.co_filename _INDEX_LOCK_LINES: Final = frozenset( number for number, line in enumerate(linecache.getlines(_ROUTER_FILE), start=1) diff --git a/tests/unit/test_router/test_router_entry_points.py b/tests/unit/test_router/test_router_entry_points.py index 366e16a8e91..81c1982b987 100644 --- a/tests/unit/test_router/test_router_entry_points.py +++ b/tests/unit/test_router/test_router_entry_points.py @@ -359,7 +359,7 @@ def test_settings_snapshot_is_frozen(router): def test_unset_setting_raises_attribute_error_through_old_name(): r = _router() try: - del r.__dict__["_core_kubernetes_pod_discovery"] + del r._core.__dict__["_core_kubernetes_pod_discovery"] with warnings.catch_warnings(): warnings.simplefilter("ignore") assert getattr(r, "kubernetes_pod_discovery", "missing") == "missing" @@ -372,14 +372,14 @@ def test_unset_setting_raises_attribute_error_through_old_name(): def test_unset_attribute_error_names_the_public_attribute(router): # Reading (or deleting) an old attribute the router never set raises what reading it raised before. - router.__dict__.pop("_core_slack_alerting_logger", None) + router._core.__dict__.pop("_core_slack_alerting_logger", None) with warnings.catch_warnings(): warnings.simplefilter("ignore") with pytest.raises(AttributeError) as read: _ = router.slack_alerting_logger with pytest.raises(AttributeError) as deleted: del router.slack_alerting_logger - del router.__dict__["_core_kubernetes_pod_discovery"] + del router._core.__dict__["_core_kubernetes_pod_discovery"] with pytest.raises(AttributeError) as setting: _ = router.kubernetes_pod_discovery for caught, name in ((read, "slack_alerting_logger"), (setting, "kubernetes_pod_discovery")): diff --git a/tests/unit/test_router/test_router_facade.py b/tests/unit/test_router/test_router_facade.py new file mode 100644 index 00000000000..0539ebe43b9 --- /dev/null +++ b/tests/unit/test_router/test_router_facade.py @@ -0,0 +1,52 @@ +""" +The ``litellm.Router`` facade's own helpers in litellm/router.py: how it binds the constructor +arguments, which members it leaves to the facade, and the module attributes it forwards to the core. +""" + +import pytest + +import litellm.router as router_module +from litellm import router_python_core +from litellm.router import Router, _class_descriptor, router_arguments + + +def test_router_arguments_binds_positional_arguments_to_their_parameter_names(): + model_list = [{"model_name": "g", "litellm_params": {"model": "openai/gpt-4o-mini"}}] + + arguments = router_arguments(model_list, num_retries=2) + + assert dict(arguments) == {"model_list": model_list, "num_retries": 2} + + +def test_router_arguments_leaves_unset_parameters_out(): + assert dict(router_arguments()) == {} + + +def test_router_arguments_rejects_an_unknown_argument_naming_the_router(): + with pytest.raises(TypeError, match=r"^Router\.__init__\(\) got an unexpected keyword argument 'not_a_setting'"): + router_arguments(not_a_setting=1) + + +def test_router_arguments_are_read_only(): + arguments = router_arguments(num_retries=2) + + with pytest.raises(TypeError): + arguments["num_retries"] = 3 # pyright: ignore[reportIndexIssue] # proving the mapping is read-only + + +def test_class_descriptor_is_true_for_a_data_descriptor_on_the_class(): + assert _class_descriptor(Router, "_core") + + +def test_class_descriptor_is_false_for_a_plain_method_or_a_missing_name(): + assert not _class_descriptor(Router, "from_config") + assert not _class_descriptor(Router, "no_such_member") + + +def test_module_getattr_forwards_old_helpers_to_the_core_module(): + assert router_module.__getattr__("_stream_pre_content_error") is router_python_core._stream_pre_content_error + + +def test_module_getattr_raises_attribute_error_for_an_unknown_name(): + with pytest.raises(AttributeError): + router_module.__getattr__("no_such_router_helper") diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 8929c5fd345..4964dea498f 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -1182,7 +1182,7 @@ def test_deployment_callback_respects_cooldown_time(model_list): }, } - with patch("litellm.router.set_cooldown_deployments") as mock_set: + with patch("litellm.router_python_core.set_cooldown_deployments") as mock_set: router.deployment_callback_on_failure( kwargs=kwargs, completion_response=None, @@ -1554,7 +1554,7 @@ def _rpm_tpm_router(model_id: str) -> Router: def _pin_router_clock(monkeypatch: pytest.MonkeyPatch, pinned: datetime) -> None: - monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) + monkeypatch.setattr("litellm.router_python_core.get_utc_datetime", lambda: pinned) monkeypatch.setattr( "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: pinned ) @@ -2398,10 +2398,10 @@ def _routers_seen_in_success_loop_discarding_first(first: Router, second: Router for callback in litellm.success_callback: if not isinstance(callback, MethodType): continue - if callback.__self__ is first: + if callback.__self__ is first._core: first.discard() yield first - if callback.__self__ is second: + if callback.__self__ is second._core: yield second diff --git a/tests/unit/test_router/test_router_provider_endpoints.py b/tests/unit/test_router/test_router_provider_endpoints.py index f8b57410872..d2331c00804 100644 --- a/tests/unit/test_router/test_router_provider_endpoints.py +++ b/tests/unit/test_router/test_router_provider_endpoints.py @@ -53,7 +53,7 @@ def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: @pytest.fixture def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> None: pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) - monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) + monkeypatch.setattr("litellm.router_python_core.get_utc_datetime", lambda: pinned) class _RouterLoggingCapture(CustomLogger): diff --git a/tests/unit/test_router_silent_experiment.py b/tests/unit/test_router_silent_experiment.py index b696a61aa3c..fc787fcf74a 100644 --- a/tests/unit/test_router_silent_experiment.py +++ b/tests/unit/test_router_silent_experiment.py @@ -620,7 +620,7 @@ def test_sync_shadow_gets_kwargs_snapshot_taken_before_primary_mutates_them(reco router = Router(model_list=_tagged_primary_model_list()) with patch( # test-quality-ok: Router has no thread factory to inject; deferring start is the only deterministic way to expose the race - "litellm.router.threading", SimpleNamespace(Thread=_DeferredThread) + "litellm.router_python_core.threading", SimpleNamespace(Thread=_DeferredThread) ): response = router.completion( model="primary-model", @@ -652,7 +652,7 @@ def test_sync_shadow_workers_do_not_share_metadata_with_each_other(recording_log router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"])) with patch( # test-quality-ok: Router has no thread factory to inject; deferring start is the only deterministic way to expose the race - "litellm.router.threading", SimpleNamespace(Thread=_DeferredThread) + "litellm.router_python_core.threading", SimpleNamespace(Thread=_DeferredThread) ): router.completion( model="primary-model", diff --git a/tests/unit/test_router_weighted_failover.py b/tests/unit/test_router_weighted_failover.py index 8b7fc81dbde..c00670473af 100644 --- a/tests/unit/test_router_weighted_failover.py +++ b/tests/unit/test_router_weighted_failover.py @@ -338,7 +338,7 @@ async def test_maybe_run_weighted_failover_persists_excluded_ids_to_kwargs(monke async def _stub_run_async_fallback(*args, **kwargs): return "ok" - monkeypatch.setattr("litellm.router.run_async_fallback", _stub_run_async_fallback) + monkeypatch.setattr("litellm.router_python_core.run_async_fallback", _stub_run_async_fallback) exc = Exception("fail") exc.failed_deployment_id = "A" @@ -807,11 +807,11 @@ async def test_maybe_run_weighted_failover_skips_when_remaining_all_in_cooldown( run_async_fallback_called = True return "should not reach here" - monkeypatch.setattr("litellm.router.run_async_fallback", _should_not_be_called) + monkeypatch.setattr("litellm.router_python_core.run_async_fallback", _should_not_be_called) # Patch cooldown so B and C appear in cooldown. with patch( - "litellm.router.async_get_cooldown_deployments", + "litellm.router_python_core.async_get_cooldown_deployments", new=AsyncMock(return_value=["B", "C"]), ): result = await router._maybe_run_weighted_failover( @@ -867,10 +867,10 @@ async def test_maybe_run_weighted_failover_proceeds_when_one_healthy_remains( run_async_fallback_called = True return "ok from C" - monkeypatch.setattr("litellm.router.run_async_fallback", _stub_run_async_fallback) + monkeypatch.setattr("litellm.router_python_core.run_async_fallback", _stub_run_async_fallback) with patch( - "litellm.router.async_get_cooldown_deployments", + "litellm.router_python_core.async_get_cooldown_deployments", new=AsyncMock(return_value=["B"]), ): result = await router._maybe_run_weighted_failover( @@ -936,7 +936,7 @@ async def test_failover_falls_through_to_external_fallback_when_remaining_in_coo # Put B in cooldown so weighted failover can't use it after A fails. with patch( - "litellm.router.async_get_cooldown_deployments", + "litellm.router_python_core.async_get_cooldown_deployments", new=AsyncMock(return_value=["B"]), ): response = await router.acompletion(