refactor(router): extract PythonCore behind RouterCore and pass the router its config (#49119)

* refactor(router): move litellm/router.py to litellm/router_python_core.py

A pure rename with no content change, so git and review tools track the router body as a move.
The next commit turns it into PythonCore and adds the Router facade back at litellm/router.py

* refactor(router): extract PythonCore behind RouterCore and pass the router its config

The routing logic moves unchanged from litellm/router.py into PythonCore in
litellm/router_python_core.py. Router becomes a facade that builds a RouterConfig, picks a
RouterCore with make_core and delegates every member to it. The core no longer reads or writes
litellm.* directly: process settings come from RouterDynamicConfig, and inference calls, sleeps,
callback lists and the global cache go through PythonHost. A CI check bans litellm.* in the router
modules, and the proxy's construction sites build the config outside the router

Settings are static: the constructor arguments and changeset values are copied when the router is
built or the change applied, and a write to a setting attribute becomes apply(ChangeSet(assign=...)).
A Router subclass's core class also subclasses the facade, so an override's super(Sub, self) works,
and an instance-level patch of a core method is kept on the facade so nested patches unwind.

tests/unit/test_router/core tests the core in isolation: ScriptedHost serves every host operation
(scripted inference answers, recorded sleeps, settings, callbacks, cache), and make_core runs each
test once per core in CORE_BUILDERS

litellm/llms/anthropic/messages/adapters/handler.py

@@ -118,9 +118,10 @@ def _extract_proxy_litellm_metadata(
 
 def _requested_router(kwargs: Mapping[str, object]) -> "Router | None":
     from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
     requested: Final = kwargs.get("litellm_router")
-    return requested if isinstance(requested, Router) else None
+    return requested if isinstance(requested, (Router, PythonCore)) else None  # pyright: ignore[reportReturnType]  # the router passes its core, which carries Router's members
 
 
 def request_safeguards_evaluator(

litellm/proxy/proxy_server.py

@@ -846,6 +846,8 @@ from litellm.proxy.telemetry.middleware import TelemetryMiddleware
 from litellm.proxy.telemetry.runtime import TelemetryRuntime
 from litellm.proxy.telemetry.settings import describe_errors as describe_telemetry_errors
 from litellm.proxy.telemetry.settings import load_settings as load_telemetry_settings
+from litellm.router import router_arguments
+from litellm.router_config import build_router_config
 
 try:
     from litellm.proxy.enterprise_billing.billing_metrics import (
@@ -7407,20 +7409,24 @@ class ProxyConfig:
                     verbose_proxy_logger.warning(
                         "Key '%s' is not a valid argument for Router.__init__(). Ignoring this key.", k
                     )
-        router = litellm.Router(
-            **router_params,
-            assistants_config=assistants_config,
-            search_tools=search_tools,
-            router_general_settings=RouterGeneralSettings(
-                async_only_mode=True  # only init async clients
-            ),
-            ignore_invalid_deployments=True,  # don't raise an error if a deployment is invalid
-            fallback_access_check=router_fallback_access_check,
-            fallback_budget_check=RouterFallbackBudgetCheck(
-                is_enforced=router_fallback_budget_check.is_enforced,
-                team_member_model_budget_limiter=model_max_budget_limiter,
-            ),
-            auto_router_capability_limit=_license_check.auto_router_capability_limit,
+        router = litellm.Router.from_config(
+            build_router_config(
+                router_arguments(
+                    **router_params,
+                    assistants_config=assistants_config,
+                    search_tools=search_tools,
+                    router_general_settings=RouterGeneralSettings(
+                        async_only_mode=True  # only init async clients
+                    ),
+                    ignore_invalid_deployments=True,  # don't raise an error if a deployment is invalid
+                    fallback_access_check=router_fallback_access_check,
+                    fallback_budget_check=RouterFallbackBudgetCheck(
+                        is_enforced=router_fallback_budget_check.is_enforced,
+                        team_member_model_budget_limiter=model_max_budget_limiter,
+                    ),
+                    auto_router_capability_limit=_license_check.auto_router_capability_limit,
+                )
+            )
         )
 
         if redis_usage_cache is not None and router.health().store.redis_cache is None:
@@ -7910,19 +7916,23 @@ class ProxyConfig:
                 # Router can function with model_list=[] if search_tools are configured
                 if len(_model_list) > 0 or search_tools:
                     verbose_proxy_logger.debug("_model_list: %s", _model_list)
-                    llm_router = litellm.Router(
-                        model_list=_model_list,
-                        router_general_settings=RouterGeneralSettings(
-                            async_only_mode=True  # only init async clients
-                        ),
-                        search_tools=search_tools,
-                        ignore_invalid_deployments=True,
-                        fallback_access_check=router_fallback_access_check,
-                        fallback_budget_check=RouterFallbackBudgetCheck(
-                            is_enforced=router_fallback_budget_check.is_enforced,
-                            team_member_model_budget_limiter=model_max_budget_limiter,
-                        ),
-                        auto_router_capability_limit=_license_check.auto_router_capability_limit,
+                    llm_router = litellm.Router.from_config(
+                        build_router_config(
+                            router_arguments(
+                                model_list=_model_list,
+                                router_general_settings=RouterGeneralSettings(
+                                    async_only_mode=True  # only init async clients
+                                ),
+                                search_tools=search_tools,
+                                ignore_invalid_deployments=True,
+                                fallback_access_check=router_fallback_access_check,
+                                fallback_budget_check=RouterFallbackBudgetCheck(
+                                    is_enforced=router_fallback_budget_check.is_enforced,
+                                    team_member_model_budget_limiter=model_max_budget_limiter,
+                                ),
+                                auto_router_capability_limit=_license_check.auto_router_capability_limit,
+                            )
+                        )
                     )
                     verbose_proxy_logger.debug("updated llm_router: %s", llm_router)
             else:

litellm/proxy/route_llm_request.py

@@ -18,7 +18,9 @@ from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+from litellm.router import router_arguments
 from litellm.router_api import router_init_args
+from litellm.router_config import build_router_config
 
 # Client-supplied params that make the router or the call path fabricate a
 # failure or a delay instead of calling the provider. The ``mock_testing_*``
@@ -98,7 +100,9 @@ async def _route_user_config_request(data: dict, route_type: str):
     filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
 
     logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type)
-    cached_router: Final = user_config_routers.get(filtered_config, lambda: litellm.Router(**filtered_config))
+    cached_router: Final = user_config_routers.get(
+        filtered_config, lambda: litellm.Router.from_config(build_router_config(router_arguments(**filtered_config)))
+    )
     cached_router.observe(logging_obj)
     return await getattr(cached_router, f"{route_type}")(**{**data, "litellm_logging_obj": logging_obj})
 

litellm/responses/streaming_iterator.py

@@ -233,6 +233,11 @@ def spawn_logging_task(
 
 
 async def drain_logging_tasks(timeout: float = 5.0) -> None:
+    # A task that finished as its loop closed never ran its discard callback, and a
+    # task from another loop cannot be awaited here: forget both, wait on the rest.
+    running_loop: Final = asyncio.get_running_loop()
+    stale: Final = {task for task in _PENDING_LOGGING_TASKS if task.done() or task.get_loop() is not running_loop}
+    _PENDING_LOGGING_TASKS.difference_update(stale)
     if not _PENDING_LOGGING_TASKS:
         return
     _, pending = await asyncio.wait(tuple(_PENDING_LOGGING_TASKS), timeout=timeout)

litellm/router_api.py

@@ -42,6 +42,8 @@ from dataclasses import dataclass, field
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, overload
 
+from litellm.router_config import detached
+
 if TYPE_CHECKING:
     from typing import Any  # noqa: TID251  # only to spell the cores' untyped (Unknown) containers and one Any type
 
@@ -176,9 +178,9 @@ def deployment_usable_by_team(deployment: "Mapping[str, object] | Deployment", t
     Whether ``team_id`` may use ``deployment``: a team-scoped deployment (``model_info.team_id``
     set) only by that team, any other deployment by everyone. Needs no router instance.
     """
-    from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
-    return Router._core_deployment_usable_by_team(deployment, team_id)
+    return PythonCore._core_deployment_usable_by_team(deployment, team_id)
 
 
 def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str:
@@ -200,9 +202,9 @@ def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str
 
 def router_init_args() -> tuple[str, ...]:
     """The arguments ``Router.__init__`` accepts (was ``Router.get_valid_args``). Needs no router instance."""
-    from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
-    return tuple(Router.get_valid_args())
+    return tuple(PythonCore.get_valid_args())
 
 
 def mark_advisor_orchestration_failure(exception: BaseException) -> None:
@@ -266,9 +268,9 @@ def deployment_metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, ob
 
 
 def _json_default_stable_id(obj: object) -> str:
-    from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
-    return Router._json_default_stable_id(obj)
+    return PythonCore._json_default_stable_id(obj)
 
 
 class RouterRegistry:
@@ -529,10 +531,10 @@ class RouterRegistry:
         ``should_include_deployment``), or, without ``model_name``, whether the team may use it at
         all (was ``deployment_usable_by_team``).
         """
-        from litellm.router import Router
+        from litellm.router_python_core import PythonCore
 
         if model_name is None:
-            return Router._core_deployment_usable_by_team(deployment, team_id)
+            return PythonCore._core_deployment_usable_by_team(deployment, team_id)
         return self._router._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id)
 
     @overload
@@ -1053,6 +1055,12 @@ SETTING_FIELDS: Final[tuple[str, ...]] = (
     "cache_responses",
 )
 
+# Runtime settings kept as plain router attributes rather than ``settings()`` fields. A write
+# to one (``router.num_retries = 2``) is a changeset too: ``ChangeSet(assign={...})``.
+PLAIN_SETTING_ATTRIBUTES: Final[frozenset[str]] = frozenset({"num_retries", "max_fallbacks", "retry_after", "timeout"})
+# Former attributes whose value lives under another name, assignable through a changeset.
+ASSIGNABLE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"model_group_alias": "_core_model_group_alias"})
+
 _MISSING: Final = object()
 
 
@@ -1168,16 +1176,16 @@ class RouterHealth:
     @staticmethod
     def is_blocked(deployment: "Deployment") -> bool:
         """Whether an admin paused this deployment (was ``is_deployment_blocked``)."""
-        from litellm.router import Router
+        from litellm.router_python_core import PythonCore
 
-        return Router._core_is_deployment_blocked(deployment)
+        return PythonCore._core_is_deployment_blocked(deployment)
 
     @staticmethod
     def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool:
         """Whether every deployment in the list is paused (was ``are_all_deployments_blocked``)."""
-        from litellm.router import Router
+        from litellm.router_python_core import PythonCore
 
-        return Router._core_are_all_deployments_blocked(deployments)
+        return PythonCore._core_are_all_deployments_blocked(deployments)
 
     def fully_blocked_model_names(self) -> set[str]:  # mutable-ok: the core's type, a fresh set
         """Groups whose every deployment is paused (was ``get_fully_blocked_model_names``)."""
@@ -1296,13 +1304,17 @@ class RoutingObservers:
 def apply_changes(router: "Router", changes: ChangeSet) -> ApplyResult:
     if changes.refresh_model_info:
         raise TypeError("ChangeSet(refresh_model_info=True) needs aapply()")
-    unknown: Final = sorted(set(changes.assign) - set(SETTING_FIELDS))
+    unknown: Final = sorted(set(changes.assign) - {*SETTING_FIELDS, *PLAIN_SETTING_ATTRIBUTES, *ASSIGNABLE_ALIASES})
     if unknown:
         raise ValueError(f"not assignable router settings: {unknown}")
     for name, value in changes.assign.items():
-        setattr(router, "_core_" + name, value)
+        setattr(
+            router,
+            name if name in PLAIN_SETTING_ATTRIBUTES else ASSIGNABLE_ALIASES.get(name, "_core_" + name),
+            detached(value),
+        )
     if changes.settings:
-        router._core_update_settings(**changes.settings)
+        router._core_update_settings(**{name: detached(value) for name, value in changes.settings.items()})
     upserted: Final = [router._core_upsert_deployment(deployment=d) for d in changes.upsert]
     deleted: Final = [router._core_delete_deployment(id=i) for i in changes.delete]
     if changes.redis_cache is not None:
@@ -1388,7 +1400,7 @@ class RoutingContext:
         return self._router.observe()
 
     def belongs_to(self, router: "Router") -> bool:
-        return self._router is router
+        return self._router is router or router.__dict__.get("_core") is self._router
 
     @property
     def explicit_allowed_fails(self) -> int | None:

litellm/router_config.py

@@ -0,0 +1,157 @@
+"""
+The router's configuration, built outside the router.
+
+``RouterConfig`` is static: the constructor arguments, frozen for the router's life.
+``RouterDynamicConfig`` is a snapshot of the process settings the router reads while it serves
+(``litellm.num_retries``, ``litellm.model_cost``, the fallback lists, ...). A call reads one snapshot
+at its start. This module is the only router code that reads ``litellm.*`` settings.
+"""
+
+import itertools
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final
+
+import litellm
+
+if TYPE_CHECKING:
+    from litellm.caching.caching import Cache
+    from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
+
+
+@dataclass(frozen=True, slots=True)
+class RouterConfig:
+    """The arguments the router was built with, exactly as passed (unset ones are absent)."""
+
+    arguments: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
+
+    def get(self, name: str, default: object = None) -> object:
+        return self.arguments.get(name, default)
+
+
+@dataclass(frozen=True, slots=True)
+class RouterDynamicConfig:
+    """Process settings the router reads per call. Values are the live objects, not copies."""
+
+    version: int
+    num_retries: int | None
+    max_fallbacks: int | None
+    router_max_fallbacks: int
+    allowed_fails: int
+    request_timeout: float
+    fallbacks: Sequence[object] | None
+    default_fallbacks: Sequence[str] | None
+    context_window_fallbacks: Sequence[object] | None
+    content_policy_fallbacks: Sequence[object] | None
+    drop_params: bool | None
+    provider_list: Sequence[str]
+    expose_router_debug_in_errors: bool
+    redact_user_api_key_info: bool | None
+    model_alias_map: Mapping[str, str]
+    model_cost: Mapping[str, object]
+    known_custom_logger_compatible_callbacks: Sequence[str]
+    exception_types: Sequence[type[BaseException]]
+    callbacks: Sequence[object]
+    success_callback: Sequence[object]
+    failure_callback: Sequence[object]
+    input_callback: Sequence[object]
+    callback_manager: "LoggingCallbackManager"
+    cache: "Cache | None"
+
+
+_versions: Final = itertools.count(1)
+
+
+def detached(value: object) -> object:
+    """``value`` with its plain containers (dict, list, tuple, set) copied, all the way down, so the
+    caller's own objects can't change the router afterwards. Other objects (clients, caches,
+    callbacks, pydantic models) are kept as they are."""
+    if isinstance(value, dict):
+        return {key: detached(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [detached(item) for item in value]
+    if isinstance(value, tuple) and type(value) is tuple:
+        return tuple(detached(item) for item in value)
+    if isinstance(value, set):
+        return {detached(item) for item in value}
+    return value
+
+
+def build_router_config(arguments: Mapping[str, object]) -> RouterConfig:
+    """The constructor arguments, detached from the caller's objects: settings are static."""
+    return RouterConfig(arguments=MappingProxyType({name: detached(value) for name, value in arguments.items()}))
+
+
+def current_dynamic_config() -> RouterDynamicConfig:
+    """A snapshot of the process settings as they are right now."""
+    return RouterDynamicConfig(
+        version=next(_versions),
+        num_retries=litellm.num_retries,
+        max_fallbacks=litellm.max_fallbacks,
+        router_max_fallbacks=litellm.ROUTER_MAX_FALLBACKS,
+        allowed_fails=litellm.allowed_fails,
+        request_timeout=litellm.request_timeout,
+        fallbacks=litellm.fallbacks,
+        default_fallbacks=litellm.default_fallbacks,
+        context_window_fallbacks=litellm.context_window_fallbacks,
+        content_policy_fallbacks=litellm.content_policy_fallbacks,
+        drop_params=litellm.drop_params,
+        provider_list=litellm.provider_list,
+        expose_router_debug_in_errors=litellm.expose_router_debug_in_errors,
+        redact_user_api_key_info=litellm.redact_user_api_key_info,
+        model_alias_map=litellm.model_alias_map,
+        model_cost=litellm.model_cost,
+        known_custom_logger_compatible_callbacks=litellm._known_custom_logger_compatible_callbacks,
+        exception_types=litellm.LITELLM_EXCEPTION_TYPES,
+        callbacks=litellm.callbacks,
+        success_callback=litellm.success_callback,
+        failure_callback=litellm.failure_callback,
+        input_callback=litellm.input_callback,
+        callback_manager=litellm.logging_callback_manager,
+        cache=litellm.cache,
+    )
+
+
+_LIVE_SOURCES: Final = MappingProxyType(
+    {
+        "num_retries": "num_retries",
+        "max_fallbacks": "max_fallbacks",
+        "router_max_fallbacks": "ROUTER_MAX_FALLBACKS",
+        "allowed_fails": "allowed_fails",
+        "request_timeout": "request_timeout",
+        "fallbacks": "fallbacks",
+        "default_fallbacks": "default_fallbacks",
+        "context_window_fallbacks": "context_window_fallbacks",
+        "content_policy_fallbacks": "content_policy_fallbacks",
+        "drop_params": "drop_params",
+        "provider_list": "provider_list",
+        "expose_router_debug_in_errors": "expose_router_debug_in_errors",
+        "redact_user_api_key_info": "redact_user_api_key_info",
+        "model_alias_map": "model_alias_map",
+        "model_cost": "model_cost",
+        "known_custom_logger_compatible_callbacks": "_known_custom_logger_compatible_callbacks",
+        "exception_types": "LITELLM_EXCEPTION_TYPES",
+        "callbacks": "callbacks",
+        "success_callback": "success_callback",
+        "failure_callback": "failure_callback",
+        "input_callback": "input_callback",
+        "callback_manager": "logging_callback_manager",
+        "cache": "cache",
+    }
+)
+
+
+class LiveDynamicConfig:
+    """``RouterDynamicConfig``'s fields read from the process settings at access time, with no snapshot."""
+
+    __slots__ = ()
+
+    def __getattr__(self, name: str) -> object:
+        source: Final = _LIVE_SOURCES.get(name)
+        if source is None:
+            raise AttributeError(name)
+        return getattr(litellm, source)
+
+
+live_dynamic_config: Final = LiveDynamicConfig()

litellm/router_core.py

@@ -0,0 +1,56 @@
+"""
+``RouterCore``: the interface ``litellm.Router`` delegates to, and ``make_core``, which picks one.
+
+``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. The Rust core
+(litellm-rust/crates/router-core) implements the same operations. The core is picked once, when
+the router is built, from its ``RouterConfig``.
+"""
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+from litellm.router_config import RouterConfig
+
+if TYPE_CHECKING:
+    from litellm.router_api import (
+        ApplyResult,
+        ChangeSet,
+        RouterHealth,
+        RouterRegistry,
+        RouterSettings,
+        RoutingObservers,
+    )
+    from litellm.router_python_core import PythonCore
+    from litellm.router_python_host import PythonHost
+    from litellm.types.router import CustomRoutingStrategyBase
+
+
+@runtime_checkable
+class RouterCore(Protocol):
+    async def call(self, op: str, request: Mapping[str, object]) -> object: ...
+
+    def registry(self) -> "RouterRegistry": ...
+
+    def settings(self) -> "RouterSettings": ...
+
+    def health(self) -> "RouterHealth": ...
+
+    async def select(
+        self,
+        model: str,
+        request_kwargs: dict[str, object] | None = None,  # mutable-ok: the selection cores take a dict
+    ) -> object: ...
+
+    def apply(self, changes: "ChangeSet") -> "ApplyResult": ...
+
+    def observe(self) -> "RoutingObservers": ...
+
+    def register_plugin(self, plugin: "CustomRoutingStrategyBase") -> None: ...
+
+    def retire(self) -> None: ...
+
+
+def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore:
+    from litellm.router_python_core import PythonCore
+
+    return (core_class or PythonCore)(**config.arguments, _host=host)  # pyright: ignore[reportArgumentType]  # the arguments were bound against PythonCore's own signature

litellm/router_deprecated.py

@@ -31,7 +31,7 @@ from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload
 
-from litellm.router_api import SETTING_FIELDS, ChangeSet
+from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet
 
 if TYPE_CHECKING:
     from typing import Any  # noqa: TID251  # only to repeat two old signatures that had Any, unchanged
@@ -65,7 +65,7 @@ _INTERNAL_PREFIX: Final = "litellm.router"
 
 
 # Router's own attribute hooks: a write or delete through an old name passes through them first.
-_ATTRIBUTE_HOOKS: Final = frozenset({"__setattr__", "__delattr__"})
+_ATTRIBUTE_HOOKS: Final = frozenset({"__getattribute__", "__setattr__", "__delattr__"})
 
 
 def _note(name: str, replacement: str) -> None:
@@ -88,9 +88,10 @@ def _note(name: str, replacement: str) -> None:
 def _as_router(obj: "DeprecatedRouterMembers") -> "Router":
     """The Router this mixin is part of (it is mixed into ``litellm.Router`` only)."""
     from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
-    if isinstance(obj, Router):
-        return obj
+    if isinstance(obj, (Router, PythonCore)):
+        return obj  # pyright: ignore[reportReturnType]  # a PythonCore carries every member the mixin reads
     raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only")
 
 
@@ -134,7 +135,7 @@ class _DeprecatedAttribute(Generic[_T]):
 
     def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None:
         _note(self.name, self.replacement)
-        if self.name in SETTING_FIELDS:
+        if self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES:
             _as_router(obj).apply(ChangeSet(assign={self.name: value}))
         else:
             setattr(obj, "_core_" + self.name, value)
@@ -169,6 +170,13 @@ _OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] =
 )
 
 
+def _override_key(router: object, name: str) -> tuple[int, str]:
+    """The running-override key for ``router``: a Router and its core share one, so an old member
+    entered on the Router is seen by the override routing that runs on the core."""
+    core: Final = getattr(router, "__dict__", {}).get("_core")
+    return (id(router if core is None else core), name)
+
+
 def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., object]:
     """
     A direct call of a subclass override (``router.<name>()``) marks it running, so its
@@ -184,7 +192,7 @@ def _enter_override(name: str, override: Callable[..., object]) -> Callable[...,
             *args: object,
             **kwargs: object,  # kwargs-ok: passes the override's arguments through unchanged
         ) -> object:
-            token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)})
+            token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)})
             try:
                 return await async_override(router, *args, **kwargs)
             finally:
@@ -198,7 +206,7 @@ def _enter_override(name: str, override: Callable[..., object]) -> Callable[...,
         *args: object,
         **kwargs: object,  # kwargs-ok: passes the override's arguments through unchanged
     ) -> object:
-        token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)})
+        token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)})
         try:
             return override(router, *args, **kwargs)
         finally:
@@ -236,7 +244,7 @@ def _route_override(cls: type, name: str, override: Callable[..., object]) -> Ca
             *args: object,
             **kwargs: object,  # kwargs-ok: passes the override's arguments through unchanged
         ) -> object:
-            key: Final = (id(router), name)
+            key: Final = _override_key(router, name)
             running: Final = _OVERRIDES_RUNNING.get()
             if key in running:
                 parent_async: Final[Callable[..., Awaitable[object]]] = getattr(super(cls, router), core)
@@ -255,7 +263,7 @@ def _route_override(cls: type, name: str, override: Callable[..., object]) -> Ca
         *args: object,
         **kwargs: object,  # kwargs-ok: passes the override's arguments through unchanged
     ) -> object:
-        key: Final = (id(router), name)
+        key: Final = _override_key(router, name)
         running: Final = _OVERRIDES_RUNNING.get()
         if key in running:
             return parent(router)(*args, **kwargs)
@@ -502,9 +510,9 @@ class DeprecatedRouterMembers:
     @staticmethod
     def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool:
         _note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)")
-        from litellm.router import Router
+        from litellm.router_python_core import PythonCore
 
-        return Router._core_deployment_usable_by_team(model, team_id)
+        return PythonCore._core_deployment_usable_by_team(model, team_id)
 
     _deployment_usable_by_team = deployment_usable_by_team
 

litellm/router_python_host.py

@@ -0,0 +1,135 @@
+"""
+``PythonHost``: what the router core reaches outside itself, served from the litellm package.
+
+The core never touches ``litellm.*`` directly. Inference calls (``invoke``), backoff sleeps,
+callback registration and the process-wide cache all go through the host, and the process settings
+come from ``dynamic()`` (``litellm/router_config.py``). A Rust core gets the same operations.
+"""
+
+import asyncio
+from collections.abc import Awaitable, Callable, Sequence
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    Literal,
+    TypeAlias,
+    cast,  # noqa: TID251  # untyped litellm globals: the live config view and the callback lists
+)
+
+from typing_extensions import overload
+
+import litellm
+from litellm.router_config import RouterDynamicConfig, live_dynamic_config
+
+if TYPE_CHECKING:
+    from pydantic import BaseModel
+
+    from litellm.caching.caching import Cache
+    from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
+    from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
+    from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
+    from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIFileObject
+    from litellm.types.rerank import RerankResponse
+    from litellm.types.utils import (
+        AdapterCompletionStreamWrapper,
+        EmbeddingResponse,
+        ImageResponse,
+        LiteLLMBatch,
+        ModelInfo,
+        ModelResponse,
+        TextCompletionResponse,
+        TranscriptionResponse,
+    )
+
+CallbackList: TypeAlias = Literal[
+    "success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback", "input_callback"
+]
+
+Endpoint: TypeAlias = Callable[..., object]
+
+
+class PythonHost:
+    __slots__ = ()
+
+    # The endpoints whose results the router core reads, typed as litellm declares them; any other
+    # op gets the untyped Endpoint.
+    @overload
+    def invoke(self, op: Literal["acompletion"]) -> Callable[..., Awaitable["ModelResponse | CustomStreamWrapper"]]: ...
+    @overload
+    def invoke(
+        self, op: Literal["atext_completion"]
+    ) -> Callable[..., Awaitable["TextCompletionResponse | TextCompletionStreamWrapper"]]: ...
+    @overload
+    def invoke(
+        self, op: Literal["aadapter_completion"]
+    ) -> Callable[..., Awaitable["BaseModel | AdapterCompletionStreamWrapper | None"]]: ...
+    @overload
+    def invoke(self, op: Literal["aembedding"]) -> Callable[..., Awaitable["EmbeddingResponse"]]: ...
+    @overload
+    def invoke(self, op: Literal["aimage_generation"]) -> Callable[..., Awaitable["ImageResponse"]]: ...
+    @overload
+    def invoke(self, op: Literal["atranscription"]) -> Callable[..., Awaitable["TranscriptionResponse"]]: ...
+    @overload
+    def invoke(self, op: Literal["aspeech"]) -> Callable[..., Awaitable["HttpxBinaryResponseContent"]]: ...
+    @overload
+    def invoke(self, op: Literal["arerank"]) -> Callable[..., Awaitable["RerankResponse"]]: ...
+    @overload
+    def invoke(self, op: Literal["acreate_file"]) -> Callable[..., Awaitable["OpenAIFileObject"]]: ...
+    @overload
+    def invoke(self, op: Literal["acreate_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
+    @overload
+    def invoke(self, op: Literal["aretrieve_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
+    @overload
+    def invoke(self, op: Literal["acancel_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
+    @overload
+    def invoke(self, op: Literal["alist_batches"]) -> Callable[..., Awaitable[object]]: ...
+    @overload
+    def invoke(self, op: Literal["get_llm_provider"]) -> Callable[..., "tuple[str, str, str | None, str | None]"]: ...
+    @overload
+    def invoke(self, op: Literal["get_model_info"]) -> Callable[..., "ModelInfo"]: ...
+    @overload
+    def invoke(self, op: Literal["get_supported_openai_params"]) -> Callable[..., "Sequence[str] | None"]: ...
+    @overload
+    def invoke(self, op: Literal["token_counter"]) -> Callable[..., int]: ...
+    @overload
+    def invoke(self, op: Literal["calculate_retry_after"]) -> Callable[..., float]: ...
+    @overload
+    def invoke(self, op: str) -> Endpoint: ...
+    def invoke(self, op: str) -> Endpoint:
+        """The litellm function named ``op`` (``acompletion``, ``get_model_info``, ...), looked up at call time."""
+        endpoint: Final[object] = getattr(litellm, op, None)
+        if endpoint is None:
+            raise AttributeError(f"litellm has no endpoint {op!r}")
+        if not callable(endpoint):
+            raise TypeError(f"litellm.{op} is not an endpoint")
+        return endpoint
+
+    def sleep(self, seconds: float) -> Awaitable[None]:
+        return asyncio.sleep(seconds)
+
+    def dynamic(self) -> RouterDynamicConfig:
+        """The process settings, read live on each access (``current_dynamic_config()`` takes a snapshot)."""
+        return cast(RouterDynamicConfig, live_dynamic_config)  # cast-ok: the live view reads the same fields
+
+    @property
+    def callbacks(self) -> "LoggingCallbackManager":
+        return litellm.logging_callback_manager
+
+    def callback_list(self, name: CallbackList) -> list[object]:  # mutable-ok: the process callback list itself
+        value: Final = getattr(litellm, name)
+        return cast(list[object], value)  # cast-ok: the process callback list is an untyped litellm global
+
+    def set_callback_list(self, name: CallbackList, value: list[object]) -> None:  # mutable-ok: assigned as given
+        setattr(litellm, name, value)
+
+    def global_cache(self) -> "Cache | None":
+        return litellm.cache
+
+    def set_global_cache(self, cache: "Cache | None") -> None:
+        litellm.cache = cache
+
+    def suppress_debug_info(self) -> None:
+        litellm.suppress_debug_info = True
+
+
+python_host: Final = PythonHost()

tests/code_coverage_tests/recursive_detector.py

@@ -3,6 +3,7 @@ import os
 
 IGNORE_FUNCTIONS = [
     "_json_cost",  # bounded at depth 32 and consumed under byte/node limits.
+    "detached",  # router_config: copies only the plain containers of the router's own constructor arguments, once per build or changeset
     "_format_type",
     "remove_additional_properties",
     "remove_strict_from_schema",

tests/code_coverage_tests/router_litellm_globals_check.py

@@ -0,0 +1,54 @@
+"""
+The router and its cores never read or write ``litellm.*``.
+
+Process settings reach the router through ``RouterDynamicConfig`` (litellm/router_config.py) and
+everything else outside it (inference calls, callbacks, the global cache) through ``PythonHost``
+(litellm/router_python_host.py). Those two modules are the only router code allowed to touch the
+``litellm`` package namespace. This check fails on ``import litellm`` (or ``import litellm.<x>``,
+which binds the same name) and on any ``litellm.<attr>`` access in the files below.
+
+Run: python tests/code_coverage_tests/router_litellm_globals_check.py
+"""
+
+import ast
+import os
+import sys
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+CHECKED = (
+    "litellm/router.py",
+    "litellm/router_api.py",
+    "litellm/router_core.py",
+    "litellm/router_deprecated.py",
+    "litellm/router_python_core.py",
+)
+
+
+def violations(rel: str) -> list[str]:
+    tree = ast.parse(open(os.path.join(ROOT, rel)).read(), rel)
+    found = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            found.extend(
+                f"{rel}:{node.lineno}: import {alias.name}"
+                for alias in node.names
+                if alias.asname is None and (alias.name == "litellm" or alias.name.startswith("litellm."))
+            )
+        elif isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id == "litellm":
+            found.append(f"{rel}:{node.lineno}: litellm.{node.attr}")
+    return found
+
+
+def main() -> int:
+    found = [line for rel in CHECKED for line in violations(rel)]
+    for line in found:
+        print(line)
+    if found:
+        print(f"{len(found)} litellm.* uses in router modules; read settings from RouterDynamicConfig and use PythonHost")
+        return 1
+    print("router modules use no litellm.* globals")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/code_coverage_tests/router_surface_check.py

@@ -38,11 +38,15 @@ ALLOWLIST = os.path.join(os.path.dirname(__file__), "router_surface_allowlist.js
 ROUTER_FILES = {
     "litellm/router.py",
     "litellm/router_api.py",
+    "litellm/router_config.py",
+    "litellm/router_core.py",
+    "litellm/router_python_core.py",
+    "litellm/router_python_host.py",
     "litellm/router_deprecated.py",
     "litellm/router_host.py",
 }
-ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_deprecated.py"]
-ROUTER_CLASS_NAMES = {"Router", "DeprecatedRouterMembers"}
+ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_python_core.py", "litellm/router_deprecated.py"]
+ROUTER_CLASS_NAMES = {"Router", "PythonCore", "DeprecatedRouterMembers"}
 
 SCAN_ROOTS = ["litellm", "enterprise"]
 SKIP_DIRS = {

tests/unit/router_strategy/test_router_routing_groups.py

@@ -236,7 +236,7 @@ async def test_async_dispatch_uses_group_strategy_for_grouped_model():
             wraps=group_selector.async_get_available_deployments,
         ) as latency_spy,
         patch(
-            "litellm.router.simple_shuffle", wraps=litellm.router.simple_shuffle
+            "litellm.router_python_core.simple_shuffle", wraps=litellm.router_python_core.simple_shuffle
         ) as shuffle_spy,
     ):
         await router.async_get_available_deployment(
@@ -271,7 +271,7 @@ async def test_async_dispatch_falls_back_to_default_for_ungrouped_models():
             wraps=group_selector.async_get_available_deployments,
         ) as latency_spy,
         patch(
-            "litellm.router.simple_shuffle", wraps=litellm.router.simple_shuffle
+            "litellm.router_python_core.simple_shuffle", wraps=litellm.router_python_core.simple_shuffle
         ) as shuffle_spy,
     ):
         await router.async_get_available_deployment(
@@ -331,7 +331,7 @@ async def test_async_dispatch_uses_default_selector_when_constructed_with_enum()
             wraps=default_selector.async_get_available_deployments,
         ) as latency_spy,
         patch(
-            "litellm.router.simple_shuffle", wraps=litellm.router.simple_shuffle
+            "litellm.router_python_core.simple_shuffle", wraps=litellm.router_python_core.simple_shuffle
         ) as shuffle_spy,
     ):
         await router.async_get_available_deployment(
@@ -1376,7 +1376,7 @@ async def test_group_call_dispatches_via_group_selector():
             "async_get_available_deployments",
             wraps=group_selector.async_get_available_deployments,
         ) as latency_spy,
-        patch("litellm.router.simple_shuffle", wraps=litellm.router.simple_shuffle) as shuffle_spy,
+        patch("litellm.router_python_core.simple_shuffle", wraps=litellm.router_python_core.simple_shuffle) as shuffle_spy,
     ):
         deployment = await router.async_get_available_deployment(model="quality", request_kwargs={})
 

tests/unit/router_strategy/test_router_tag_routing.py

@@ -2686,7 +2686,7 @@ async def test_plain_tag_exhaustion_with_universal_default_tag_raises_by_default
     router = _quality_high_cost_low_router()
 
     with patch(
-        "litellm.router.async_get_cooldown_deployments",
+        "litellm.router_python_core.async_get_cooldown_deployments",
         new=AsyncMock(return_value=["quality-high-1", "quality-high-2"]),
     ):
         with pytest.raises(Exception, match='Not allowed to access model due to tags configuration\\.') as exc_info:
@@ -2711,7 +2711,7 @@ async def test_plain_tag_exhaustion_with_universal_default_tag_falls_open_when_a
     from unittest.mock import AsyncMock, patch
 
     with patch(
-        "litellm.router.async_get_cooldown_deployments",
+        "litellm.router_python_core.async_get_cooldown_deployments",
         new=AsyncMock(return_value=["quality-high-1", "quality-high-2"]),
     ):
         response = await router.acompletion(

tests/unit/router_utils/test_cooldown_handlers.py

@@ -1303,7 +1303,7 @@ class TestDeploymentCallbackOnFailureCooldownTimePrecedence:
             },
         }
 
-        with patch("litellm.router.set_cooldown_deployments") as mock_set_cooldown:
+        with patch("litellm.router_python_core.set_cooldown_deployments") as mock_set_cooldown:
             router.deployment_callback_on_failure(
                 kwargs=kwargs,
                 completion_response=None,
@@ -1339,7 +1339,7 @@ class TestDeploymentCallbackOnFailureCooldownTimePrecedence:
             },
         }
 
-        with patch("litellm.router.set_cooldown_deployments") as mock_set_cooldown:
+        with patch("litellm.router_python_core.set_cooldown_deployments") as mock_set_cooldown:
             router.deployment_callback_on_failure(
                 kwargs=kwargs,
                 completion_response=None,

tests/unit/router_utils/test_kubernetes_pod_discovery.py

@@ -17,6 +17,7 @@ import respx
 from openai import AsyncOpenAI, OpenAI
 
 import litellm
+import litellm.router_python_core
 from litellm.constants import (
     CALLER_SESSION_ID_METADATA_KEY,
     KUBERNETES_POD_ROUTING_KEY,
@@ -281,7 +282,7 @@ def _patch_router_discovery(
     *discoveries: KubernetesPodDiscovery,
 ) -> None:
     discovery_iter: Final = iter(discoveries)
-    monkeypatch.setattr(litellm.router, "KubernetesPodDiscovery", lambda: next(discovery_iter))
+    monkeypatch.setattr(litellm.router_python_core, "KubernetesPodDiscovery", lambda: next(discovery_iter))
 
 
 def _stub_sync_dns(monkeypatch: pytest.MonkeyPatch, *ips: str) -> None:

tests/unit/test_conftest_isolation.py

@@ -117,12 +117,12 @@ def test_live_router_membership_is_scoped_to_this_test():
             }
         ]
     )
-    assert _CanaryRouterHolder.router in litellm_router_module._live_routers
+    assert _CanaryRouterHolder.router._core in litellm_router_module._live_routers
 
 
 def test_live_router_membership_was_rolled_back():
     assert _CanaryRouterHolder.router is not None
-    assert _CanaryRouterHolder.router not in litellm_router_module._live_routers
+    assert _CanaryRouterHolder.router._core not in litellm_router_module._live_routers
 
 
 def test_aws_cache_reset_sees_new_handlers_and_replaced_caches(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -175,7 +175,7 @@ def registered_collection_models() -> Iterator[None]:
     yield
     litellm_utils_module._runtime_registered_model_cost.clear()
     litellm_utils_module._runtime_registered_model_cost.update(original_registrations)
-    litellm_router_module._live_routers.discard(collection_router)
+    litellm_router_module._live_routers.discard(collection_router._core)
     litellm_router_module._live_routers.update(original_routers)
 
 

tests/unit/test_router/core/conftest.py

@@ -0,0 +1,26 @@
+from collections.abc import Callable
+
+import pytest
+
+from litellm.router_core import RouterCore
+from tests.unit.test_router.core.scripted_host import CORE_BUILDERS, CoreDeclined, ScriptedHost
+
+
+@pytest.fixture
+def host() -> ScriptedHost:
+    return ScriptedHost()
+
+
+@pytest.fixture(params=sorted(CORE_BUILDERS))
+def make_core(request: pytest.FixtureRequest, host: ScriptedHost) -> Callable[..., RouterCore]:
+    """``make_core(**router_arguments)``: the core under test, on the ``host`` fixture. Every test that
+    uses it runs once per core in ``CORE_BUILDERS`` (``-k python`` or ``-k rust`` picks one)."""
+    builder = CORE_BUILDERS[request.param]
+
+    def build(**arguments: object) -> RouterCore:  # kwargs-ok: the router's constructor arguments
+        try:
+            return builder(host, **arguments)
+        except CoreDeclined as declined:
+            pytest.skip(f"{request.param} core declines this configuration: {declined}")
+
+    return build

tests/unit/test_router/core/scripted_host.py

@@ -0,0 +1,487 @@
+"""
+A host for driving ``PythonCore`` in isolation.
+
+``PythonCore`` reaches outside itself only through its host (``litellm/router_python_host.py``):
+inference calls, backoff sleeps, the process settings, callback registration and the process-wide
+cache. ``ScriptedHost`` serves all of those from the test. Inference ops answer from a per-model
+script and record every call; sleeps are recorded and return at once; the process settings are a
+fixed ``RouterDynamicConfig`` the test can replace; callbacks live in the host's own lists, and an
+answered call runs the success or failure callbacks the core registered, as litellm's logging does.
+Ops that are not inference (``get_model_info``, ``should_retry``, ...) are litellm's own functions.
+"""
+
+import dataclasses
+import itertools
+from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping
+from types import MethodType
+from typing import Final
+
+import litellm
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
+from litellm.router_config import RouterDynamicConfig, build_router_config
+from litellm.router_core import RouterCore
+from litellm.router_python_core import PythonCore
+from litellm.router_python_host import CallbackList, Endpoint, PythonHost
+from litellm.types.utils import EmbeddingResponse, ModelResponse
+
+INFERENCE_OPS: Final = frozenset(
+    {
+        "acompletion",
+        "completion",
+        "atext_completion",
+        "text_completion",
+        "aembedding",
+        "embedding",
+        "aimage_generation",
+        "image_generation",
+        "aspeech",
+        "atranscription",
+        "arerank",
+        "rerank",
+        "amoderation",
+        "aresponses",
+        "responses",
+        "anthropic_messages",
+        "agenerate_content",
+        "aadapter_generate_content",
+        "aocr",
+        "asearch",
+        "avideo_generation",
+        "acreate_batch",
+        "acreate_file",
+        "afile_content",
+        "afile_delete",
+        "acreate_fine_tuning_job",
+        "aget_responses",
+        "adelete_responses",
+        "acancel_responses",
+        "alist_input_items",
+    }
+)
+
+Outcome = object
+"""What a scripted call answers: a ``BaseException`` instance is raised, a callable is called with
+the request's keyword arguments (and its result used in turn), anything else is returned as is.
+``None`` builds a default response for the op."""
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class Call:
+    op: str
+    model: str
+    kwargs: Mapping[str, object]
+
+    @property
+    def deployment_id(self) -> str | None:
+        model_info: Final = self.kwargs.get("model_info")
+        return str(model_info["id"]) if isinstance(model_info, Mapping) and "id" in model_info else None
+
+    @property
+    def model_group(self) -> str | None:
+        metadata: Final = self.kwargs.get("metadata") or self.kwargs.get("litellm_metadata")
+        return metadata.get("model_group") if isinstance(metadata, Mapping) else None
+
+
+def completion_response(content: str = "ok", total_tokens: int = 10, model: str = "gpt-4o") -> ModelResponse:
+    return ModelResponse(
+        model=model,
+        choices=[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": content}}],
+        usage={
+            "prompt_tokens": total_tokens // 2,
+            "completion_tokens": total_tokens - total_tokens // 2,
+            "total_tokens": total_tokens,
+        },
+    )
+
+
+def embedding_response(model: str = "text-embedding-3-small") -> EmbeddingResponse:
+    return EmbeddingResponse(
+        model=model,
+        data=[{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
+        usage={"prompt_tokens": 2, "total_tokens": 2},
+    )
+
+
+def _fresh_error(template: BaseException) -> BaseException:
+    """A copy of ``template`` with the same attributes (``copy.copy`` rebuilds litellm errors from
+    their args and drops ``status_code`` and the rest)."""
+    error: Final = BaseException.__new__(type(template))
+    error.args = template.args
+    error.__dict__.update(template.__dict__)
+    return error
+
+
+def _default_response(op: str, model: str) -> object:
+    if op in {"aembedding", "embedding"}:
+        return embedding_response(model)
+    return completion_response(model=model)
+
+
+def rate_limit(model: str = "gpt-4o", message: str = "slow down") -> litellm.RateLimitError:
+    return litellm.RateLimitError(message=message, llm_provider="openai", model=model)
+
+
+def bad_request(model: str = "gpt-4o", message: str = "bad input") -> litellm.BadRequestError:
+    return litellm.BadRequestError(message=message, llm_provider="openai", model=model)
+
+
+def context_window(model: str = "gpt-4o") -> litellm.ContextWindowExceededError:
+    return litellm.ContextWindowExceededError(message="too long", llm_provider="openai", model=model)
+
+
+def content_policy(model: str = "gpt-4o") -> litellm.ContentPolicyViolationError:
+    return litellm.ContentPolicyViolationError(message="blocked", llm_provider="openai", model=model)
+
+
+def server_error(model: str = "gpt-4o") -> litellm.InternalServerError:
+    return litellm.InternalServerError(message="boom", llm_provider="openai", model=model)
+
+
+def timeout(model: str = "gpt-4o") -> litellm.Timeout:
+    return litellm.Timeout(message="timed out", llm_provider="openai", model=model)
+
+
+def auth_error(model: str = "gpt-4o") -> litellm.AuthenticationError:
+    return litellm.AuthenticationError(message="bad key", llm_provider="openai", model=model)
+
+
+def dynamic_config(**changes: object) -> RouterDynamicConfig:  # kwargs-ok: RouterDynamicConfig fields
+    """Process settings as a fresh litellm process has them, with no callbacks, cache or fallbacks."""
+    base: Final = RouterDynamicConfig(
+        version=1,
+        num_retries=None,
+        max_fallbacks=None,
+        router_max_fallbacks=5,
+        allowed_fails=3,
+        request_timeout=6000.0,
+        fallbacks=None,
+        default_fallbacks=None,
+        context_window_fallbacks=None,
+        content_policy_fallbacks=None,
+        drop_params=None,
+        provider_list=litellm.provider_list,
+        expose_router_debug_in_errors=False,
+        redact_user_api_key_info=None,
+        model_alias_map={},
+        model_cost=litellm.model_cost,
+        known_custom_logger_compatible_callbacks=litellm._known_custom_logger_compatible_callbacks,
+        exception_types=litellm.LITELLM_EXCEPTION_TYPES,
+        callbacks=[],
+        success_callback=[],
+        failure_callback=[],
+        input_callback=[],
+        callback_manager=litellm.logging_callback_manager,
+        cache=None,
+    )
+    return dataclasses.replace(base, **changes)
+
+
+class HostCallbacks:
+    """The callback registration the core asks its host for, over the host's own lists."""
+
+    def __init__(self, host: "ScriptedHost") -> None:
+        self._host: Final = host
+        self._keys: Final = LoggingCallbackManager()
+
+    def _add(self, name: CallbackList, callback: object) -> None:
+        target: Final = self._host.lists[name]
+        if callback not in target:
+            target.append(callback)
+
+    def add_litellm_callback(self, callback: object) -> None:
+        self._add_to("callbacks", callback)
+
+    def _add_to(self, name: str, callback: object) -> None:
+        target: Final = self._host.lists.setdefault(name, [])
+        if callback not in target:
+            target.append(callback)
+
+    def add_litellm_success_callback(self, callback: object) -> None:
+        self._add("success_callback", callback)
+
+    def add_litellm_async_success_callback(self, callback: object) -> None:
+        self._add("_async_success_callback", callback)
+
+    def add_litellm_failure_callback(self, callback: object) -> None:
+        self._add("failure_callback", callback)
+
+    def add_litellm_async_failure_callback(self, callback: object) -> None:
+        self._add("_async_failure_callback", callback)
+
+    def add_litellm_input_callback(self, callback: object) -> None:
+        self._add("input_callback", callback)
+
+    def get_custom_logger_key(self, custom_logger: CustomLogger) -> str:
+        return self._keys.get_custom_logger_key(custom_logger)
+
+    def with_custom_logger_keys(
+        self, candidates: Iterable[CustomLogger], keys: frozenset[str]
+    ) -> tuple[CustomLogger, ...]:
+        return self._keys.with_custom_logger_keys(candidates, keys)
+
+    def without_custom_logger_keys(self, callbacks: Iterable[object], keys: frozenset[str]) -> Iterator[object]:
+        return self._keys.without_custom_logger_keys(callbacks, keys)  # pyright: ignore[reportArgumentType]  # any callback list
+
+    def get_all_callbacks(self) -> list[object]:  # mutable-ok: matches LoggingCallbackManager
+        return [callback for callbacks in self._host.lists.values() for callback in callbacks]
+
+    def get_custom_loggers_for_type(
+        self, callback_type: type[CustomLogger]
+    ) -> list[CustomLogger]:  # mutable-ok: matches LoggingCallbackManager
+        found: Final[list[CustomLogger]] = []
+        for callback in self.get_all_callbacks():
+            if isinstance(callback, callback_type) and callback not in found:
+                found.append(callback)
+        return found
+
+    def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]:
+        dropped_ids: Final = frozenset(map(id, callbacks))
+        owner_ids: Final = frozenset(map(id, method_owners))
+
+        def keep(callback: object) -> bool:
+            if id(callback) in dropped_ids:
+                return False
+            return not (isinstance(callback, MethodType) and id(callback.__self__) in owner_ids)
+
+        vacated: Final = frozenset(
+            self.get_custom_logger_key(callback)
+            for callback in self._host.lists.get("callbacks", [])
+            if isinstance(callback, CustomLogger) and id(callback) in dropped_ids
+        )
+        for name, callbacks_in_list in self._host.lists.items():
+            self._host.lists[name] = [callback for callback in callbacks_in_list if keep(callback)]
+        return vacated
+
+
+class ScriptedHost(PythonHost):
+    """
+    ``script`` maps a deployment's ``litellm_params.model`` (or ``"*"`` for any) to the outcomes its
+    calls answer in turn; the last outcome repeats. A model with no script answers a default response.
+    """
+
+    __slots__ = ("_dynamic", "_scripts", "_turns", "cache", "calls", "lists", "raised", "sleeps", "suppressed")
+
+    def __init__(
+        self, script: Mapping[str, Iterable[Outcome]] | None = None, **dynamic: object
+    ) -> None:  # kwargs-ok: RouterDynamicConfig fields
+        self._scripts: dict[str, tuple[Outcome, ...]] = {k: tuple(v) for k, v in (script or {}).items()}
+        self._turns: dict[str, Iterator[int]] = {}
+        self._dynamic: RouterDynamicConfig = dynamic_config(**dynamic)
+        self.calls: Final[list[Call]] = []
+        self.sleeps: Final[list[float]] = []
+        # The errors the scripted calls raised, in order: each is a fresh copy of its scripted template.
+        self.raised: Final[list[BaseException]] = []
+        self.lists: Final[dict[str, list[object]]] = {
+            "callbacks": [],
+            "success_callback": [],
+            "_async_success_callback": [],
+            "failure_callback": [],
+            "_async_failure_callback": [],
+            "input_callback": [],
+        }
+        self.cache: object = None
+        self.suppressed = False
+
+    # --- what the test controls -------------------------------------------------------------
+
+    def answer(self, model: str, *outcomes: Outcome) -> None:
+        """Script the calls to ``model`` from now on."""
+        self._scripts[model] = outcomes
+        self._turns.pop(model, None)
+
+    def set_dynamic(self, **changes: object) -> None:  # kwargs-ok: RouterDynamicConfig fields
+        self._dynamic = dataclasses.replace(self._dynamic, **changes, version=self._dynamic.version + 1)
+
+    def calls_to(self, op: str | None = None) -> list[Call]:  # mutable-ok: a fresh list
+        return [call for call in self.calls if op is None or call.op == op]
+
+    def deployments_called(self, op: str | None = None) -> list[str | None]:  # mutable-ok: a fresh list
+        return [call.deployment_id for call in self.calls_to(op)]
+
+    # --- the host operations ----------------------------------------------------------------
+
+    def invoke(self, op: str) -> Endpoint:
+        if op not in INFERENCE_OPS:
+            return super().invoke(op)
+        if op.startswith("a") or op in {"anthropic_messages", "_arealtime"}:
+
+            async def endpoint(*args: object, **kwargs: object) -> object:  # kwargs-ok: the endpoint's request
+                return await self._answer_async(op, args, kwargs)
+
+            return endpoint
+
+        def sync_endpoint(*args: object, **kwargs: object) -> object:  # kwargs-ok: the endpoint's request
+            return self._answer_sync(op, args, kwargs)
+
+        return sync_endpoint
+
+    def sleep(self, seconds: float) -> Awaitable[None]:
+        self.sleeps.append(seconds)
+
+        async def done() -> None:
+            return None
+
+        return done()
+
+    def dynamic(self) -> RouterDynamicConfig:
+        return self._dynamic
+
+    @property
+    def callbacks(self) -> HostCallbacks:  # pyright: ignore[reportIncompatibleVariableOverride]  # the core uses the same methods
+        return HostCallbacks(self)
+
+    def callback_list(self, name: CallbackList) -> list[object]:  # mutable-ok: the host's own list
+        return self.lists.setdefault(name, [])
+
+    def set_callback_list(self, name: CallbackList, value: list[object]) -> None:  # mutable-ok: assigned as given
+        self.lists[name] = value
+
+    def global_cache(self) -> object:  # pyright: ignore[reportIncompatibleMethodOverride]  # any cache object
+        return self.cache
+
+    def set_global_cache(self, cache: object) -> None:  # pyright: ignore[reportIncompatibleMethodOverride]  # any cache object
+        self.cache = cache
+
+    def suppress_debug_info(self) -> None:
+        self.suppressed = True
+
+    # --- answering ------------------------------------------------------------------------
+
+    def _outcome(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> tuple[Call, object]:
+        model: Final = str(kwargs.get("model") or (args[0] if args else ""))
+        call: Final = Call(op=op, model=model, kwargs=dict(kwargs))
+        self.calls.append(call)
+        key: Final = model if model in self._scripts else "*" if "*" in self._scripts else None
+        if key is None:
+            return call, None
+        outcomes: Final = self._scripts[key]
+        turn: Final = next(self._turns.setdefault(key, itertools.count()))
+        outcome: object = outcomes[min(turn, len(outcomes) - 1)] if outcomes else None
+        while callable(outcome) and not isinstance(outcome, BaseException | type):
+            outcome = outcome(**kwargs)  # rebind-ok: a callable outcome answers with what it returns
+        if isinstance(outcome, BaseException):
+            # A fresh error per call: the core stamps attributes (num_retries, ...) on the one it raises.
+            outcome = _fresh_error(outcome)  # rebind-ok: the scripted error is a template
+            self.raised.append(outcome)
+        return call, outcome
+
+    def _logging_kwargs(
+        self, call: Call, error: BaseException | None, response: object
+    ) -> dict[str, object]:  # mutable-ok: litellm's logging kwargs
+        kwargs: Final = call.kwargs
+        usage: Final = getattr(response, "usage", None)
+        metadata: Final = kwargs.get("metadata") or kwargs.get("litellm_metadata") or {}
+        model_info: Final = kwargs.get("model_info") or {}
+        return {
+            "call_type": call.op,
+            "model": call.model,
+            "exception": error,
+            "litellm_params": {
+                "metadata": metadata,
+                "model_info": model_info,
+                "api_base": kwargs.get("api_base"),
+                **({"cooldown_time": kwargs["cooldown_time"]} if "cooldown_time" in kwargs else {}),
+            },
+            "standard_logging_object": {
+                "total_tokens": getattr(usage, "total_tokens", 0) or 0,
+                "model_group": metadata.get("model_group"),
+                "model_id": model_info.get("id"),
+                "hidden_params": {"litellm_model_name": call.model},
+            },
+        }
+
+    def _finish(self, op: str, call: Call, outcome: object) -> tuple[BaseException | None, object]:
+        if isinstance(outcome, type) and issubclass(outcome, BaseException):
+            outcome = outcome("scripted failure")  # rebind-ok: an exception class raises an instance
+        if isinstance(outcome, BaseException):
+            return outcome, None
+        return None, _default_response(op, call.model) if outcome is None else outcome
+
+    async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object:
+        call, outcome = self._outcome(op, args, kwargs)
+        error, response = self._finish(op, call, outcome)
+        logged: Final = self._logging_kwargs(call, error, response)
+        if error is not None:
+            for callback in list(self.lists.get("failure_callback", ())):
+                if callable(callback):
+                    callback(logged, None, 0.0, 0.0)
+            for callback in list(self.lists.get("_async_failure_callback", ())):
+                if callable(callback):
+                    await callback(logged, None, 0.0, 0.0)
+            raise error
+        for callback in list(self.lists.get("_async_success_callback", ())):
+            if callable(callback):
+                await callback(logged, response, 0.0, 0.0)
+        return response
+
+    def _answer_sync(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object:
+        call, outcome = self._outcome(op, args, kwargs)
+        error, response = self._finish(op, call, outcome)
+        logged: Final = self._logging_kwargs(call, error, response)
+        if error is not None:
+            for callback in list(self.lists.get("failure_callback", ())):
+                if callable(callback):
+                    callback(logged, None, 0.0, 0.0)
+            raise error
+        for callback in list(self.lists.get("success_callback", ())):
+            if callable(callback):
+                callback(logged, response, 0.0, 0.0)
+        return response
+
+
+def deployment(
+    model_name: str,
+    deployment_id: str,
+    model: str | None = None,
+    **params: object,  # kwargs-ok: extra litellm_params
+) -> dict[str, object]:  # mutable-ok: a model_list entry
+    """One ``model_list`` entry; ``model`` defaults to ``openai/<deployment_id>`` so each deployment
+    answers its own script."""
+    model_info: Final = params.pop("model_info", None)
+    return {
+        "model_name": model_name,
+        "litellm_params": {"model": model or f"openai/{deployment_id}", "api_key": "sk-test", **params},
+        "model_info": {"id": deployment_id, **(model_info if isinstance(model_info, dict) else {})},
+    }
+
+
+def python_core(host: ScriptedHost, **arguments: object) -> PythonCore:  # kwargs-ok: the router's constructor arguments
+    """A ``PythonCore`` built the way ``Router`` builds one, on ``host``."""
+    config: Final = build_router_config(arguments)
+    return PythonCore(**config.arguments, _host=host)  # pyright: ignore[reportArgumentType]  # the same arguments Router binds
+
+
+CoreBuilder = Callable[..., RouterCore]
+"""``builder(host, **router_arguments)``: a router core on ``host``."""
+
+CORE_BUILDERS: Final[dict[str, CoreBuilder]] = {"python": python_core}
+"""Every core the suite runs on. Another core adds its builder here; a builder that cannot take a
+configuration raises ``CoreDeclined``, and the test is skipped for that core."""
+
+
+class CoreDeclined(Exception):
+    """The core does not serve this configuration (the router would stay on the Python core)."""
+
+
+__all__ = [
+    "Call",
+    "HostCallbacks",
+    "INFERENCE_OPS",
+    "CORE_BUILDERS",
+    "CoreBuilder",
+    "CoreDeclined",
+    "ScriptedHost",
+    "auth_error",
+    "bad_request",
+    "completion_response",
+    "content_policy",
+    "context_window",
+    "deployment",
+    "dynamic_config",
+    "embedding_response",
+    "python_core",
+    "rate_limit",
+    "server_error",
+    "timeout",
+]

tests/unit/test_router/core/test_endpoints.py

@@ -0,0 +1,445 @@
+"""
+What a router core sends to its host for each endpoint, and what it hands back.
+
+Every test builds its core through ``make_core`` on the ``host`` fixture, so it runs once per core.
+"""
+
+import datetime
+import io
+from collections.abc import AsyncIterator, Callable, Coroutine, Mapping
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
+from litellm.router_core import RouterCore
+from litellm.types.utils import EmbeddingResponse, ModelResponse, ModelResponseStream
+from tests.unit.test_router.core.scripted_host import (
+    ScriptedHost,
+    bad_request,
+    completion_response,
+    deployment,
+    embedding_response,
+)
+
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+MakeCore = Callable[..., RouterCore]
+
+EndpointCall = Callable[[RouterCore], object]
+
+ENDPOINTS: Final[dict[str, tuple[str, EndpointCall]]] = {
+    "acompletion": ("acompletion", lambda core: core.acompletion(model="g", messages=MESSAGES)),
+    "completion": ("completion", lambda core: core.completion(model="g", messages=MESSAGES)),
+    "aembedding": ("aembedding", lambda core: core.aembedding(model="g", input=["a"])),
+    "embedding": ("embedding", lambda core: core.embedding(model="g", input=["a"])),
+    "atext_completion": ("atext_completion", lambda core: core.atext_completion(model="g", prompt="hi")),
+    "aimage_generation": ("aimage_generation", lambda core: core.aimage_generation(model="g", prompt="a cat")),
+    "image_generation": ("image_generation", lambda core: core.image_generation(model="g", prompt="a cat")),
+    "arerank": ("arerank", lambda core: core.arerank(model="g", query="q", documents=["a"])),
+    "aspeech": ("aspeech", lambda core: core.aspeech(model="g", input="hi", voice="alloy")),
+    "atranscription": ("atranscription", lambda core: core.atranscription(model="g", file=io.BytesIO(b"x"))),
+    "aresponses": ("aresponses", lambda core: core.aresponses(model="g", input="hi")),
+    "aanthropic_messages": (
+        "anthropic_messages",
+        lambda core: core.aanthropic_messages(model="g", messages=MESSAGES, max_tokens=5),
+    ),
+    "call-acompletion": ("acompletion", lambda core: core.call("acompletion", {"model": "g", "messages": MESSAGES})),
+    "call-completion": ("completion", lambda core: core.call("completion", {"model": "g", "messages": MESSAGES})),
+}
+SYNC_ENDPOINTS: Final = frozenset({"completion", "embedding", "image_generation"})
+LITELLM_METADATA_ENDPOINTS: Final = frozenset({"aresponses", "aanthropic_messages"})
+EMBEDDING_ENDPOINTS: Final = frozenset({"aembedding", "embedding"})
+
+
+async def _invoke(core: RouterCore, endpoint: str) -> object:
+    result: Final = ENDPOINTS[endpoint][1](core)
+    if endpoint in SYNC_ENDPOINTS:
+        return result
+    return await result  # pyright: ignore[reportGeneralTypeIssues]  # the async endpoints return awaitables
+
+
+def _one_group(
+    make_core: MakeCore, **litellm_params: object
+) -> RouterCore:  # kwargs-ok: the deployment's litellm_params
+    return make_core(model_list=[deployment("g", "d1", **litellm_params)])
+
+
+def _metadata(kwargs: Mapping[str, object], endpoint: str) -> Mapping[str, object]:
+    bucket: Final = kwargs["litellm_metadata" if endpoint in LITELLM_METADATA_ENDPOINTS else "metadata"]
+    assert isinstance(bucket, Mapping)
+    return bucket
+
+
+@pytest.mark.parametrize("endpoint", sorted(ENDPOINTS))
+async def test_each_endpoint_sends_the_deployment_litellm_params_to_the_host(
+    make_core: MakeCore, host: ScriptedHost, endpoint: str
+):
+    core: Final = _one_group(make_core, api_base="https://d1.example", temperature=0.3, seed=7)
+    await _invoke(core, endpoint)
+    sent: Final = host.calls[0].kwargs
+    assert (sent["model"], sent["api_key"], sent["api_base"], sent["temperature"], sent["seed"]) == (
+        "openai/d1",
+        "sk-test",
+        "https://d1.example",
+        0.3,
+        7,
+    )
+
+
+@pytest.mark.parametrize("endpoint", sorted(ENDPOINTS))
+async def test_each_endpoint_invokes_its_host_op_once_with_routing_metadata(
+    make_core: MakeCore, host: ScriptedHost, endpoint: str
+):
+    core: Final = _one_group(make_core, api_base="https://d1.example")
+    await _invoke(core, endpoint)
+    assert [call.op for call in host.calls] == [ENDPOINTS[endpoint][0]]
+    call: Final = host.calls[0]
+    metadata: Final = _metadata(call.kwargs, endpoint)
+    assert (call.model_group, call.deployment_id) == ("g", "d1")
+    assert (metadata["model_group"], metadata["deployment"], metadata["api_base"]) == (
+        "g",
+        "openai/d1",
+        "https://d1.example",
+    )
+    assert metadata["model_info"]["id"] == "d1"  # pyright: ignore[reportIndexIssue]  # metadata values are untyped
+
+
+@pytest.mark.parametrize("endpoint", sorted(ENDPOINTS))
+async def test_each_endpoint_returns_the_host_response_object_with_routing_hidden_params(
+    make_core: MakeCore, host: ScriptedHost, endpoint: str
+):
+    answer: Final[ModelResponse | EmbeddingResponse] = (
+        embedding_response() if endpoint in EMBEDDING_ENDPOINTS else completion_response("scripted")
+    )
+    host.answer("openai/d1", answer)
+    core: Final = _one_group(make_core)
+    response: Final = await _invoke(core, endpoint)
+    assert response is answer
+    assert response._hidden_params["model_id"] == "d1"
+    assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "g"
+
+
+@pytest.mark.parametrize("endpoint", sorted(SYNC_ENDPOINTS | {"call-completion"}))
+async def test_sync_endpoints_return_the_response_not_an_awaitable(
+    make_core: MakeCore, host: ScriptedHost, endpoint: str
+):
+    answer: Final = embedding_response() if endpoint == "embedding" else completion_response("sync")
+    host.answer("openai/d1", answer)
+    core: Final = _one_group(make_core)
+    if endpoint == "call-completion":
+        assert await core.call("completion", {"model": "g", "messages": MESSAGES}) is answer
+    else:
+        assert ENDPOINTS[endpoint][1](core) is answer
+
+
+@pytest.mark.parametrize(
+    "endpoint, request_body",
+    [("acompletion", {"messages": MESSAGES}), ("completion", {"messages": MESSAGES})],
+)
+async def test_router_only_include_fallback_errors_is_not_sent_to_the_host(
+    make_core: MakeCore, host: ScriptedHost, endpoint: str, request_body: dict[str, object]
+):
+    core: Final = _one_group(make_core)
+    await core.call(endpoint, {"model": "g", "include_fallback_errors": True, **request_body})
+    assert host.calls[0].op == endpoint
+    assert "include_fallback_errors" not in host.calls[0].kwargs
+
+
+@pytest.mark.parametrize("router_only_param", ["rpm", "tpm", "max_parallel_requests"])
+async def test_deployment_limit_params_are_forwarded_with_the_rest_of_litellm_params(
+    make_core: MakeCore, host: ScriptedHost, router_only_param: str
+):
+    core: Final = _one_group(make_core, **{router_only_param: 50})
+    await core.acompletion(model="g", messages=MESSAGES)
+    assert host.calls[0].kwargs[router_only_param] == 50
+
+
+@pytest.mark.parametrize("endpoint", ["acompletion", "aembedding", "completion", "embedding"])
+async def test_request_metadata_reaches_the_host_beside_the_routing_metadata(
+    make_core: MakeCore, host: ScriptedHost, endpoint: str
+):
+    core: Final = _one_group(make_core)
+    request: Final[dict[str, object]] = {"messages": MESSAGES} if "completion" in endpoint else {"input": ["a"]}
+    await core.call(endpoint, {"model": "g", "metadata": {"trace": "t-1"}, **request})
+    metadata: Final = _metadata(host.calls[0].kwargs, endpoint)
+    assert (metadata["trace"], metadata["model_group"]) == ("t-1", "g")
+
+
+async def test_aresponses_sends_request_metadata_as_metadata_and_routing_metadata_as_litellm_metadata(
+    make_core: MakeCore, host: ScriptedHost
+):
+    core: Final = _one_group(make_core)
+    await core.aresponses(model="g", input="hi", metadata={"trace": "t-1"})
+    sent: Final = host.calls[0].kwargs
+    assert sent["metadata"] == {"trace": "t-1"}
+    assert sent["litellm_metadata"]["model_group"] == "g"  # pyright: ignore[reportIndexIssue]  # untyped kwargs
+
+
+@pytest.mark.parametrize(
+    "router_arguments, deployment_params, expected_timeout",
+    [
+        ({}, {}, 6000.0),
+        ({"timeout": 42}, {}, 42),
+        ({"timeout": 42}, {"timeout": 11}, 11.0),
+        ({}, {"timeout": 11}, 11.0),
+    ],
+)
+async def test_acompletion_timeout_comes_from_deployment_then_router_then_process_setting(
+    make_core: MakeCore,
+    host: ScriptedHost,
+    router_arguments: dict[str, object],
+    deployment_params: dict[str, object],
+    expected_timeout: float,
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", **deployment_params)], **router_arguments)
+    await core.acompletion(model="g", messages=MESSAGES)
+    assert host.calls[0].kwargs["timeout"] == expected_timeout
+
+
+@pytest.mark.parametrize("endpoint", ["acompletion", "aembedding", "aresponses", "aanthropic_messages"])
+async def test_request_timeout_overrides_router_and_deployment_timeouts(
+    make_core: MakeCore, host: ScriptedHost, endpoint: str
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", timeout=11)], timeout=42)
+    request: Final[dict[str, object]] = (
+        {"input": ["a"]}
+        if endpoint == "aembedding"
+        else {"input": "hi"}
+        if endpoint == "aresponses"
+        else {"messages": MESSAGES, "max_tokens": 5}
+    )
+    await core.call(endpoint, {"model": "g", "timeout": 7, **request})
+    assert host.calls[0].kwargs["timeout"] == 7
+
+
+async def test_aresponses_timeout_falls_back_to_router_timeout(make_core: MakeCore, host: ScriptedHost):
+    core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42)
+    await core.aresponses(model="g", input="hi")
+    assert host.calls[0].kwargs["timeout"] == 42
+
+
+@pytest.mark.parametrize(
+    "request_stream_timeout, expected_stream_timeout",
+    [(None, 3.0), (2, 2)],
+)
+async def test_streaming_acompletion_sends_deployment_or_request_stream_timeout(
+    make_core: MakeCore, host: ScriptedHost, request_stream_timeout: int | None, expected_stream_timeout: float
+):
+    core: Final = _one_group(make_core, timeout=11, stream_timeout=3)
+    extra: Final = {} if request_stream_timeout is None else {"stream_timeout": request_stream_timeout}
+    await core.acompletion(model="g", messages=MESSAGES, stream=True, **extra)
+    sent: Final = host.calls[0].kwargs
+    assert (sent["stream"], sent["timeout"], sent["stream_timeout"]) == (True, 11.0, expected_stream_timeout)
+
+
+@pytest.mark.parametrize(
+    "request_params, expected",
+    [
+        ({}, ("u1", 0.9)),
+        ({"temperature": 0.1}, ("u1", 0.1)),
+        ({"user": "u2"}, ("u2", 0.9)),
+    ],
+)
+async def test_default_litellm_params_fill_what_the_request_leaves_unset(
+    make_core: MakeCore, host: ScriptedHost, request_params: dict[str, object], expected: tuple[str, float]
+):
+    core: Final = make_core(
+        model_list=[deployment("g", "d1")], default_litellm_params={"user": "u1", "temperature": 0.9}
+    )
+    await core.acompletion(model="g", messages=MESSAGES, **request_params)
+    sent: Final = host.calls[0].kwargs
+    assert (sent["user"], sent["temperature"]) == expected
+
+
+@pytest.mark.parametrize(
+    "request_credentials, expected_api_base",
+    [
+        ({"api_key": "sk-client"}, "https://d1.example"),
+        ({"api_key": "sk-client", "api_base": "https://client.example"}, "https://client.example"),
+    ],
+)
+async def test_request_level_api_key_replaces_the_deployment_key_on_the_host_call(
+    make_core: MakeCore, host: ScriptedHost, request_credentials: dict[str, str], expected_api_base: str
+):
+    core: Final = _one_group(make_core, api_base="https://d1.example")
+    await core.acompletion(model="g", messages=MESSAGES, **request_credentials)
+    call: Final = host.calls[0]
+    assert (call.kwargs["api_key"], call.kwargs["api_base"], call.model_group) == (
+        "sk-client",
+        expected_api_base,
+        "g",
+    )
+
+
+async def test_construction_registers_one_success_and_one_failure_callback_per_mode(
+    make_core: MakeCore, host: ScriptedHost
+):
+    _one_group(make_core)
+    registered: Final = {name: len(callbacks) for name, callbacks in host.lists.items()}
+    assert registered == {
+        "callbacks": 0,
+        "success_callback": 1,
+        "_async_success_callback": 1,
+        "failure_callback": 1,
+        "_async_failure_callback": 1,
+        "input_callback": 0,
+    }
+    assert all(callable(callbacks[0]) for callbacks in host.lists.values() if callbacks)
+
+
+async def test_retire_removes_the_callbacks_the_core_registered(make_core: MakeCore, host: ScriptedHost):
+    core: Final = _one_group(make_core)
+    core.retire()
+    assert {name: callbacks for name, callbacks in host.lists.items() if callbacks} == {}
+
+
+async def test_sync_completion_counts_its_tokens_and_request_in_group_usage(make_core: MakeCore, host: ScriptedHost):
+    core: Final = _one_group(make_core, tpm=100, rpm=5)
+    assert await core.health().usage("g") == (None, None)
+    core.completion(model="g", messages=MESSAGES)
+    assert await core.health().usage("g") == (10, 1)
+    assert await core.health().usage("g", kind="remaining") == {
+        "x-ratelimit-remaining-tokens": 90,
+        "x-ratelimit-limit-tokens": 100,
+        "x-ratelimit-remaining-requests": 4,
+        "x-ratelimit-limit-requests": 5,
+    }
+
+
+class DeferredLoggingHost(ScriptedHost):
+    """``ScriptedHost`` that, like litellm, runs async success callbacks after the endpoint returns."""
+
+    __slots__ = ("pending",)
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.pending: list[Coroutine[object, object, None]] = []
+
+    async def _answer_async(self, op: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> object:
+        callbacks: Final = self.lists["_async_success_callback"]
+        self.lists["_async_success_callback"] = []
+        try:
+            response: Final = await super()._answer_async(op, args, kwargs)
+        finally:
+            self.lists["_async_success_callback"] = callbacks
+        call: Final = self.calls[-1]
+        logged: Final = self._logging_kwargs(call, None, response)
+        self.pending.extend(callback(logged, response, 0.0, 0.0) for callback in callbacks if callable(callback))
+        return response
+
+    async def drain(self) -> None:
+        while self.pending:
+            await self.pending.pop(0)
+
+
+class TestUsageWhenTheHostLogsAfterTheResponse:
+    """litellm runs async success callbacks after the endpoint has returned; this host does the same."""
+
+    @pytest.fixture
+    def host(self) -> DeferredLoggingHost:
+        return DeferredLoggingHost()
+
+    async def test_one_acompletion_counts_its_tokens_and_request_once(
+        self, make_core: MakeCore, host: DeferredLoggingHost
+    ):
+        core: Final = _one_group(make_core, tpm=100, rpm=5)
+        await core.acompletion(model="g", messages=MESSAGES)
+        assert await core.health().usage("g") == (10, 1)
+        await host.drain()
+        assert await core.health().usage("g") == (10, 1)
+
+
+async def test_usage_based_routing_registers_its_logger_on_the_host(make_core: MakeCore, host: ScriptedHost):
+    make_core(model_list=[deployment("g", "d1", rpm=1)], routing_strategy="usage-based-routing-v2")
+    assert [type(callback).__name__ for callback in host.lists["callbacks"]] == ["LowestTPMLoggingHandler_v2"]
+
+
+async def test_a_successful_call_uses_up_the_deployment_rpm_for_the_next_selection(
+    make_core: MakeCore, host: ScriptedHost
+):
+    core: Final = make_core(model_list=[deployment("g", "d1", rpm=1)], routing_strategy="usage-based-routing-v2")
+    await core.acompletion(model="g", messages=MESSAGES)
+    with pytest.raises(litellm.RateLimitError, match="No deployments available"):
+        await core.acompletion(model="g", messages=MESSAGES)
+    assert host.deployments_called() == ["d1"]
+
+
+async def test_host_error_reaches_the_caller_as_raised(make_core: MakeCore, host: ScriptedHost):
+    error: Final = bad_request(message="scripted bad input")
+    host.answer("openai/d1", error)
+    core: Final = _one_group(make_core)
+    with pytest.raises(litellm.BadRequestError, match="scripted bad input"):
+        await core.acompletion(model="g", messages=MESSAGES)
+    assert host.deployments_called("acompletion")[0] == "d1"
+
+
+@pytest.mark.parametrize("endpoint", ["acompletion", "completion"])
+async def test_unknown_model_group_raises_without_calling_the_host(
+    make_core: MakeCore, host: ScriptedHost, endpoint: str
+):
+    core: Final = _one_group(make_core)
+    with pytest.raises(litellm.BadRequestError, match="model=missing"):
+        await core.call(endpoint, {"model": "missing", "messages": MESSAGES})
+    assert host.calls == []
+
+
+@pytest.mark.parametrize("op", ["no_such_endpoint", "acompletions", ""])
+async def test_call_with_an_unknown_op_raises_attribute_error_without_calling_the_host(
+    make_core: MakeCore, host: ScriptedHost, op: str
+):
+    core: Final = _one_group(make_core)
+    with pytest.raises(AttributeError):
+        await core.call(op, {"model": "g"})
+    assert host.calls == []
+
+
+async def test_sync_text_completion_invokes_the_sync_host_op_with_deployment_credentials(
+    make_core: MakeCore, host: ScriptedHost
+):
+    core: Final = _one_group(make_core, api_base="https://d1.example")
+    core.text_completion(model="g", prompt="hi")
+    call: Final = host.calls[0]
+    assert (call.op, call.kwargs["api_base"], call.kwargs["prompt"], call.model_group) == (
+        "text_completion",
+        "https://d1.example",
+        "hi",
+        "g",
+    )
+
+
+def _scripted_stream(*texts: str) -> Callable[..., CustomStreamWrapper]:
+    def answer(**_request: object) -> CustomStreamWrapper:  # kwargs-ok: the host passes the request kwargs
+        async def chunks() -> AsyncIterator[ModelResponseStream]:
+            for text in texts:
+                yield ModelResponseStream(choices=[{"index": 0, "delta": {"role": "assistant", "content": text}}])
+
+        logging_obj: Final = LiteLLMLogging(
+            model="d1",
+            messages=MESSAGES,
+            stream=True,
+            call_type="acompletion",
+            start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
+            litellm_call_id="stream-test",
+            function_id="stream-test",
+        )
+        logging_obj.update_environment_variables(model="d1", user=None, optional_params={}, litellm_params={})
+        return CustomStreamWrapper(
+            completion_stream=chunks(), model="d1", custom_llm_provider="openai", logging_obj=logging_obj
+        )
+
+    return answer
+
+
+async def test_streaming_acompletion_yields_the_host_stream_chunks_with_routing_hidden_params(
+    make_core: MakeCore, host: ScriptedHost
+):
+    host.answer("openai/d1", _scripted_stream("he", "llo"))
+    core: Final = _one_group(make_core)
+    stream: Final = await core.acompletion(model="g", messages=MESSAGES, stream=True)
+    texts: Final = [chunk.choices[0].delta.content async for chunk in stream]  # pyright: ignore[reportUnknownMemberType]  # stream chunks are untyped
+    assert "".join(text for text in texts if text) == "hello"
+    assert stream._hidden_params["model_id"] == "d1"
+    assert stream._hidden_params["additional_headers"]["x-litellm-model-group"] == "g"
+    assert (host.calls[0].op, host.calls[0].kwargs["stream"]) == ("acompletion", True)

tests/unit/test_router/core/test_fallbacks.py

@@ -0,0 +1,756 @@
+"""Cross-group fallbacks of a router core: where the chain comes from, how it is walked, and what is raised."""
+
+from collections.abc import Callable, Mapping
+
+import pytest
+
+import litellm
+from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
+from litellm.router_api import ChangeSet
+from litellm.router_core import RouterCore
+from tests.unit.test_router.core.scripted_host import (
+    ScriptedHost,
+    bad_request,
+    content_policy,
+    context_window,
+    deployment,
+    rate_limit,
+    server_error,
+    timeout,
+)
+
+MakeCore = Callable[..., RouterCore]
+
+MESSAGES = [{"role": "user", "content": "hi"}]
+GROUPS_ABC = [deployment("a", "a"), deployment("b", "b"), deployment("c", "c")]
+GROUPS_ABCD = [*GROUPS_ABC, deployment("d", "d")]
+CHAIN_ABCD = [{"a": ["b"]}, {"b": ["c"]}, {"c": ["d"]}]
+TIERED_A = [deployment("a", "a1", order=1), deployment("a", "a2", order=2), deployment("b", "b")]
+
+
+def build(make_core: MakeCore, **arguments: object) -> RouterCore:  # kwargs-ok: the router's constructor arguments
+    return make_core(**{"model_list": GROUPS_ABC, "num_retries": 0, **arguments})
+
+
+async def complete(core: RouterCore, model: str = "a", **request: object) -> object:  # kwargs-ok: request kwargs
+    return await core.acompletion(model=model, messages=MESSAGES, **request)  # pyright: ignore[reportAttributeAccessIssue]  # endpoint method
+
+
+def headers(response: object) -> Mapping[str, object]:
+    return getattr(response, "_hidden_params")["additional_headers"]
+
+
+def show_router_debug(host: ScriptedHost) -> None:
+    host.set_dynamic(expose_router_debug_in_errors=True)
+
+
+# --- where the fallback chain comes from -------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("constructor", "process", "request_kwargs", "expected"),
+    [
+        ({"fallbacks": [{"a": ["b"]}]}, None, {}, ["a", "b"]),
+        ({}, [{"a": ["c"]}], {}, ["a", "c"]),
+        ({}, None, {"fallbacks": [{"a": ["b"]}]}, ["a", "b"]),
+        ({}, None, {"fallbacks": ["c"]}, ["a", "c"]),
+        ({"fallbacks": [{"a": ["b"]}]}, [{"a": ["c"]}], {}, ["a", "b"]),
+        ({"fallbacks": [{"a": ["b"]}]}, None, {"fallbacks": [{"a": ["c"]}]}, ["a", "c"]),
+    ],
+    ids=[
+        "constructor",
+        "process-setting",
+        "request-kwarg",
+        "request-plain-list",
+        "constructor-over-process-setting",
+        "request-over-constructor",
+    ],
+)
+async def test_a_failed_group_falls_back_to_the_chain_from_the_winning_source(
+    make_core: MakeCore,
+    host: ScriptedHost,
+    constructor: dict[str, object],
+    process: list[object] | None,
+    request_kwargs: dict[str, object],
+    expected: list[str],
+) -> None:
+    if process is not None:
+        host.set_dynamic(fallbacks=process)
+    core = build(make_core, **constructor)
+    host.answer("openai/a", rate_limit())
+
+    response = await complete(core, **request_kwargs)
+
+    assert host.deployments_called("acompletion") == expected
+    assert headers(response)["x-litellm-model-group"] == expected[-1]
+
+
+@pytest.mark.asyncio
+async def test_an_empty_request_fallback_list_overrides_the_constructor_chain(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core, fallbacks=[])
+
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("field", ["assign", "settings"])
+async def test_fallbacks_applied_by_changeset_take_effect(make_core: MakeCore, host: ScriptedHost, field: str) -> None:
+    core = build(make_core)
+    core.apply(ChangeSet(**{field: {"fallbacks": [{"a": ["c"]}]}}))  # pyright: ignore[reportArgumentType]  # assign or settings
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "c"]
+
+
+# --- resolving the group's chain ----------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_the_generic_star_chain_catches_a_group_with_no_entry(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"b": ["a"]}, {"*": ["c"]}])
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "c"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "fallbacks",
+    [[{"*": ["c"]}, {"a": ["b"]}], [{"a": ["b"]}, {"*": ["c"]}]],
+    ids=["star-first", "star-last"],
+)
+async def test_a_group_specific_chain_wins_over_the_generic_star_chain(
+    make_core: MakeCore, host: ScriptedHost, fallbacks: list[object]
+) -> None:
+    core = build(make_core, fallbacks=fallbacks)
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("group", "fallback_key"),
+    [("openai/gpt-x", "gpt-x"), ("gpt-4o", "openai/gpt-4o")],
+    ids=["provider-stripped-key", "provider-prefixed-key"],
+)
+async def test_a_chain_keyed_with_or_without_the_provider_prefix_matches_the_group(
+    make_core: MakeCore, host: ScriptedHost, group: str, fallback_key: str
+) -> None:
+    core = build(
+        make_core, model_list=[deployment(group, "p"), deployment("b", "b")], fallbacks=[{fallback_key: ["b"]}]
+    )
+    host.answer("openai/p", rate_limit())
+
+    await complete(core, model=group)
+
+    assert host.deployments_called("acompletion") == ["p", "b"]
+
+
+@pytest.mark.asyncio
+async def test_a_prefixed_key_does_not_match_a_group_whose_provider_cannot_be_inferred(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(
+        make_core, model_list=[deployment("mygroup", "p"), deployment("b", "b")], fallbacks=[{"openai/mygroup": ["b"]}]
+    )
+    host.answer("openai/p", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core, model="mygroup")
+
+    assert host.deployments_called("acompletion") == ["p"]
+
+
+@pytest.mark.asyncio
+async def test_a_dict_target_names_its_group_under_model(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": [{"model": "c"}]}])
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "c"]
+
+
+# --- walking the chain ------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("failing", "expected"),
+    [(["a"], ["a", "b"]), (["a", "b"], ["a", "b", "c"])],
+    ids=["first-target-answers", "second-target-answers"],
+)
+async def test_targets_are_tried_in_listed_order_until_one_answers(
+    make_core: MakeCore, host: ScriptedHost, failing: list[str], expected: list[str]
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b", "c"]}])
+    for group in failing:
+        host.answer(f"openai/{group}", rate_limit())
+
+    response = await complete(core)
+
+    assert host.deployments_called("acompletion") == expected
+    assert headers(response)["x-litellm-model-group"] == expected[-1]
+
+
+@pytest.mark.asyncio
+async def test_a_failed_fallback_group_follows_its_own_chain(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}, {"b": ["c"]}])
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", rate_limit())
+
+    response = await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b", "c"]
+    assert headers(response)["x-litellm-model-group"] == "c"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("fallbacks", "expected"),
+    [
+        ([{"a": ["b"]}, {"b": ["a", "c"]}, {"c": ["a", "b"]}], ["a", "b", "c"]),
+        ([{"a": ["a", "b"]}], ["a", "b"]),
+        ([{"a": ["b", "b", "c"]}], ["a", "b", "c"]),
+    ],
+    ids=["cyclic-graph", "group-lists-itself", "target-listed-twice"],
+)
+async def test_no_group_is_tried_twice_in_one_request(
+    make_core: MakeCore, host: ScriptedHost, fallbacks: list[object], expected: list[str]
+) -> None:
+    core = build(make_core, fallbacks=fallbacks)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("constructor", "process", "expected"),
+    [
+        ({"max_fallbacks": 0}, {}, ["a"]),
+        ({"max_fallbacks": 1}, {}, ["a", "b"]),
+        ({"max_fallbacks": 2}, {}, ["a", "b", "c"]),
+        ({}, {"max_fallbacks": 2}, ["a", "b", "c"]),
+        ({}, {"router_max_fallbacks": 2}, ["a", "b", "c"]),
+        ({}, {}, ["a", "b", "c", "d"]),
+        ({"max_fallbacks": 3}, {"max_fallbacks": 1}, ["a", "b", "c", "d"]),
+        ({}, {"max_fallbacks": 1, "router_max_fallbacks": 3}, ["a", "b"]),
+    ],
+    ids=[
+        "constructor-0",
+        "constructor-1",
+        "constructor-2",
+        "process-max_fallbacks-2",
+        "process-router_max_fallbacks-2",
+        "default",
+        "constructor-over-process",
+        "process-max_fallbacks-over-router_max_fallbacks",
+    ],
+)
+async def test_max_fallbacks_bounds_the_depth_of_a_chained_walk(
+    make_core: MakeCore,
+    host: ScriptedHost,
+    constructor: dict[str, object],
+    process: dict[str, object],
+    expected: list[str],
+) -> None:
+    if process:
+        host.set_dynamic(**process)
+    core = build(make_core, model_list=GROUPS_ABCD, fallbacks=CHAIN_ABCD, **constructor)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == expected
+
+
+@pytest.mark.asyncio
+async def test_max_fallbacks_applied_by_changeset_bounds_the_walk(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, model_list=GROUPS_ABCD, fallbacks=CHAIN_ABCD)
+    core.apply(ChangeSet(assign={"max_fallbacks": 1}))
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+
+
+@pytest.mark.asyncio
+async def test_a_fallback_group_with_no_deployments_is_passed_over(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["missing", "c"]}])
+    host.answer("openai/a", rate_limit())
+
+    response = await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "c"]
+    assert headers(response)["x-litellm-model-group"] == "c"
+
+
+@pytest.mark.asyncio
+async def test_a_chain_of_only_empty_groups_raises_the_primary_error(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["missing"]}])
+    host.answer("openai/a", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+@pytest.mark.asyncio
+async def test_a_fallback_group_runs_after_the_primary_group_spends_its_retries(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}], num_retries=2)
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "a", "a", "b"]
+
+
+# --- error-specific chains ---------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("error", "field"),
+    [(context_window, "context_window_fallbacks"), (content_policy, "content_policy_fallbacks")],
+    ids=["context-window", "content-policy"],
+)
+@pytest.mark.parametrize("source", ["constructor", "process-setting", "request-kwarg"])
+async def test_an_error_specific_chain_wins_over_the_general_chain(
+    make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, source: str
+) -> None:
+    chain = [{"a": ["c"]}]
+    if source == "process-setting":
+        host.set_dynamic(**{field: chain})
+    core = build(make_core, fallbacks=[{"a": ["b"]}], **({field: chain} if source == "constructor" else {}))
+    host.answer("openai/a", error())
+
+    response = await complete(core, **({field: chain} if source == "request-kwarg" else {}))
+
+    assert host.deployments_called("acompletion") == ["a", "c"]
+    assert headers(response)["x-litellm-model-group"] == "c"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("error", "field", "raised"),
+    [
+        (context_window, "context_window_fallbacks", litellm.ContextWindowExceededError),
+        (content_policy, "content_policy_fallbacks", litellm.ContentPolicyViolationError),
+    ],
+    ids=["context-window", "content-policy"],
+)
+async def test_an_error_specific_chain_without_the_group_raises_without_trying_the_general_chain(
+    make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, raised: type[Exception]
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: [{"other": ["c"]}]})
+    host.answer("openai/a", error())
+
+    with pytest.raises(raised):
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("error", [context_window, content_policy], ids=["context-window", "content-policy"])
+async def test_without_an_error_specific_chain_the_general_chain_applies(
+    make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception]
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", error())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+
+
+@pytest.mark.asyncio
+async def test_a_context_window_fallback_that_fails_otherwise_follows_the_general_chain(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, context_window_fallbacks=[{"a": ["b"]}], fallbacks=[{"b": ["c"]}])
+    host.answer("openai/a", context_window())
+    host.answer("openai/b", rate_limit())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b", "c"]
+
+
+# --- default fallbacks -------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("source", ["constructor", "process-setting"])
+async def test_default_fallbacks_catch_any_group(make_core: MakeCore, host: ScriptedHost, source: str) -> None:
+    if source == "process-setting":
+        host.set_dynamic(default_fallbacks=["c"])
+    core = build(make_core, **({"default_fallbacks": ["c"]} if source == "constructor" else {}))
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "c"]
+
+
+@pytest.mark.asyncio
+async def test_default_fallbacks_join_the_configured_chains_as_the_star_entry(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"b": ["a"]}], default_fallbacks=["c"])
+    host.answer("openai/a", rate_limit())
+
+    await complete(core)
+
+    assert core.settings().fallbacks == [{"b": ["a"]}, {"*": ["c"]}]
+    assert host.deployments_called("acompletion") == ["a", "c"]
+
+
+def test_default_fallbacks_given_as_a_string_are_refused(make_core: MakeCore) -> None:
+    with pytest.raises(TypeError, match=r"default_fallbacks must be a list of model names, got 'c'"):
+        build(make_core, default_fallbacks="c")
+
+
+# --- order tiers inside one group ----------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("failing", "expected", "model_group"),
+    [
+        (["a1"], ["a1", "a2"], "a"),
+        (["a1", "a2"], ["a1", "a2", "b"], "b"),
+    ],
+    ids=["next-tier-answers", "external-fallback-after-every-tier"],
+)
+async def test_order_tiers_are_tried_before_the_external_fallbacks(
+    make_core: MakeCore, host: ScriptedHost, failing: list[str], expected: list[str], model_group: str
+) -> None:
+    core = build(make_core, model_list=TIERED_A, fallbacks=[{"a": ["b"]}])
+    for deployment_id in failing:
+        host.answer(f"openai/{deployment_id}", rate_limit())
+
+    response = await complete(core)
+
+    assert host.deployments_called("acompletion") == expected
+    assert headers(response)["x-litellm-model-group"] == model_group
+
+
+@pytest.mark.asyncio
+async def test_order_tiers_are_walked_without_any_configured_fallbacks(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, model_list=[*TIERED_A, deployment("a", "a3", order=3)])
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a1", "a2", "a3"]
+
+
+@pytest.mark.asyncio
+async def test_a_context_window_error_skips_the_order_tiers_for_its_own_chain(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, model_list=TIERED_A, context_window_fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a1", context_window())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a1", "b"]
+
+
+# --- when no fallback runs -----------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_disable_fallbacks_on_the_request_raises_the_primary_error(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError):
+        await complete(core, disable_fallbacks=True)
+
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "error",
+    [
+        GuardrailRaisedException(guardrail_name="g", message="no"),
+        BlockedPiiEntityError(entity_type="EMAIL", guardrail_name="g"),
+    ],
+    ids=["guardrail-raised", "blocked-pii"],
+)
+async def test_a_guardrail_intervention_never_falls_back(
+    make_core: MakeCore, host: ScriptedHost, error: Exception
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", error)
+
+    with pytest.raises(type(error)) as raised:
+        await complete(core)
+
+    assert raised.value is host.raised[-1]
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+def unsupported_params() -> litellm.UnsupportedParamsError:
+    return litellm.UnsupportedParamsError(message="nope", llm_provider="openai", model="a")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("error", [bad_request, unsupported_params], ids=["bad-request", "unsupported-params"])
+async def test_a_bad_request_does_not_fall_back_when_fallback_on_unsupported_params_is_off(
+    make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception]
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}], fallback_on_unsupported_params=False)
+    host.answer("openai/a", error())
+
+    with pytest.raises(type(error())):
+        await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("error", "fallback_on_unsupported_params"),
+    [
+        (bad_request, True),
+        (unsupported_params, True),
+        (context_window, False),
+        (content_policy, False),
+        (rate_limit, False),
+        (server_error, False),
+    ],
+    ids=[
+        "bad-request-flag-on",
+        "unsupported-params-flag-on",
+        "context-window-flag-off",
+        "content-policy-flag-off",
+        "rate-limit-flag-off",
+        "server-error-flag-off",
+    ],
+)
+async def test_errors_outside_the_unsupported_params_rule_fall_back(
+    make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], fallback_on_unsupported_params: bool
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}], fallback_on_unsupported_params=fallback_on_unsupported_params)
+    host.answer("openai/a", error())
+
+    await complete(core)
+
+    assert host.deployments_called("acompletion") == ["a", "b"]
+
+
+# --- the error when every group fails --------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_when_every_fallback_fails_the_primary_groups_error_is_raised(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b", "c"]}], max_fallbacks=1)
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", timeout())
+    host.answer("openai/c", server_error())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    assert str(raised.value) == "litellm.RateLimitError: slow down"
+    assert host.deployments_called("acompletion") == ["a", "b", "c"]
+
+
+@pytest.mark.asyncio
+async def test_without_router_debug_the_error_message_is_not_extended(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    assert str(raised.value) == "litellm.RateLimitError: slow down"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("arguments", "suffix"),
+    [
+        ({}, "\n\nLiteLLM: model group 'a' failed with the error above. No fallback was attempted."),
+        (
+            {"fallbacks": [{"a": ["b", "c"]}]},
+            "\n\nLiteLLM: model group 'a' failed with the error above. "
+            "Fallback to b, c also failed: litellm.RateLimitError: slow down",
+        ),
+        (
+            {"fallbacks": [{"a": ["b"]}, {"b": ["c"]}]},
+            "\n\nLiteLLM: model group 'a' failed with the error above. "
+            "Fallback to b also failed: litellm.RateLimitError: slow down",
+        ),
+        (
+            {"fallbacks": [{"b": ["c"]}]},
+            "\n\nLiteLLM: model group 'a' failed with the error above and no fallback model group was found for it, "
+            "so the request was not retried on another model. Fallbacks are configured for: b. "
+            "Add a fallbacks entry for that model group (Router fallbacks or proxy router_settings.fallbacks) "
+            "to retry on another model.",
+        ),
+    ],
+    ids=["no-fallbacks", "fallback-list-failed", "chained-fallback-failed", "no-chain-for-group"],
+)
+async def test_router_debug_explains_the_fallback_outcome_once(
+    make_core: MakeCore, host: ScriptedHost, arguments: dict[str, object], suffix: str
+) -> None:
+    core = build(make_core, **arguments)
+    show_router_debug(host)
+    host.answer("*", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    assert str(raised.value) == f"litellm.RateLimitError: slow down{suffix}"
+
+
+@pytest.mark.asyncio
+async def test_router_debug_names_the_first_fallback_groups_error(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    show_router_debug(host)
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", server_error())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    assert str(raised.value).endswith(
+        "LiteLLM: model group 'a' failed with the error above. Fallback to b also failed: litellm.InternalServerError: boom"
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_debug_reports_a_fallback_group_with_no_deployments(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["missing"]}])
+    show_router_debug(host)
+    host.answer("openai/a", rate_limit())
+
+    with pytest.raises(litellm.RateLimitError) as raised:
+        await complete(core)
+
+    assert (
+        "LiteLLM: model group 'a' failed with the error above. Fallback to missing also failed: "
+        "litellm.BadRequestError: You passed in model=missing. There are no healthy deployments for this model"
+    ) in str(raised.value)
+
+
+@pytest.mark.asyncio
+async def test_router_debug_on_an_unmatched_context_window_chain_says_no_fallback_was_attempted(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core, context_window_fallbacks=[{"other": ["c"]}])
+    show_router_debug(host)
+    host.answer("openai/a", context_window())
+
+    with pytest.raises(litellm.ContextWindowExceededError) as raised:
+        await complete(core)
+
+    assert str(raised.value).endswith(
+        "\n\nLiteLLM: model group 'a' failed with the error above. No fallback was attempted."
+    )
+
+
+@pytest.mark.asyncio
+async def test_router_debug_on_content_policy_without_any_chain_hints_at_content_policy_fallbacks(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = build(make_core)
+    show_router_debug(host)
+    host.answer("openai/a", content_policy())
+
+    with pytest.raises(litellm.ContentPolicyViolationError) as raised:
+        await complete(core)
+
+    message = str(raised.value)
+    assert "model=a. content_policy_fallback=None. fallbacks=None." in message
+    assert message.endswith("LiteLLM: model group 'a' failed with the error above. No fallback was attempted.")
+
+
+# --- response headers ------------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("failing", "attempted", "model_group"),
+    [([], 0, "a"), (["a"], 1, "b")],
+    ids=["primary-answers", "one-fallback"],
+)
+async def test_success_headers_name_the_answering_group_and_the_fallbacks_attempted(
+    make_core: MakeCore, host: ScriptedHost, failing: list[str], attempted: int, model_group: str
+) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    for group in failing:
+        host.answer(f"openai/{group}", rate_limit())
+
+    response = await complete(core)
+
+    assert headers(response)["x-litellm-attempted-fallbacks"] == attempted
+    assert headers(response)["x-litellm-model-group"] == model_group
+
+
+@pytest.mark.asyncio
+async def test_each_attempt_carries_its_own_model_group_to_the_host(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    await core.call("acompletion", {"model": "a", "messages": MESSAGES})
+
+    assert [call.model_group for call in host.calls_to("acompletion")] == ["a", "b"]
+    assert [call.model for call in host.calls_to("acompletion")] == ["openai/a", "openai/b"]
+
+
+@pytest.mark.asyncio
+async def test_embedding_falls_back_like_completion(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, model_list=[deployment("e", "e"), deployment("f", "f")], fallbacks=[{"e": ["f"]}])
+    host.answer("openai/e", rate_limit())
+
+    response = await core.aembedding(model="e", input=["x"])  # pyright: ignore[reportAttributeAccessIssue]  # endpoint method
+
+    assert host.deployments_called("aembedding") == ["e", "f"]
+    assert headers(response)["x-litellm-attempted-fallbacks"] == 1
+
+
+def test_sync_completion_falls_back_like_async(make_core: MakeCore, host: ScriptedHost) -> None:
+    core = build(make_core, fallbacks=[{"a": ["b"]}])
+    host.answer("openai/a", rate_limit())
+
+    response = core.completion(model="a", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # endpoint method
+
+    assert host.deployments_called("completion") == ["a", "b"]
+    assert headers(response)["x-litellm-attempted-fallbacks"] == 1

tests/unit/test_router/core/test_registry_changesets.py

@@ -0,0 +1,657 @@
+"""
+The core's registry and its settings, which are static after construction: ``apply(ChangeSet)``
+(and ``aapply``) is the only way to change them, and a direct attribute write is routed into one.
+"""
+
+import copy
+from collections.abc import Callable, Mapping
+
+import pytest
+
+import litellm
+from litellm.router_api import ApplyResult, ChangeSet
+from litellm.router_core import RouterCore
+from litellm.types.router import Deployment
+from tests.unit.test_router.core.scripted_host import (
+    ScriptedHost,
+    context_window,
+    deployment,
+    server_error,
+)
+
+MakeCore = Callable[..., RouterCore]
+
+MESSAGES = [{"role": "user", "content": "hi"}]
+
+
+def _model_list() -> list[dict[str, object]]:  # mutable-ok: a fresh model_list
+    return [deployment("gpt", "a"), deployment("gpt", "b"), deployment("emb", "e")]
+
+
+def _fresh(error: Callable[[], BaseException]) -> Callable[..., BaseException]:
+    """A scripted outcome raising a new exception per call: the core stamps num_retries on the instance it saw."""
+    return lambda **_: error()
+
+
+def _ids(deployments: object) -> list[str]:  # mutable-ok: a fresh list
+    return [entry["model_info"]["id"] for entry in deployments]  # pyright: ignore[reportGeneralTypeIssues, reportIndexIssue]  # deployment dicts
+
+
+def _deployment(model_name: str, deployment_id: str, **params: object) -> Deployment:  # kwargs-ok: litellm_params
+    return Deployment(**deployment(model_name, deployment_id, **params))  # pyright: ignore[reportArgumentType]  # a model_list entry
+
+
+# -- registry lookups after construction --------------------------------------------------------
+
+
+@pytest.mark.parametrize(("deployment_id", "group"), [("b", "gpt"), ("e", "emb")])
+def test_registry_finds_each_configured_deployment_by_id(make_core: MakeCore, deployment_id: str, group: str) -> None:
+    registry = make_core(model_list=_model_list()).registry()
+
+    found = registry.deployment(deployment_id)
+
+    assert found is not None
+    assert (found.model_name, found.model_info.id, found.litellm_params.model) == (
+        group,
+        deployment_id,
+        f"openai/{deployment_id}",
+    )
+    assert registry.has_deployment(deployment_id)
+
+
+def test_registry_has_no_deployment_for_an_unknown_id(make_core: MakeCore) -> None:
+    registry = make_core(model_list=_model_list()).registry()
+
+    assert registry.deployment("missing") is None
+    assert not registry.has_deployment("missing")
+
+
+@pytest.mark.parametrize(("group", "ids"), [("gpt", ["a", "b"]), ("unknown", [])])
+def test_registry_lists_a_groups_deployments_in_configured_order(
+    make_core: MakeCore, group: str, ids: list[str]
+) -> None:
+    registry = make_core(model_list=_model_list()).registry()
+
+    assert _ids(registry.deployments(group)) == ids
+    assert registry.names(group, kind="ids") == ids
+
+
+def test_registry_names_every_group_and_deployment_id(make_core: MakeCore) -> None:
+    registry = make_core(model_list=_model_list()).registry()
+
+    assert registry.names(kind="group_set") == {"gpt", "emb"}
+    assert registry.names(kind="ids") == ["a", "b", "e"]
+    assert _ids(registry.deployments()) == ["a", "b", "e"]
+
+
+@pytest.mark.parametrize(
+    ("name", "kind", "resolved"),
+    [
+        ("fast", "group", "gpt"),
+        ("fast", "alias", "gpt"),
+        ("gpt", "alias", None),
+        ("e", "from_id", "emb"),
+        ("fast", "recognized", True),
+        ("nope", "recognized", False),
+    ],
+)
+def test_registry_resolves_names_through_groups_aliases_and_ids(
+    make_core: MakeCore, name: str, kind: str, resolved: object
+) -> None:
+    registry = make_core(model_list=_model_list(), model_group_alias={"fast": "gpt"}).registry()
+
+    assert registry.resolve(name, kind=kind) == resolved  # pyright: ignore[reportCallIssue, reportArgumentType]  # kind is one of the literals
+
+
+def test_registry_aliases_are_the_configured_model_group_alias(make_core: MakeCore) -> None:
+    registry = make_core(model_list=_model_list(), model_group_alias={"fast": "gpt"}).registry()
+
+    assert registry.aliases() == {"fast": "gpt"}
+    assert _ids(registry.deployments("fast")) == ["a", "b"]
+
+
+def test_registry_version_is_unchanged_by_reads(make_core: MakeCore) -> None:
+    core = make_core(model_list=_model_list())
+    before = core.registry().version
+
+    core.registry().deployments("gpt")
+    core.registry().names()
+    core.registry().resolve("gpt")
+    core.settings()
+
+    assert core.registry().version == before
+
+
+# -- registry().version on every write path -----------------------------------------------------
+
+
+async def _apply_upsert_new(core: RouterCore) -> None:
+    core.apply(ChangeSet(upsert=[_deployment("gpt", "c")]))
+
+
+async def _apply_upsert_changed(core: RouterCore) -> None:
+    core.apply(ChangeSet(upsert=[_deployment("gpt", "a", api_base="https://changed.example")]))
+
+
+async def _apply_delete(core: RouterCore) -> None:
+    core.apply(ChangeSet(delete=["a"]))
+
+
+async def _aapply_upsert_new(core: RouterCore) -> None:
+    await core.aapply(ChangeSet(upsert=[_deployment("gpt", "c")]))  # pyright: ignore[reportAttributeAccessIssue]  # aapply is on every core
+
+
+async def _add_deployment(core: RouterCore) -> None:
+    core.add_deployment(_deployment("gpt", "c"))  # pyright: ignore[reportAttributeAccessIssue]  # public on the core
+
+
+async def _set_model_list(core: RouterCore) -> None:
+    core.set_model_list([deployment("gpt", "c")])  # pyright: ignore[reportAttributeAccessIssue]  # public on the core
+
+
+async def _upsert_deployment(core: RouterCore) -> None:
+    with pytest.deprecated_call():
+        core.upsert_deployment(_deployment("gpt", "c"))  # pyright: ignore[reportAttributeAccessIssue]  # deprecated, still public
+
+
+async def _delete_deployment(core: RouterCore) -> None:
+    with pytest.deprecated_call():
+        core.delete_deployment("a")  # pyright: ignore[reportAttributeAccessIssue]  # deprecated, still public
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("write", "ids_after"),
+    [
+        (_apply_upsert_new, ["a", "b", "e", "c"]),
+        (_apply_upsert_changed, ["a", "b", "e"]),
+        (_apply_delete, ["b", "e"]),
+        (_aapply_upsert_new, ["a", "b", "e", "c"]),
+        (_add_deployment, ["a", "b", "e", "c"]),
+        (_set_model_list, ["c"]),
+        (_upsert_deployment, ["a", "b", "e", "c"]),
+        (_delete_deployment, ["b", "e"]),
+    ],
+    ids=lambda value: getattr(value, "__name__", None),
+)
+async def test_every_deployment_write_advances_the_registry_version(
+    make_core: MakeCore, write: Callable[[RouterCore], object], ids_after: list[str]
+) -> None:
+    core = make_core(model_list=_model_list())
+    before = core.registry().version
+
+    await write(core)  # pyright: ignore[reportGeneralTypeIssues]  # each write is a coroutine function
+
+    assert core.registry().version > before
+    assert sorted(core.registry().names(kind="ids")) == sorted(ids_after)
+
+
+@pytest.mark.asyncio
+async def test_a_changed_upsert_is_what_the_registry_returns_afterwards(make_core: MakeCore) -> None:
+    core = make_core(model_list=_model_list())
+
+    await _apply_upsert_changed(core)
+
+    changed = core.registry().deployment("a")
+    assert changed is not None
+    assert changed.litellm_params.api_base == "https://changed.example"
+
+
+@pytest.mark.parametrize(
+    "changes",
+    [
+        ChangeSet(),
+        ChangeSet(upsert=[_deployment("gpt", "a")]),
+        ChangeSet(delete=["missing"]),
+        ChangeSet(assign={"num_retries": 4, "cooldown_time": 9}),
+        ChangeSet(settings={"num_retries": 4, "allowed_fails": 2}),
+    ],
+    ids=["empty", "identical_upsert", "unknown_delete", "assign", "settings"],
+)
+def test_a_changeset_that_changes_no_deployment_keeps_the_registry_version(
+    make_core: MakeCore, changes: ChangeSet
+) -> None:
+    core = make_core(model_list=_model_list())
+    before = core.registry().version
+
+    result = core.apply(changes)
+
+    assert core.registry().version == before
+    assert result.version == before
+
+
+def test_adding_a_deployment_whose_id_exists_changes_nothing(make_core: MakeCore) -> None:
+    core = make_core(model_list=_model_list())
+    before = core.registry().version
+
+    added = core.add_deployment(_deployment("emb", "a"))  # pyright: ignore[reportAttributeAccessIssue]  # public on the core
+
+    assert added is None
+    assert core.registry().version == before
+    assert core.registry().resolve("a", kind="from_id") == "gpt"
+
+
+# -- ApplyResult --------------------------------------------------------------------------------
+
+
+def test_apply_result_carries_the_new_version_and_the_upserted_deployment(make_core: MakeCore) -> None:
+    core = make_core(model_list=_model_list())
+
+    result = core.apply(ChangeSet(upsert=[_deployment("gpt", "c")]))
+
+    assert isinstance(result, ApplyResult)
+    assert result.version == core.registry().version
+    assert [None if d is None else d.model_info.id for d in result.upserted] == ["c"]
+    assert list(result.deleted) == []
+
+
+def test_apply_result_reports_none_for_an_upsert_that_changed_nothing(make_core: MakeCore) -> None:
+    core = make_core(model_list=_model_list())
+
+    result = core.apply(ChangeSet(upsert=[_deployment("gpt", "a"), _deployment("gpt", "c")]))
+
+    assert [None if d is None else d.model_info.id for d in result.upserted] == [None, "c"]
+
+
+def test_apply_result_lists_each_delete_in_order_with_none_for_an_unknown_id(make_core: MakeCore) -> None:
+    core = make_core(model_list=_model_list())
+
+    result = core.apply(ChangeSet(delete=["e", "missing", "a"]))
+
+    assert [None if d is None else d["model_info"]["id"] for d in result.deleted] == ["e", None, "a"]  # pyright: ignore[reportIndexIssue]  # the core returns the model_list dict
+    assert core.registry().names(kind="ids") == ["b"]
+
+
+def test_apply_upserts_before_it_deletes(make_core: MakeCore) -> None:
+    core = make_core(model_list=_model_list())
+
+    result = core.apply(ChangeSet(upsert=[_deployment("gpt", "c")], delete=["c"]))
+
+    assert [d is not None for d in result.upserted] == [True]
+    assert [d is not None for d in result.deleted] == [True]
+    assert not core.registry().has_deployment("c")
+
+
+def test_settings_and_deployments_change_together_in_one_apply(make_core: MakeCore) -> None:
+    core = make_core(model_list=_model_list())
+
+    result = core.apply(ChangeSet(assign={"cooldown_time": 17}, upsert=[_deployment("emb", "f")], delete=["b"]))
+
+    assert result.version == core.registry().version
+    assert core.settings().cooldown_time == 17
+    assert core.registry().names("emb", kind="ids") == ["e", "f"]
+    assert core.registry().names("gpt", kind="ids") == ["a"]
+
+
+# -- settings changed by a changeset change later calls -----------------------------------------
+
+
+def _assign(**values: object) -> ChangeSet:  # kwargs-ok: settings by name
+    return ChangeSet(assign=values)
+
+
+def _settings(**values: object) -> ChangeSet:  # kwargs-ok: settings by name
+    return ChangeSet(settings=values)
+
+
+CHANGESET_KINDS = pytest.mark.parametrize("changeset", [_assign, _settings], ids=["assign", "settings"])
+
+
+@pytest.mark.asyncio
+@CHANGESET_KINDS
+@pytest.mark.parametrize("num_retries", [0, 3])
+async def test_num_retries_from_a_changeset_governs_the_next_failing_call(
+    make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet], num_retries: int
+) -> None:
+    core = make_core(model_list=[deployment("gpt", "a")], num_retries=2)
+    host.answer("openai/a", _fresh(server_error))
+    with pytest.raises(litellm.InternalServerError):
+        await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+    host.calls.clear()
+
+    core.apply(changeset(num_retries=num_retries))
+    with pytest.raises(litellm.InternalServerError):
+        await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+
+    assert host.deployments_called("acompletion") == ["a"] * (num_retries + 1)
+    assert core.settings().as_dict()["num_retries"] == num_retries
+
+
+@pytest.mark.asyncio
+@CHANGESET_KINDS
+async def test_fallbacks_from_a_changeset_are_used_by_the_next_call(
+    make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet]
+) -> None:
+    core = make_core(model_list=[deployment("gpt", "a"), deployment("backup", "f")], num_retries=0)
+    host.answer("openai/a", _fresh(server_error))
+    with pytest.raises(litellm.InternalServerError):
+        await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+    host.calls.clear()
+
+    core.apply(changeset(fallbacks=[{"gpt": ["backup"]}]))
+    response = await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+
+    assert host.deployments_called("acompletion") == ["a", "f"]
+    assert response._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1
+    assert core.settings().fallbacks == [{"gpt": ["backup"]}]
+
+
+@pytest.mark.asyncio
+@CHANGESET_KINDS
+async def test_context_window_fallbacks_from_a_changeset_are_used_by_the_next_call(
+    make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet]
+) -> None:
+    core = make_core(model_list=[deployment("gpt", "a"), deployment("long", "l")], num_retries=0)
+    host.answer("openai/a", _fresh(context_window))
+    with pytest.raises(litellm.ContextWindowExceededError):
+        await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+    host.calls.clear()
+
+    core.apply(changeset(context_window_fallbacks=[{"gpt": ["long"]}]))
+    response = await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+
+    assert host.deployments_called("acompletion") == ["a", "l"]
+    assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "long"
+    assert core.settings().context_window_fallbacks == [{"gpt": ["long"]}]
+
+
+@pytest.mark.asyncio
+@CHANGESET_KINDS
+async def test_timeout_from_a_changeset_is_passed_on_the_next_call(
+    make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet]
+) -> None:
+    core = make_core(model_list=[deployment("gpt", "a")], timeout=30)
+    await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+
+    core.apply(changeset(timeout=12))
+    await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+
+    assert [call.kwargs["timeout"] for call in host.calls_to("acompletion")] == [30, 12]
+    assert core.settings().as_dict()["timeout"] == 12
+
+
+@pytest.mark.asyncio
+@CHANGESET_KINDS
+async def test_model_group_alias_from_a_changeset_routes_the_next_call(
+    make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet]
+) -> None:
+    core = make_core(model_list=[deployment("gpt", "a"), deployment("emb", "e")], model_group_alias={"fast": "gpt"})
+
+    core.apply(changeset(model_group_alias={"fast": "emb"}))
+    await core.acompletion(model="fast", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+
+    assert host.deployments_called("acompletion") == ["e"]
+    assert core.registry().aliases() == {"fast": "emb"}
+    assert core.registry().resolve("fast") == "emb"
+
+
+@pytest.mark.asyncio
+@CHANGESET_KINDS
+async def test_allowed_fails_and_cooldown_time_from_a_changeset_govern_the_next_cooldown(
+    make_core: MakeCore, changeset: Callable[..., ChangeSet]
+) -> None:
+    core = make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")])
+    assert core.health().record_failure("a", server_error(), 500) is False
+
+    core.apply(changeset(allowed_fails=0, cooldown_time=42))
+
+    assert core.health().record_failure("b", server_error(), 500) is True
+    assert [
+        (i, entry["cooldown_time"]) for i, entry in core.health().cooldowns.get_active_cooldowns(["a", "b"], None)
+    ] == [("b", 42)]
+    assert (core.settings().allowed_fails, core.settings().cooldown_time) == (0, 42)
+
+
+@pytest.mark.asyncio
+async def test_a_deployment_cooled_down_under_applied_allowed_fails_is_not_picked_again(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")], num_retries=0)
+    core.apply(ChangeSet(assign={"allowed_fails": 0, "cooldown_time": 60}))
+    host.answer("*", _fresh(server_error))
+
+    for _ in range(2):
+        with pytest.raises(litellm.InternalServerError):
+            await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+
+    assert sorted(host.deployments_called("acompletion")) == ["a", "b"]
+
+
+@pytest.mark.asyncio
+@CHANGESET_KINDS
+async def test_tag_filtering_switched_on_by_a_changeset_restricts_the_next_pick(
+    make_core: MakeCore, host: ScriptedHost, changeset: Callable[..., ChangeSet]
+) -> None:
+    core = make_core(model_list=[deployment("gpt", "a", tags=["paid"]), deployment("gpt", "b", tags=["free"])])
+    tagged = {"metadata": {"tags": ["paid"]}}
+    assert sorted(_ids(await core.select("gpt", dict(tagged), kind="healthy"))) == ["a", "b"]  # pyright: ignore[reportCallIssue]  # select takes kind
+
+    core.apply(changeset(enable_tag_filtering=True))
+
+    assert _ids(await core.select("gpt", dict(tagged), kind="healthy")) == ["a"]  # pyright: ignore[reportCallIssue]  # select takes kind
+    for _ in range(3):
+        await core.acompletion(model="gpt", messages=MESSAGES, metadata={"tags": ["paid"]})  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+    assert host.deployments_called("acompletion") == ["a", "a", "a"]
+    assert core.settings().enable_tag_filtering is True
+
+
+@pytest.mark.asyncio
+async def test_routing_strategy_from_settings_is_reported_and_still_routes(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = make_core(model_list=[deployment("gpt", "a")])
+    strategy = "least-busy"
+
+    core.apply(ChangeSet(settings={"routing_strategy": strategy}))
+    await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+
+    assert core.settings().routing_strategy == strategy
+    assert core.settings().as_dict()["routing_strategy"] == strategy
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+@pytest.mark.asyncio
+async def test_default_litellm_params_from_a_changeset_are_sent_on_the_next_call(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = make_core(model_list=[deployment("gpt", "a")], default_litellm_params={"temperature": 0.2})
+
+    core.apply(ChangeSet(assign={"default_litellm_params": {"temperature": 0.7}}))
+    await core.acompletion(model="gpt", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+
+    assert [call.kwargs.get("temperature") for call in host.calls_to("acompletion")] == [0.7]
+
+
+@pytest.mark.parametrize(
+    ("name", "value"),
+    [
+        ("cooldown_time", 7.5),
+        ("allowed_fails", 9),
+        ("disable_cooldowns", True),
+        ("content_policy_fallbacks", [{"gpt": ["emb"]}]),
+    ],
+)
+def test_settings_reflect_an_assigned_field(make_core: MakeCore, name: str, value: object) -> None:
+    core = make_core(model_list=_model_list())
+
+    core.apply(ChangeSet(assign={name: value}))
+
+    assert getattr(core.settings(), name) == value
+    assert core.settings().is_set(name)
+
+
+# -- what apply refuses -------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("name", ["bogus", "model_list", "_core_num_retries"])
+def test_assigning_an_unknown_setting_raises_and_applies_nothing(make_core: MakeCore, name: str) -> None:
+    core = make_core(model_list=_model_list(), num_retries=1)
+    settings_before = core.settings()
+
+    with pytest.raises(ValueError, match=name):
+        core.apply(ChangeSet(assign={"num_retries": 5, "cooldown_time": 3, name: 1}, upsert=[_deployment("gpt", "c")]))
+
+    assert core.settings().as_dict()["num_retries"] == 1
+    assert core.settings() is settings_before
+    assert not core.registry().has_deployment("c")
+
+
+def test_apply_refuses_refresh_model_info_and_applies_nothing(make_core: MakeCore) -> None:
+    core = make_core(model_list=_model_list(), num_retries=1)
+    version_before = core.registry().version
+
+    with pytest.raises(TypeError, match="aapply"):
+        core.apply(ChangeSet(refresh_model_info=True, assign={"num_retries": 5}, delete=["a"]))
+
+    assert core.settings().as_dict()["num_retries"] == 1
+    assert core.registry().version == version_before
+    assert core.registry().has_deployment("a")
+
+
+# -- the settings() snapshot --------------------------------------------------------------------
+
+
+def test_settings_snapshot_is_reused_between_reads(make_core: MakeCore) -> None:
+    core = make_core(model_list=_model_list())
+
+    assert core.settings() is core.settings()
+
+
+@CHANGESET_KINDS
+def test_applying_a_setting_replaces_the_snapshot_and_leaves_the_old_one_as_it_was(
+    make_core: MakeCore, changeset: Callable[..., ChangeSet]
+) -> None:
+    core = make_core(model_list=_model_list(), cooldown_time=5)
+    before = core.settings()
+
+    core.apply(changeset(cooldown_time=11))
+
+    after = core.settings()
+    assert after is not before
+    assert (before.cooldown_time, after.cooldown_time) == (5, 11)
+    assert core.settings() is after
+
+
+@pytest.mark.parametrize(
+    "changes",
+    [ChangeSet(upsert=[_deployment("gpt", "c")]), ChangeSet(delete=["a"])],
+    ids=["upsert", "delete"],
+)
+def test_a_changeset_without_settings_keeps_the_snapshot(make_core: MakeCore, changes: ChangeSet) -> None:
+    core = make_core(model_list=_model_list())
+    before = core.settings()
+
+    core.apply(changes)
+
+    assert core.settings() is before
+
+
+# -- a direct attribute write is a changeset ----------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("name", "value"),
+    [
+        ("num_retries", 4),
+        ("timeout", 9.0),
+        ("retry_after", 3),
+        ("allowed_fails", 1),
+        ("cooldown_time", 11.0),
+        ("fallbacks", [{"gpt": ["emb"]}]),
+        ("context_window_fallbacks", [{"gpt": ["emb"]}]),
+        ("enable_tag_filtering", True),
+        ("model_group_alias", {"fast": "emb"}),
+    ],
+)
+def test_writing_a_setting_attribute_is_the_same_as_assigning_it_in_a_changeset(
+    make_core: MakeCore, name: str, value: object
+) -> None:
+    written = make_core(model_list=_model_list())
+    applied = make_core(model_list=_model_list())
+    snapshot_before = written.settings()
+    version_before = written.registry().version
+
+    setattr(written, name, copy.deepcopy(value))
+    applied.apply(ChangeSet(assign={name: copy.deepcopy(value)}))
+
+    assert written.settings().as_dict() == applied.settings().as_dict()
+    assert written.registry().aliases() == applied.registry().aliases()
+    if written.settings().is_set(name):
+        assert getattr(written.settings(), name) == getattr(applied.settings(), name) == value
+        assert written.settings() is not snapshot_before
+    assert written.registry().version == version_before
+
+
+# -- constructor arguments ----------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("name", "value"),
+    [
+        ("model_list", _model_list()),
+        ("fallbacks", [{"gpt": ["emb"]}]),
+        ("model_group_alias", {"fast": "gpt"}),
+        ("model_group_retry_policy", {"gpt": {"RateLimitErrorRetries": 1}}),
+        ("allowed_fails_policy", {"RateLimitErrorAllowedFails": 2}),
+    ],
+)
+def test_construction_leaves_the_arguments_it_was_given_unchanged(
+    make_core: MakeCore, name: str, value: object
+) -> None:
+    given = copy.deepcopy(value)
+    arguments: dict[str, object] = {"model_list": _model_list(), name: given}
+
+    make_core(**arguments)
+
+    assert given == value
+
+
+@pytest.mark.asyncio
+async def test_mutating_the_model_list_after_construction_changes_nothing(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    model_list = _model_list()
+    core = make_core(model_list=model_list)
+    version_before = core.registry().version
+
+    model_list.append(deployment("gpt", "late"))
+    model_list[2]["model_name"] = "gpt"
+    model_list[0]["litellm_params"]["model"] = "openai/elsewhere"  # pyright: ignore[reportIndexIssue]  # a model_list entry
+    model_list[0]["model_info"]["id"] = "renamed"  # pyright: ignore[reportIndexIssue]  # a model_list entry
+    await core.aembedding(model="emb", input=["x"])  # pyright: ignore[reportAttributeAccessIssue]  # an endpoint method
+
+    assert core.registry().names(kind="ids") == ["a", "b", "e"]
+    assert core.registry().names(kind="group_set") == {"gpt", "emb"}
+    assert core.registry().version == version_before
+    assert [(call.model, call.deployment_id) for call in host.calls_to("aembedding")] == [("openai/e", "e")]
+
+
+# -- retire --------------------------------------------------------------------------------------
+
+
+def _registered(host: ScriptedHost) -> Mapping[str, int]:
+    return {name: len(callbacks) for name, callbacks in host.lists.items() if callbacks}
+
+
+def test_retire_releases_every_callback_the_core_registered_on_its_host(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core = make_core(model_list=_model_list())
+    assert _registered(host) != {}
+
+    core.retire()
+
+    assert _registered(host) == {}
+
+
+def test_retire_leaves_another_cores_callbacks_registered(make_core: MakeCore, host: ScriptedHost) -> None:
+    kept = make_core(model_list=_model_list())
+    retired = make_core(model_list=[deployment("other", "o")])
+    registered_by_both = _registered(host)
+
+    retired.retire()
+
+    assert _registered(host) != {}
+    assert sum(_registered(host).values()) < sum(registered_by_both.values())
+    kept.retire()
+    assert _registered(host) == {}

tests/unit/test_router/core/test_retries_cooldowns.py

@@ -0,0 +1,832 @@
+"""
+Retries and cooldowns of a router core, driven through its public surface on a ``ScriptedHost``.
+
+Backoff values carry litellm's random jitter (up to ``JITTER`` seconds), so sleeps are asserted
+inside ``[base, base + JITTER)`` rather than exactly.
+"""
+
+from collections.abc import Callable, Mapping
+from typing import Final
+
+import httpx
+import pytest
+
+import litellm
+from litellm.constants import DEFAULT_COOLDOWN_TIME_SECONDS, INITIAL_RETRY_DELAY, JITTER
+from litellm.router_api import ChangeSet
+from litellm.router_core import RouterCore
+from litellm.types.router import AllowedFailsPolicy, RetryPolicy
+from tests.unit.test_router.core.scripted_host import (
+    Call,
+    ScriptedHost,
+    auth_error,
+    bad_request,
+    content_policy,
+    context_window,
+    deployment,
+    rate_limit,
+    server_error,
+    timeout,
+)
+
+MESSAGES: Final = [{"role": "user", "content": "hi"}]
+
+MakeCore = Callable[..., RouterCore]
+ErrorBuilder = Callable[[], Exception]
+
+
+class LitellmParamsLoggingHost(ScriptedHost):
+    """``ScriptedHost`` whose failure logging carries the call's ``cooldown_time`` in ``litellm_params``, as
+    litellm's own logging does (``get_litellm_params``)."""
+
+    def _logging_kwargs(
+        self, call: Call, error: BaseException | None, response: object
+    ) -> dict[str, object]:  # mutable-ok: litellm's logging kwargs
+        logged: Final = super()._logging_kwargs(call, error, response)
+        litellm_params: Final = logged["litellm_params"]
+        if isinstance(litellm_params, dict) and "cooldown_time" in call.kwargs:
+            litellm_params["cooldown_time"] = call.kwargs["cooldown_time"]
+        return logged
+
+
+@pytest.fixture
+def host() -> ScriptedHost:
+    return LitellmParamsLoggingHost()
+
+
+def fresh(builder: ErrorBuilder) -> Callable[..., Exception]:
+    """A scripted outcome that raises a new error per call, so stamps from one call never leak into the next."""
+    return lambda **_: builder()
+
+
+def rate_limited(retry_after: str | None = None) -> litellm.RateLimitError:
+    headers: Final = {"retry-after": retry_after} if retry_after is not None else {}
+    return litellm.RateLimitError(
+        message="slow down", llm_provider="openai", model="gpt-4o", response=httpx.Response(429, headers=headers)
+    )
+
+
+async def attempt(core: RouterCore, model: str = "g", **kwargs: object) -> object:  # kwargs-ok: request kwargs
+    """The response, or the error the core raised."""
+    try:
+        return await core.acompletion(model=model, messages=MESSAGES, **kwargs)  # pyright: ignore[reportAttributeAccessIssue]  # every core serves acompletion
+    except Exception as error:
+        return error
+
+
+def cooled_down(core: RouterCore, ids: list[str]) -> dict[str, Mapping[str, object]]:  # mutable-ok: a fresh dict
+    return dict(core.health().cooldowns.get_active_cooldowns(ids, None))
+
+
+def headers_of(response: object) -> Mapping[str, object]:
+    hidden: Final = getattr(response, "_hidden_params", None) or {}
+    return hidden.get("additional_headers") or {}
+
+
+def two_deployments(
+    **params: object,
+) -> list[dict[str, object]]:  # kwargs-ok: litellm_params  # mutable-ok: a model_list
+    return [deployment("g", "a", **params), deployment("g", "b", **params)]
+
+
+# --- how many retries ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("constructor", "request_value", "process", "expected_retries"),
+    [
+        (None, None, None, 2),
+        (3, None, None, 3),
+        (None, None, 4, 4),
+        (1, None, 4, 1),
+        (3, 1, None, 1),
+        (3, 0, None, 0),
+        (None, 2, 5, 2),
+        (0, 2, None, 2),
+    ],
+)
+@pytest.mark.asyncio
+async def test_num_retries_comes_from_request_then_constructor_then_process_then_default(
+    make_core: MakeCore,
+    host: ScriptedHost,
+    constructor: int | None,
+    request_value: int | None,
+    process: int | None,
+    expected_retries: int,
+) -> None:
+    if process is not None:
+        host.set_dynamic(num_retries=process)
+    arguments: Final = {"num_retries": constructor} if constructor is not None else {}
+    core: Final = make_core(model_list=[deployment("g", "a")], **arguments)
+    host.answer("openai/a", fresh(server_error))
+    request: Final = {"num_retries": request_value} if request_value is not None else {}
+
+    error: Final = await attempt(core, **request)
+
+    assert isinstance(error, litellm.InternalServerError)
+    assert host.deployments_called("acompletion") == ["a"] * (expected_retries + 1)
+
+
+@pytest.mark.asyncio
+async def test_process_num_retries_is_read_once_when_the_core_is_built(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    host.set_dynamic(num_retries=5)
+    host.answer("openai/a", fresh(server_error))
+
+    await attempt(core)
+
+    assert len(host.calls_to("acompletion")) == 3
+
+
+@pytest.mark.asyncio
+async def test_num_retries_applied_through_settings_changes_later_requests(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3)
+    core.apply(ChangeSet(settings={"num_retries": 1}))
+    host.answer("openai/a", fresh(server_error))
+
+    await attempt(core)
+
+    assert len(host.calls_to("acompletion")) == 2
+
+
+# --- which errors retry -------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("builder", "expected_calls"),
+    [
+        (bad_request, 1),
+        (auth_error, 1),
+        (context_window, 1),
+        (content_policy, 1),
+        (rate_limit, 3),
+        (server_error, 3),
+        (timeout, 3),
+    ],
+)
+@pytest.mark.asyncio
+async def test_lone_deployment_retries_only_retryable_errors(
+    make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder, expected_calls: int
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2)
+    host.answer("openai/a", fresh(builder))
+
+    error: Final = await attempt(core)
+
+    assert type(error) is type(builder())
+    assert len(host.calls_to("acompletion")) == expected_calls
+
+
+@pytest.mark.parametrize(
+    ("builder", "expected_calls"),
+    [(bad_request, 1), (context_window, 1), (content_policy, 1), (server_error, 3), (timeout, 3)],
+)
+@pytest.mark.asyncio
+async def test_two_deployment_group_retries_only_retryable_errors(
+    make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder, expected_calls: int
+) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=2)
+    host.answer("*", fresh(builder))
+
+    error: Final = await attempt(core)
+
+    assert type(error) is type(builder())
+    assert len(host.calls_to("acompletion")) == expected_calls
+
+
+@pytest.mark.asyncio
+async def test_auth_error_retries_on_the_other_deployment_of_the_group(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=2)
+    host.answer("*", fresh(auth_error))
+
+    error: Final = await attempt(core)
+
+    assert isinstance(error, litellm.AuthenticationError)
+    assert sorted(host.deployments_called("acompletion")) == ["a", "b"]
+
+
+@pytest.mark.asyncio
+async def test_rate_limits_stop_retrying_once_every_deployment_is_cooling_down(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=5)
+    host.answer("*", fresh(rate_limit))
+
+    error: Final = await attempt(core)
+
+    assert isinstance(error, litellm.RateLimitError)
+    assert sorted(host.deployments_called("acompletion")) == ["a", "b"]
+    assert set(cooled_down(core, ["a", "b"])) == {"a", "b"}
+
+
+@pytest.mark.parametrize("builder", [bad_request, auth_error, context_window, content_policy])
+@pytest.mark.asyncio
+async def test_an_error_raised_without_retrying_carries_no_retry_counts(
+    make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2)
+    host.answer("openai/a", fresh(builder))
+
+    error: Final = await attempt(core)
+
+    assert getattr(error, "num_retries", None) is None
+    assert getattr(error, "max_retries", None) is None
+
+
+@pytest.mark.parametrize("builder", [rate_limit, server_error, timeout])
+@pytest.mark.parametrize("num_retries", [1, 3])
+@pytest.mark.asyncio
+async def test_exhausted_retries_stamp_num_retries_and_max_retries_on_the_final_error(
+    make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder, num_retries: int
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=num_retries)
+    host.answer("openai/a", fresh(builder))
+
+    error: Final = await attempt(core)
+
+    assert getattr(error, "num_retries", None) == num_retries
+    assert getattr(error, "max_retries", None) == num_retries
+    assert f"LiteLLM Retried: {num_retries} times" in str(error)
+
+
+@pytest.mark.asyncio
+async def test_the_final_error_is_the_latest_one_not_the_first(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2)
+    first: Final = server_error()
+    last: Final = timeout()
+    host.answer("openai/a", first, fresh(server_error), last)
+
+    error: Final = await attempt(core)
+
+    assert error is host.raised[-1]
+    assert type(error) is type(last)
+
+
+# --- retry policies -----------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("field", "builder", "retries"),
+    [
+        ("BadRequestErrorRetries", bad_request, 2),
+        ("BadRequestErrorRetries", context_window, 1),
+        ("ContentPolicyViolationErrorRetries", content_policy, 1),
+        ("RateLimitErrorRetries", rate_limit, 0),
+        ("TimeoutErrorRetries", timeout, 1),
+        ("InternalServerErrorRetries", server_error, 4),
+        ("DefaultRetries", server_error, 1),
+    ],
+)
+@pytest.mark.asyncio
+async def test_retry_policy_sets_retries_per_error_type(
+    make_core: MakeCore, host: ScriptedHost, field: str, builder: ErrorBuilder, retries: int
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")], num_retries=3, retry_policy=RetryPolicy(**{field: retries})
+    )
+    host.answer("openai/a", fresh(builder))
+
+    error: Final = await attempt(core)
+
+    assert type(error) is type(builder())
+    assert len(host.calls_to("acompletion")) == retries + 1
+
+
+@pytest.mark.asyncio
+async def test_retry_policy_given_as_a_dict_works_like_a_retry_policy(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")], num_retries=3, retry_policy={"BadRequestErrorRetries": 1}
+    )
+    host.answer("openai/a", fresh(bad_request))
+
+    await attempt(core)
+
+    assert len(host.calls_to("acompletion")) == 2
+
+
+@pytest.mark.asyncio
+async def test_retry_policy_for_one_error_type_leaves_others_to_num_retries(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")], num_retries=1, retry_policy=RetryPolicy(RateLimitErrorRetries=4)
+    )
+    host.answer("openai/a", fresh(timeout))
+
+    await attempt(core)
+
+    assert len(host.calls_to("acompletion")) == 2
+
+
+@pytest.mark.asyncio
+async def test_model_group_retry_policy_applies_only_to_its_group(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("h", "c")],
+        num_retries=2,
+        model_group_retry_policy={"g": RetryPolicy(InternalServerErrorRetries=0)},
+    )
+    host.answer("*", fresh(server_error))
+
+    await attempt(core, model="g")
+    await attempt(core, model="h")
+
+    assert host.deployments_called("acompletion") == ["a", "c", "c", "c"]
+
+
+@pytest.mark.asyncio
+async def test_model_group_retry_policy_wins_over_retry_policy(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")],
+        retry_policy=RetryPolicy(InternalServerErrorRetries=4),
+        model_group_retry_policy={"g": RetryPolicy(InternalServerErrorRetries=1)},
+    )
+    host.answer("openai/a", fresh(server_error))
+
+    await attempt(core)
+
+    assert len(host.calls_to("acompletion")) == 2
+
+
+@pytest.mark.parametrize(("request_value", "expected_calls"), [(4, 2), (0, 1)])
+@pytest.mark.asyncio
+async def test_retry_policy_wins_over_request_num_retries_unless_the_request_opts_out(
+    make_core: MakeCore, host: ScriptedHost, request_value: int, expected_calls: int
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], retry_policy=RetryPolicy(InternalServerErrorRetries=1))
+    host.answer("openai/a", fresh(server_error))
+
+    await attempt(core, num_retries=request_value)
+
+    assert len(host.calls_to("acompletion")) == expected_calls
+
+
+# --- per-deployment num_retries -----------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("deployment_value", "request_kwargs", "expected_calls"),
+    [
+        (1, {}, 2),
+        ("1", {}, 2),
+        (0, {}, 1),
+        (4, {}, 5),
+        (4, {"num_retries": 0}, 1),
+        (0, {"num_retries": 2}, 3),
+    ],
+)
+@pytest.mark.asyncio
+async def test_deployment_num_retries_beats_the_router_but_not_the_request(
+    make_core: MakeCore,
+    host: ScriptedHost,
+    deployment_value: object,
+    request_kwargs: dict[str, object],
+    expected_calls: int,
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a", num_retries=deployment_value)], num_retries=3)
+    host.answer("openai/a", fresh(server_error))
+
+    error: Final = await attempt(core, **request_kwargs)
+
+    assert isinstance(error, litellm.InternalServerError)
+    assert len(host.calls_to("acompletion")) == expected_calls
+
+
+# --- backoff ------------------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_lone_deployment_backs_off_exponentially(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3)
+    host.answer("openai/a", fresh(server_error))
+
+    await attempt(core)
+
+    bases: Final = [INITIAL_RETRY_DELAY * 2**n for n in range(3)]
+    assert len(host.sleeps) >= 3
+    assert all(base <= slept < base + JITTER for base, slept in zip(bases, host.sleeps))
+
+
+@pytest.mark.asyncio
+async def test_retry_after_setting_is_the_minimum_backoff(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2, retry_after=3)
+    host.answer("openai/a", fresh(server_error))
+
+    await attempt(core)
+
+    assert len(host.sleeps) >= 2
+    assert all(3 <= slept < 3 + JITTER for slept in host.sleeps[:2])
+
+
+@pytest.mark.asyncio
+async def test_lone_deployment_waits_the_retry_after_header(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2)
+    host.answer("openai/a", lambda **_: rate_limited(retry_after="7"))
+
+    await attempt(core)
+
+    assert len(host.sleeps) >= 2
+    assert all(7 <= slept < 7 + JITTER for slept in host.sleeps[:2])
+
+
+@pytest.mark.asyncio
+async def test_lone_deployment_ignores_a_retry_after_header_over_a_minute(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2)
+    host.answer("openai/a", lambda **_: rate_limited(retry_after="120"))
+
+    await attempt(core)
+
+    assert INITIAL_RETRY_DELAY <= host.sleeps[0] < INITIAL_RETRY_DELAY + JITTER
+
+
+@pytest.mark.parametrize("failure", [fresh(server_error), fresh(timeout), lambda **_: rate_limited(retry_after="7")])
+@pytest.mark.asyncio
+async def test_a_group_with_siblings_retries_immediately(
+    make_core: MakeCore, host: ScriptedHost, failure: Callable[..., Exception]
+) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=2)
+    host.answer("*", failure, None)
+
+    response: Final = await attempt(core)
+
+    assert not isinstance(response, Exception)
+    assert host.sleeps == [0]
+
+
+@pytest.mark.asyncio
+async def test_no_sleep_when_the_error_is_not_retried(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3)
+    host.answer("openai/a", fresh(bad_request))
+
+    await attempt(core)
+
+    assert host.sleeps == []
+
+
+# --- retry headers on success -------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("failures_first", [1, 2, 3])
+@pytest.mark.asyncio
+async def test_success_after_retries_reports_attempted_and_max_retries(
+    make_core: MakeCore, host: ScriptedHost, failures_first: int
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3)
+    host.answer("openai/a", *[fresh(server_error)] * failures_first, None)
+
+    response: Final = await attempt(core)
+
+    headers: Final = headers_of(response)
+    assert headers["x-litellm-attempted-retries"] == failures_first
+    assert headers["x-litellm-max-retries"] == 3
+    assert headers["x-litellm-model-group"] == "g"
+
+
+@pytest.mark.asyncio
+async def test_first_try_success_reports_zero_retries_and_no_max(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=3)
+
+    response: Final = await attempt(core)
+
+    headers: Final = headers_of(response)
+    assert headers["x-litellm-attempted-retries"] == 0
+    assert "x-litellm-max-retries" not in headers
+
+
+@pytest.mark.asyncio
+async def test_success_headers_report_the_retry_policy_budget(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")], num_retries=1, retry_policy=RetryPolicy(InternalServerErrorRetries=4)
+    )
+    host.answer("openai/a", fresh(server_error), fresh(server_error), None)
+
+    response: Final = await attempt(core)
+
+    assert headers_of(response)["x-litellm-attempted-retries"] == 2
+    assert headers_of(response)["x-litellm-max-retries"] == 4
+
+
+# --- allowed fails and cooldowns ----------------------------------------------------------------
+
+
+@pytest.mark.parametrize("allowed_fails", [0, 1, 2])
+@pytest.mark.asyncio
+async def test_a_deployment_cools_down_once_its_failures_pass_allowed_fails(
+    make_core: MakeCore, host: ScriptedHost, allowed_fails: int
+) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=allowed_fails)
+    host.answer("*", fresh(server_error))
+
+    for _ in range(2 * allowed_fails + 2):
+        await attempt(core)
+
+    called: Final = host.deployments_called("acompletion")
+    cooled: Final = cooled_down(core, ["a", "b"])
+    assert cooled
+    assert {d for d in ("a", "b") if called.count(d) > allowed_fails} == set(cooled)
+
+
+@pytest.mark.asyncio
+async def test_process_allowed_fails_is_the_default(make_core: MakeCore, host: ScriptedHost) -> None:
+    host.set_dynamic(allowed_fails=0)
+    core: Final = make_core(model_list=two_deployments(), num_retries=0)
+    host.answer("*", fresh(server_error))
+
+    await attempt(core)
+
+    assert core.settings().allowed_fails == 0
+    assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion")
+
+
+@pytest.mark.asyncio
+async def test_a_cooled_deployment_is_skipped_by_later_picks(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0)
+    host.answer("*", fresh(server_error))
+
+    await attempt(core)
+    first: Final = host.deployments_called("acompletion")[0]
+    healthy: Final = await core.select("g", {}, kind="healthy")  # pyright: ignore[reportAttributeAccessIssue]  # every core serves select
+    await attempt(core)
+
+    assert [d["model_info"]["id"] for d in healthy] == [{"a": "b", "b": "a"}[first]]
+    assert host.deployments_called("acompletion")[1] != first
+
+
+@pytest.mark.asyncio
+async def test_a_group_with_every_deployment_cooling_down_raises_without_calling(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0)
+    host.answer("*", fresh(server_error))
+    await attempt(core)
+    await attempt(core)
+
+    error: Final = await attempt(core)
+
+    assert isinstance(error, litellm.RouterRateLimitError)
+    assert len(host.calls_to("acompletion")) == 2
+
+
+@pytest.mark.asyncio
+async def test_rate_limit_cools_down_a_deployment_with_siblings_at_once(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=0)
+    host.answer("*", fresh(rate_limit))
+
+    await attempt(core)
+
+    assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion")
+
+
+@pytest.mark.parametrize(
+    "policy",
+    [AllowedFailsPolicy(RateLimitErrorAllowedFails=2), {"RateLimitErrorAllowedFails": 2}],
+    ids=["AllowedFailsPolicy", "dict"],
+)
+@pytest.mark.asyncio
+async def test_allowed_fails_policy_lets_rate_limits_through_until_its_count(
+    make_core: MakeCore, host: ScriptedHost, policy: object
+) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails_policy=policy)
+    host.answer("*", fresh(rate_limit))
+
+    await attempt(core)
+
+    assert cooled_down(core, ["a", "b"]) == {}
+
+
+@pytest.mark.asyncio
+async def test_allowed_fails_policy_of_zero_cools_down_on_the_first_failure_of_that_type(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(
+        model_list=two_deployments(),
+        num_retries=0,
+        allowed_fails=5,
+        allowed_fails_policy=AllowedFailsPolicy(TimeoutErrorAllowedFails=0),
+    )
+    host.answer("*", fresh(server_error), fresh(timeout))
+
+    await attempt(core)
+    after_server_error: Final = cooled_down(core, ["a", "b"])
+    await attempt(core)
+
+    assert after_server_error == {}
+    assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion")[1:]
+
+
+@pytest.mark.asyncio
+async def test_deployment_allowed_fails_policy_in_model_info_cools_down_on_a_bad_request(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(
+        model_list=two_deployments(model_info={"allowed_fails_policy": {"BadRequestErrorAllowedFails": 0}}),
+        num_retries=0,
+    )
+    host.answer("*", fresh(bad_request))
+
+    await attempt(core)
+
+    assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion")
+
+
+@pytest.mark.asyncio
+async def test_bad_request_does_not_cool_down_without_a_deployment_policy(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0)
+    host.answer("*", fresh(bad_request))
+
+    await attempt(core)
+
+    assert cooled_down(core, ["a", "b"]) == {}
+
+
+# --- cooldown time ------------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("arguments", "params", "failure", "expected_seconds"),
+    [
+        ({}, {}, fresh(rate_limit), DEFAULT_COOLDOWN_TIME_SECONDS),
+        ({"cooldown_time": 42}, {}, fresh(rate_limit), 42),
+        ({}, {"cooldown_time": 30}, fresh(rate_limit), 30),
+        ({}, {"model_info": {"cooldown_time": 30}}, fresh(rate_limit), 30),
+        ({}, {}, lambda **_: rate_limited(retry_after="9"), 9),
+        ({"cooldown_time": 42}, {}, lambda **_: rate_limited(retry_after="9"), 9),
+        ({}, {"cooldown_time": 30}, lambda **_: rate_limited(retry_after="9"), 30),
+    ],
+    ids=[
+        "default",
+        "constructor",
+        "litellm_params",
+        "model_info",
+        "retry-after",
+        "header-over-router",
+        "deployment-over-header",
+    ],
+)
+@pytest.mark.asyncio
+async def test_cooldown_time_comes_from_deployment_then_retry_after_then_router(
+    make_core: MakeCore,
+    host: ScriptedHost,
+    arguments: dict[str, object],
+    params: dict[str, object],
+    failure: Callable[..., Exception],
+    expected_seconds: float,
+) -> None:
+    core: Final = make_core(model_list=two_deployments(**params), num_retries=0, **arguments)
+    host.answer("*", failure)
+
+    await attempt(core)
+
+    (entry,) = cooled_down(core, ["a", "b"]).values()
+    assert entry["cooldown_time"] == expected_seconds
+
+
+@pytest.mark.parametrize("change", [{"settings": {"cooldown_time": 20}}, {"assign": {"cooldown_time": 20}}])
+@pytest.mark.asyncio
+async def test_cooldown_time_applied_at_runtime_is_used_for_later_cooldowns(
+    make_core: MakeCore, host: ScriptedHost, change: dict[str, Mapping[str, object]]
+) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=0)
+    core.apply(ChangeSet(**change))  # pyright: ignore[reportArgumentType]  # ChangeSet's mapping fields
+    host.answer("*", fresh(rate_limit))
+
+    await attempt(core)
+
+    assert core.settings().cooldown_time == 20
+    (entry,) = cooled_down(core, ["a", "b"]).values()
+    assert entry["cooldown_time"] == 20
+
+
+@pytest.mark.asyncio
+async def test_a_deployment_cooldown_time_of_zero_skips_the_cooldown(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=two_deployments(model_info={"cooldown_time": 0}), num_retries=0)
+    host.answer("*", fresh(rate_limit))
+
+    await attempt(core)
+
+    assert cooled_down(core, ["a", "b"]) == {}
+
+
+# --- disable_cooldowns --------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("failure", [fresh(rate_limit), fresh(server_error)])
+@pytest.mark.asyncio
+async def test_disable_cooldowns_keeps_failing_deployments_routable(
+    make_core: MakeCore, host: ScriptedHost, failure: Callable[..., Exception]
+) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0, disable_cooldowns=True)
+    host.answer("*", failure)
+
+    for _ in range(4):
+        await attempt(core)
+
+    assert cooled_down(core, ["a", "b"]) == {}
+    assert len(host.calls_to("acompletion")) == 4
+
+
+@pytest.mark.asyncio
+async def test_disable_cooldowns_assigned_at_runtime_stops_cooldowns(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=0)
+    core.apply(ChangeSet(assign={"disable_cooldowns": True}))
+    host.answer("*", fresh(rate_limit))
+
+    await attempt(core)
+
+    assert core.settings().disable_cooldowns is True
+    assert cooled_down(core, ["a", "b"]) == {}
+
+
+# --- single-deployment groups -------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("arguments", [{}, {"allowed_fails": 0}], ids=["default", "allowed_fails=0"])
+@pytest.mark.parametrize("builder", [rate_limit, server_error, timeout])
+@pytest.mark.asyncio
+async def test_a_single_deployment_group_is_not_cooled_down(
+    make_core: MakeCore, host: ScriptedHost, arguments: dict[str, object], builder: ErrorBuilder
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=0, **arguments)
+    host.answer("openai/a", fresh(builder))
+
+    for _ in range(3):
+        await attempt(core)
+
+    assert cooled_down(core, ["a"]) == {}
+    assert host.deployments_called("acompletion") == ["a", "a", "a"]
+
+
+@pytest.mark.asyncio
+async def test_an_allowed_fails_policy_naming_the_error_cools_down_a_single_deployment(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a")],
+        num_retries=0,
+        allowed_fails_policy=AllowedFailsPolicy(InternalServerErrorAllowedFails=0),
+    )
+    host.answer("openai/a", fresh(server_error))
+
+    await attempt(core)
+    error: Final = await attempt(core)
+
+    assert set(cooled_down(core, ["a"])) == {"a"}
+    assert isinstance(error, litellm.RouterRateLimitError)
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+# --- health() -----------------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_health_cooldown_entry_describes_the_failure(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=two_deployments(), num_retries=0)
+    host.answer("*", fresh(rate_limit))
+
+    await attempt(core)
+
+    (entry,) = cooled_down(core, ["a", "b"]).values()
+    assert entry["status_code"] == "429"
+    assert "slow down" in str(entry["exception_received"])
+
+
+@pytest.mark.parametrize("failures", [1, 3])
+@pytest.mark.asyncio
+async def test_health_counts_failures_this_minute(make_core: MakeCore, host: ScriptedHost, failures: int) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=0)
+    host.answer("openai/a", fresh(server_error))
+
+    for _ in range(failures):
+        await attempt(core)
+
+    assert core.health().failures_this_minute("a") == failures
+    assert core.health().failures_this_minute("missing") == 0
+
+
+@pytest.mark.asyncio
+async def test_health_counts_every_retried_attempt_as_a_failure(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], num_retries=2)
+    host.answer("openai/a", fresh(server_error))
+
+    await attempt(core)
+
+    assert core.health().failures_this_minute("a") == 3
+
+
+@pytest.mark.asyncio
+async def test_health_record_failure_cools_down_past_allowed_fails(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=two_deployments(), allowed_fails=1)
+
+    first: Final = core.health().record_failure("a", server_error(), 500)
+    second: Final = core.health().record_failure("a", server_error(), 500)
+
+    assert (first, second) == (False, True)
+    assert set(cooled_down(core, ["a", "b"])) == {"a"}
+    assert core.health().failures_this_minute("a") == 2

tests/unit/test_router/core/test_selection.py

@@ -0,0 +1,624 @@
+"""
+Deployment selection on a router core: which deployment ``select`` picks and which one an endpoint
+call reaches, across groups, aliases, wildcard routes, deployment ids, teams, weights, order,
+routing strategies and cooldowns.
+"""
+
+import re
+from collections.abc import Callable, Mapping
+from typing import Final
+
+import openai
+import pytest
+
+import litellm
+from litellm.router_core import RouterCore
+from litellm.types.router import RouterRateLimitError
+from tests.unit.test_router.core.scripted_host import (
+    ScriptedHost,
+    auth_error,
+    bad_request,
+    context_window,
+    deployment,
+    rate_limit,
+    server_error,
+    timeout,
+)
+
+MESSAGES: Final = [{"role": "user", "content": "hello there, router"}]
+PICKS: Final = 20
+
+MakeCore = Callable[..., RouterCore]
+
+
+async def _picked_ids(
+    core: RouterCore, model: str, times: int = PICKS, **select_arguments: object
+) -> set[str]:  # kwargs-ok: select's keyword arguments
+    picked: Final[set[str]] = set()
+    for _ in range(times):
+        chosen = await core.select(model, **select_arguments)  # pyright: ignore[reportCallIssue]  # PythonCore's select takes keyword options
+        assert isinstance(chosen, Mapping)
+        picked.add(str(chosen["model_info"]["id"]))
+    return picked
+
+
+async def _fail_through(core: RouterCore, deployment_id: str, times: int = 1) -> None:
+    for _ in range(times):
+        with pytest.raises(openai.APIError):
+            await core.acompletion(model=deployment_id, messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]  # endpoint method
+
+
+# --- picking within a model group ----------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("model", "expected"), [("g1", "a"), ("g2", "b"), ("g3", "c")])
+async def test_select_returns_the_only_deployment_of_the_requested_group(
+    make_core: MakeCore, model: str, expected: str
+) -> None:
+    core: Final = make_core(model_list=[deployment("g1", "a"), deployment("g2", "b"), deployment("g3", "c")])
+    chosen: Final = await core.select(model)
+    assert chosen["model_info"]["id"] == expected  # pyright: ignore[reportIndexIssue]
+    assert chosen["model_name"] == model  # pyright: ignore[reportIndexIssue]
+    assert chosen["litellm_params"]["model"] == f"openai/{expected}"  # pyright: ignore[reportIndexIssue]
+
+
+@pytest.mark.asyncio
+async def test_select_only_picks_deployments_of_the_requested_group(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b"), deployment("other", "c"), deployment("other", "d")]
+    )
+    assert await _picked_ids(core, "g") <= {"a", "b"}
+    assert await _picked_ids(core, "other") <= {"c", "d"}
+
+
+@pytest.mark.asyncio
+async def test_select_does_not_call_the_provider(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    await core.select("g")
+    assert host.calls == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("op", "request_payload"),
+    [
+        ("acompletion", {"messages": MESSAGES}),
+        ("atext_completion", {"prompt": "hi"}),
+        ("aembedding", {"input": "hi"}),
+    ],
+)
+async def test_async_endpoint_calls_the_deployment_of_the_requested_group(
+    make_core: MakeCore, host: ScriptedHost, op: str, request_payload: dict[str, object]
+) -> None:
+    core: Final = make_core(model_list=[deployment("g1", "a"), deployment("g2", "b")])
+    await core.call(op, {"model": "g2", **request_payload})
+    assert host.deployments_called(op) == ["b"]
+    assert host.calls[-1].model == "openai/b"
+
+
+@pytest.mark.parametrize(
+    ("op", "request_payload"), [("completion", {"messages": MESSAGES}), ("embedding", {"input": "hi"})]
+)
+def test_sync_endpoint_calls_the_deployment_of_the_requested_group(
+    make_core: MakeCore, host: ScriptedHost, op: str, request_payload: dict[str, object]
+) -> None:
+    core: Final = make_core(model_list=[deployment("g1", "a"), deployment("g2", "b")])
+    getattr(core, op)(model="g1", **request_payload)
+    assert host.deployments_called(op) == ["a"]
+
+
+@pytest.mark.asyncio
+async def test_completion_reports_the_requested_group_in_its_headers(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g1", "a"), deployment("g2", "b")])
+    response: Final = await core.acompletion(model="g2", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "g2"
+    assert host.calls[-1].model_group == "g2"
+
+
+# --- model_group_alias ---------------------------------------------------------------------
+
+ALIASES: Final = {
+    "plain": "g",
+    "visible": {"model": "g", "hidden": False},
+    "hidden": {"model": "g", "hidden": True},
+}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("alias", sorted(ALIASES))
+async def test_select_resolves_an_alias_to_its_target_group(make_core: MakeCore, alias: str) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("other", "b")], model_group_alias=dict(ALIASES)
+    )
+    chosen: Final = await core.select(alias)
+    assert chosen["model_info"]["id"] == "a"  # pyright: ignore[reportIndexIssue]
+    assert chosen["model_name"] == "g"  # pyright: ignore[reportIndexIssue]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("alias", sorted(ALIASES))
+async def test_completion_through_an_alias_calls_the_target_group(
+    make_core: MakeCore, host: ScriptedHost, alias: str
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("other", "b")], model_group_alias=dict(ALIASES)
+    )
+    await core.acompletion(model=alias, messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called("acompletion") == ["a"]
+
+
+@pytest.mark.parametrize(("alias", "target"), [("plain", "g"), ("visible", "g"), ("hidden", "g"), ("g", None)])
+def test_registry_resolves_alias_targets(make_core: MakeCore, alias: str, target: str | None) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], model_group_alias=dict(ALIASES))
+    assert core.registry().resolve(alias, kind="alias") == target
+
+
+def test_hidden_alias_is_left_out_of_the_routable_names(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], model_group_alias=dict(ALIASES))
+    assert sorted(core.registry().names(kind="groups")) == ["g", "plain", "visible"]
+
+
+@pytest.mark.asyncio
+async def test_alias_to_an_unknown_group_fails_naming_the_target(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], model_group_alias={"dangling": "missing"})
+    with pytest.raises(
+        litellm.BadRequestError, match=re.escape("You passed in model=missing. There are no healthy deployments")
+    ):
+        await core.select("dangling")
+
+
+# --- wildcard / pattern groups ---------------------------------------------------------------
+
+WILDCARD_MODEL_LIST: Final = [
+    deployment("*", "catch_all", model="openai/*"),
+    deployment("anthropic/*", "anthropic_any", model="anthropic/*"),
+    deployment("openai/gpt-4o", "exact"),
+    deployment("openai/gpt-*", "gpt_any", model="openai/gpt-*"),
+]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("requested", "expected_id", "expected_group", "expected_model"),
+    [
+        ("openai/gpt-4o", "exact", "openai/gpt-4o", "openai/exact"),
+        ("openai/gpt-4o-mini", "gpt_any", "openai/gpt-*", "openai/gpt-4o-mini"),
+        ("anthropic/claude-x", "anthropic_any", "anthropic/*", "anthropic/claude-x"),
+        ("gpt-5", "catch_all", "*", "openai/gpt-5"),
+        ("whatever/foo", "catch_all", "*", "openai/whatever/foo"),
+    ],
+)
+async def test_select_prefers_an_exact_group_then_the_most_specific_pattern(
+    make_core: MakeCore, requested: str, expected_id: str, expected_group: str, expected_model: str
+) -> None:
+    core: Final = make_core(model_list=list(WILDCARD_MODEL_LIST))
+    chosen: Final = await core.select(requested)
+    assert chosen["model_info"]["id"] == expected_id  # pyright: ignore[reportIndexIssue]
+    assert chosen["model_name"] == expected_group  # pyright: ignore[reportIndexIssue]
+    assert chosen["litellm_params"]["model"] == expected_model  # pyright: ignore[reportIndexIssue]
+
+
+@pytest.mark.asyncio
+async def test_completion_through_a_wildcard_calls_the_requested_model(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("openai/*", "wild", model="openai/*")])
+    response: Final = await core.acompletion(model="openai/gpt-x", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called("acompletion") == ["wild"]
+    assert host.calls[-1].model == "openai/gpt-x"
+    assert response._hidden_params["additional_headers"]["x-litellm-model-group"] == "openai/gpt-x"
+
+
+@pytest.mark.asyncio
+async def test_a_pattern_that_does_not_match_is_an_unknown_model(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("anthropic/*", "anthropic_any", model="anthropic/*")])
+    with pytest.raises(litellm.BadRequestError, match="model=openai/gpt-4o"):
+        await core.select("openai/gpt-4o")
+
+
+# --- selecting by deployment id or litellm model --------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("deployment_id", ["a", "b", "c"])
+async def test_select_by_deployment_id_returns_that_deployment(make_core: MakeCore, deployment_id: str) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b"), deployment("h", "c")])
+    assert await _picked_ids(core, deployment_id) == {deployment_id}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("deployment_id", ["a", "b"])
+async def test_completion_by_deployment_id_calls_that_deployment(
+    make_core: MakeCore, host: ScriptedHost, deployment_id: str
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")])
+    for _ in range(5):
+        await core.acompletion(model=deployment_id, messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called("acompletion") == [deployment_id] * 5
+
+
+@pytest.mark.asyncio
+async def test_select_by_litellm_model_name_finds_the_deployment(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")])
+    assert await _picked_ids(core, "openai/b") == {"b"}
+
+
+@pytest.mark.asyncio
+async def test_specific_deployment_picks_by_litellm_model_across_groups(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("g1", "c", model="openai/shared"),
+            deployment("g2", "d", model="openai/shared"),
+            deployment("g3", "e"),
+        ]
+    )
+    assert await _picked_ids(core, "openai/shared", specific_deployment=True) <= {"c", "d"}
+    assert await _picked_ids(core, "openai/e", specific_deployment=True) == {"e"}
+
+
+@pytest.mark.asyncio
+async def test_completion_with_specific_deployment_calls_the_named_litellm_model(
+    make_core: MakeCore, host: ScriptedHost
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b")])
+    for _ in range(5):
+        await core.acompletion(model="openai/b", messages=MESSAGES, specific_deployment=True)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called("acompletion") == ["b"] * 5
+
+
+# --- unknown models ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("op", "request_payload"),
+    [
+        ("acompletion", {"messages": MESSAGES}),
+        ("aembedding", {"input": "hi"}),
+        ("atext_completion", {"prompt": "hi"}),
+        ("completion", {"messages": MESSAGES}),
+        ("embedding", {"input": "hi"}),
+    ],
+)
+async def test_unknown_model_raises_bad_request_naming_the_model(
+    make_core: MakeCore, host: ScriptedHost, op: str, request_payload: dict[str, object]
+) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    with pytest.raises(litellm.BadRequestError) as raised:
+        await core.call(op, {"model": "nope", **request_payload})
+    assert (
+        str(raised.value)
+        == "litellm.BadRequestError: You passed in model=nope. There are no healthy deployments for this model"
+    )
+    assert raised.value.status_code == 400
+    assert host.calls == []
+
+
+@pytest.mark.asyncio
+async def test_select_of_an_unknown_model_raises_bad_request(make_core: MakeCore) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")])
+    with pytest.raises(litellm.BadRequestError) as raised:
+        await core.select("nope")
+    assert raised.value.status_code == 400
+    assert raised.value.model == "nope"
+    assert "There are no healthy deployments for this model" in str(raised.value)
+
+
+# --- team-scoped deployments --------------------------------------------------------------------
+
+TEAM_MODEL_LIST: Final = [
+    deployment("g", "public"),
+    deployment("team-x-group", "team_x", model_info={"team_id": "tx", "team_public_model_name": "g"}),
+    deployment("team-y-group", "team_y", model_info={"team_id": "ty", "team_public_model_name": "g"}),
+]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("metadata", "expected"),
+    [
+        ({}, "public"),
+        ({"user_api_key_team_id": "tx"}, "team_x"),
+        ({"user_api_key_team_id": "ty"}, "team_y"),
+        ({"user_api_key_team_id": "unknown-team"}, "public"),
+    ],
+)
+async def test_select_serves_a_team_its_own_deployment_under_the_public_name(
+    make_core: MakeCore, metadata: dict[str, str], expected: str
+) -> None:
+    core: Final = make_core(model_list=list(TEAM_MODEL_LIST))
+    assert await _picked_ids(core, "g", request_kwargs={"metadata": metadata}) == {expected}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(("team_id", "expected"), [("tx", "team_x"), ("ty", "team_y"), ("tz", "public")])
+async def test_completion_for_a_team_calls_its_deployment_under_the_public_group(
+    make_core: MakeCore, host: ScriptedHost, team_id: str, expected: str
+) -> None:
+    core: Final = make_core(model_list=list(TEAM_MODEL_LIST))
+    await core.acompletion(model="g", messages=MESSAGES, metadata={"user_api_key_team_id": team_id})  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called("acompletion") == [expected]
+    assert host.calls[-1].model_group == "g"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("metadata", [{}, {"user_api_key_team_id": "other"}])
+async def test_team_only_public_name_is_unknown_to_other_callers(make_core: MakeCore, metadata: dict[str, str]) -> None:
+    core: Final = make_core(
+        model_list=[deployment("team-x-group", "team_x", model_info={"team_id": "tx", "team_public_model_name": "pub"})]
+    )
+    with pytest.raises(litellm.BadRequestError, match="model=pub"):
+        await core.select("pub", {"metadata": metadata})
+    assert await _picked_ids(core, "pub", request_kwargs={"metadata": {"user_api_key_team_id": "tx"}}) == {"team_x"}
+
+
+@pytest.mark.parametrize(("team_id", "names"), [(None, []), ("tx", ["pub"])])
+def test_team_public_names_are_routable_only_for_their_team(
+    make_core: MakeCore, team_id: str | None, names: list[str]
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("team-x-group", "team_x", model_info={"team_id": "tx", "team_public_model_name": "pub"})]
+    )
+    assert list(core.registry().names(kind="groups", team_id=team_id)) == names
+
+
+# --- weights, rpm and tpm under simple-shuffle -----------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("metric", ["weight", "rpm", "tpm"])
+async def test_simple_shuffle_never_picks_a_zero_weighted_deployment_beside_a_positive_one(
+    make_core: MakeCore, metric: str
+) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "unset"),
+            deployment("g", "positive", **{metric: 5}),
+            deployment("g", "zero", **{metric: 0}),
+        ]
+    )
+    assert await _picked_ids(core, "g") == {"positive"}
+
+
+@pytest.mark.asyncio
+async def test_weight_takes_precedence_over_rpm_and_tpm(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "weighted", weight=1, rpm=0, tpm=0),
+            deployment("g", "by_limits", weight=0, rpm=100, tpm=100),
+        ]
+    )
+    assert await _picked_ids(core, "g") == {"weighted"}
+
+
+@pytest.mark.asyncio
+async def test_completion_calls_only_the_weighted_deployment(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a", weight=0), deployment("g", "b", weight=3)])
+    for _ in range(10):
+        await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert set(host.deployments_called("acompletion")) == {"b"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("favoured", ["a", "b"])
+async def test_request_router_weights_override_deployment_weights(make_core: MakeCore, favoured: str) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a", weight=1), deployment("g", "b", weight=1)])
+    picked: Final = await _picked_ids(core, "g", request_kwargs={"_router_weights": {"g": {favoured: 1.0}}})
+    assert picked == {favoured}
+
+
+# --- order --------------------------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_select_picks_only_the_lowest_order(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[
+            deployment("g", "second", order=2),
+            deployment("g", "first_a", order=1),
+            deployment("g", "first_b", order=1),
+            deployment("g", "third", order=3),
+        ]
+    )
+    assert await _picked_ids(core, "g") <= {"first_a", "first_b"}
+
+
+@pytest.mark.asyncio
+async def test_completion_calls_only_the_lowest_order(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "second", order=2), deployment("g", "first", order=1)])
+    for _ in range(5):
+        await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called("acompletion") == ["first"] * 5
+
+
+@pytest.mark.asyncio
+async def test_next_order_serves_while_the_lowest_order_cools_down(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "second", order=2), deployment("g", "first", order=1)],
+        allowed_fails=0,
+        num_retries=0,
+        cooldown_time=60,
+    )
+    host.answer("openai/first", rate_limit())
+    await _fail_through(core, "first")
+    assert await _picked_ids(core, "g") == {"second"}
+
+
+# --- routing strategies ----------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("routing_strategy", ["least-busy", "usage-based-routing"])
+@pytest.mark.parametrize("ids", [("a", "b"), ("b", "a")])
+async def test_strategy_without_traffic_picks_the_first_configured_deployment(
+    make_core: MakeCore, routing_strategy: str, ids: tuple[str, str]
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", deployment_id) for deployment_id in ids], routing_strategy=routing_strategy
+    )
+    assert await _picked_ids(core, "g", messages=MESSAGES) == {ids[0]}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("cheap_first", [True, False])
+async def test_cost_based_routing_picks_the_cheapest_deployment(make_core: MakeCore, cheap_first: bool) -> None:
+    cheap: Final = deployment("g", "cheap", input_cost_per_token=0.000001, output_cost_per_token=0.000002)
+    pricey: Final = deployment("g", "pricey", input_cost_per_token=0.00001, output_cost_per_token=0.00002)
+    core: Final = make_core(
+        model_list=[cheap, pricey] if cheap_first else [pricey, cheap], routing_strategy="cost-based-routing"
+    )
+    assert await _picked_ids(core, "g", messages=MESSAGES) == {"cheap"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("ids", [("tight", "roomy"), ("roomy", "tight")])
+async def test_usage_based_v2_skips_a_deployment_whose_tpm_the_request_exceeds(
+    make_core: MakeCore, ids: tuple[str, str]
+) -> None:
+    limits: Final = {"tight": 1, "roomy": 100_000}
+    core: Final = make_core(
+        model_list=[deployment("g", deployment_id, tpm=limits[deployment_id]) for deployment_id in ids],
+        routing_strategy="usage-based-routing-v2",
+    )
+    assert await _picked_ids(core, "g", messages=MESSAGES) == {"roomy"}
+
+
+@pytest.mark.asyncio
+async def test_usage_based_v2_rate_limits_when_the_request_exceeds_every_tpm(make_core: MakeCore) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a", tpm=1), deployment("g", "b", tpm=1)],
+        routing_strategy="usage-based-routing-v2",
+    )
+    with pytest.raises(
+        litellm.RateLimitError, match=re.escape("No deployments available for selected model. Passed model=g")
+    ):
+        await core.select("g", messages=MESSAGES)  # pyright: ignore[reportCallIssue]
+
+
+# --- cooldowns ----------------------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("error", [rate_limit, auth_error, server_error, timeout], ids=lambda e: e.__name__)
+async def test_a_deployment_that_failed_past_allowed_fails_is_not_picked(
+    make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception]
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60
+    )
+    host.answer("openai/a", error())
+    await _fail_through(core, "a")
+    assert await _picked_ids(core, "g") == {"b"}
+
+
+@pytest.mark.asyncio
+async def test_completion_avoids_a_cooling_deployment(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60
+    )
+    host.answer("openai/a", rate_limit())
+    await _fail_through(core, "a")
+    for _ in range(5):
+        await core.acompletion(model="g", messages=MESSAGES)  # pyright: ignore[reportAttributeAccessIssue]
+    assert host.deployments_called("acompletion") == ["a", "b", "b", "b", "b", "b"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("error", [bad_request, context_window], ids=lambda e: e.__name__)
+async def test_client_errors_do_not_cool_a_deployment_down(
+    make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception]
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60
+    )
+    host.answer("openai/a", error())
+    await _fail_through(core, "a", times=3)
+    assert await core.select("a") is not None
+    assert "a" in await _picked_ids(core, "g", times=60)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("allowed_fails", [1, 2, 3])
+async def test_a_deployment_stays_pickable_until_it_fails_more_than_allowed_fails(
+    make_core: MakeCore, host: ScriptedHost, allowed_fails: int
+) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")],
+        allowed_fails=allowed_fails,
+        num_retries=0,
+        cooldown_time=60,
+    )
+    host.answer("openai/a", rate_limit())
+    await _fail_through(core, "a", times=allowed_fails)
+    assert "a" in await _picked_ids(core, "g", times=60)
+    await _fail_through(core, "a")
+    assert await _picked_ids(core, "g") == {"b"}
+
+
+@pytest.mark.asyncio
+async def test_a_single_deployment_group_is_never_cooled_down(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(model_list=[deployment("g", "a")], allowed_fails=0, num_retries=0, cooldown_time=60)
+    host.answer("openai/a", rate_limit())
+    await _fail_through(core, "a", times=3)
+    assert await _picked_ids(core, "g") == {"a"}
+
+
+@pytest.mark.asyncio
+async def test_selecting_a_cooling_deployment_by_id_still_returns_it(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60
+    )
+    host.answer("openai/a", rate_limit())
+    await _fail_through(core, "a")
+    assert await _picked_ids(core, "g") == {"b"}
+    assert await _picked_ids(core, "a") == {"a"}
+
+
+async def _all_cooling_core(make_core: MakeCore, host: ScriptedHost) -> RouterCore:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b")], allowed_fails=0, num_retries=0, cooldown_time=60
+    )
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", rate_limit())
+    await _fail_through(core, "a")
+    await _fail_through(core, "b")
+    return core
+
+
+ALL_COOLING_MESSAGE: Final = (
+    "No deployments available for selected model, Try again in 60 seconds. All deployments for selected "
+    "model are in cooldown. Passed model=g. pre-call-checks=False, cooldown_list=['a', 'b']"
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("entry", ["select", "acompletion", "completion"])
+async def test_every_deployment_cooling_raises_router_rate_limit_error(
+    make_core: MakeCore, host: ScriptedHost, entry: str
+) -> None:
+    core: Final = await _all_cooling_core(make_core, host)
+    calls_before: Final = len(host.calls)
+    call: Final = core.select("g") if entry == "select" else core.call(entry, {"model": "g", "messages": MESSAGES})
+    with pytest.raises(RouterRateLimitError) as raised:
+        await call
+    assert str(raised.value) == ALL_COOLING_MESSAGE
+    assert raised.value.model == "g"
+    assert raised.value.cooldown_time == 60
+    assert sorted(raised.value.cooldown_list) == ["a", "b"]
+    assert len(host.calls) == calls_before
+
+
+@pytest.mark.asyncio
+async def test_cooling_down_one_group_leaves_other_groups_pickable(make_core: MakeCore, host: ScriptedHost) -> None:
+    core: Final = make_core(
+        model_list=[deployment("g", "a"), deployment("g", "b"), deployment("h", "c"), deployment("h", "d")],
+        allowed_fails=0,
+        num_retries=0,
+        cooldown_time=60,
+    )
+    host.answer("openai/a", rate_limit())
+    host.answer("openai/b", rate_limit())
+    await _fail_through(core, "a")
+    await _fail_through(core, "b")
+    with pytest.raises(RouterRateLimitError):
+        await core.select("g")
+    assert await _picked_ids(core, "h") <= {"c", "d"}

tests/unit/test_router/test_router.py

@@ -2893,11 +2893,11 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto
         },
     )
     assert router.routing_strategy == "lar1"
-    assert "_core_async_get_available_deployment" in router.__dict__
+    assert "_core_async_get_available_deployment" in router._core.__dict__
 
     router._switch_routing_strategy("usage-based-routing-v2", {})
     assert router.lowesttpm_logger_v2 is not None
-    assert "async_get_available_deployment" not in router.__dict__
+    assert "_core_async_get_available_deployment" not in router._core.__dict__
 
 
 @pytest.mark.parametrize(
@@ -24274,7 +24274,7 @@ def _record_phase_events(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, dic
     def record(name: str, attributes: dict[str, str | int]) -> None:
         events.append((name, dict(attributes)))
 
-    monkeypatch.setattr(litellm.router, "phase_event", record)
+    monkeypatch.setattr(litellm.router_python_core, "phase_event", record)
     return events
 
 
@@ -25669,7 +25669,9 @@ def test_get_model_list_from_model_alias_should_not_iterate_for_non_alias_lookup
         ],
         model_group_alias={"alias-1": "gpt-5.5"},
     )
-    router.model_group_alias = NoItemsAliasDict({f"alias-{idx}": "gpt-5.5" for idx in range(200)})
+    # Assigning model_group_alias copies the value (a changeset detaches it), so the tripwire goes
+    # straight onto the core, where the lookup reads it.
+    router._core._core_model_group_alias = NoItemsAliasDict({f"alias-{idx}": "gpt-5.5" for idx in range(200)})
 
     model_alias_list = router.get_model_list_from_model_alias(model_name="gpt-5-mini")
     assert model_alias_list == []
@@ -25689,7 +25691,9 @@ def test_map_team_model_should_not_iterate_aliases_for_non_alias_team_model_name
         ],
         model_group_alias={"alias-1": "gpt-5.5"},
     )
-    router.model_group_alias = NoItemsAliasDict({f"alias-{idx}": "gpt-5.5" for idx in range(200)})
+    # Assigning model_group_alias copies the value (a changeset detaches it), so the tripwire goes
+    # straight onto the core, where the lookup reads it.
+    router._core._core_model_group_alias = NoItemsAliasDict({f"alias-{idx}": "gpt-5.5" for idx in range(200)})
 
     # map_team_model should return the public name unchanged (not the internal UUID name)
     # so the router can find all sibling deployments via team_id filtering
@@ -29142,13 +29146,13 @@ def test_reset_custom_routing_strategy():
     router = _create_custom_routing_router()
     router.set_custom_routing_strategy(SpecialEndpointRoutingStrategy(router))
 
-    assert "get_available_deployment" in router.__dict__
-    assert "_core_async_get_available_deployment" in router.__dict__
+    assert "get_available_deployment" in router._core.__dict__
+    assert "_core_async_get_available_deployment" in router._core.__dict__
 
     router._reset_custom_routing_strategy()
 
-    assert "get_available_deployment" not in router.__dict__
-    assert "async_get_available_deployment" not in router.__dict__
+    assert "get_available_deployment" not in router._core.__dict__
+    assert "_core_async_get_available_deployment" not in router._core.__dict__
     assert router.async_get_available_deployment.__func__ is Router.async_get_available_deployment
 
     router._reset_custom_routing_strategy()
@@ -30988,7 +30992,7 @@ async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_s
 
 
 _SYNC_SECONDS: Final = 10
-_ROUTER_FILE: Final = Router.__init__.__code__.co_filename
+_ROUTER_FILE: Final = Router._update_deployment_indices_after_removal.__code__.co_filename
 _INDEX_LOCK_LINES: Final = frozenset(
     number
     for number, line in enumerate(linecache.getlines(_ROUTER_FILE), start=1)

tests/unit/test_router/test_router_entry_points.py

@@ -359,7 +359,7 @@ def test_settings_snapshot_is_frozen(router):
 def test_unset_setting_raises_attribute_error_through_old_name():
     r = _router()
     try:
-        del r.__dict__["_core_kubernetes_pod_discovery"]
+        del r._core.__dict__["_core_kubernetes_pod_discovery"]
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
             assert getattr(r, "kubernetes_pod_discovery", "missing") == "missing"
@@ -372,14 +372,14 @@ def test_unset_setting_raises_attribute_error_through_old_name():
 
 def test_unset_attribute_error_names_the_public_attribute(router):
     # Reading (or deleting) an old attribute the router never set raises what reading it raised before.
-    router.__dict__.pop("_core_slack_alerting_logger", None)
+    router._core.__dict__.pop("_core_slack_alerting_logger", None)
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         with pytest.raises(AttributeError) as read:
             _ = router.slack_alerting_logger
         with pytest.raises(AttributeError) as deleted:
             del router.slack_alerting_logger
-        del router.__dict__["_core_kubernetes_pod_discovery"]
+        del router._core.__dict__["_core_kubernetes_pod_discovery"]
         with pytest.raises(AttributeError) as setting:
             _ = router.kubernetes_pod_discovery
     for caught, name in ((read, "slack_alerting_logger"), (setting, "kubernetes_pod_discovery")):

tests/unit/test_router/test_router_facade.py

@@ -0,0 +1,52 @@
+"""
+The ``litellm.Router`` facade's own helpers in litellm/router.py: how it binds the constructor
+arguments, which members it leaves to the facade, and the module attributes it forwards to the core.
+"""
+
+import pytest
+
+import litellm.router as router_module
+from litellm import router_python_core
+from litellm.router import Router, _class_descriptor, router_arguments
+
+
+def test_router_arguments_binds_positional_arguments_to_their_parameter_names():
+    model_list = [{"model_name": "g", "litellm_params": {"model": "openai/gpt-4o-mini"}}]
+
+    arguments = router_arguments(model_list, num_retries=2)
+
+    assert dict(arguments) == {"model_list": model_list, "num_retries": 2}
+
+
+def test_router_arguments_leaves_unset_parameters_out():
+    assert dict(router_arguments()) == {}
+
+
+def test_router_arguments_rejects_an_unknown_argument_naming_the_router():
+    with pytest.raises(TypeError, match=r"^Router\.__init__\(\) got an unexpected keyword argument 'not_a_setting'"):
+        router_arguments(not_a_setting=1)
+
+
+def test_router_arguments_are_read_only():
+    arguments = router_arguments(num_retries=2)
+
+    with pytest.raises(TypeError):
+        arguments["num_retries"] = 3  # pyright: ignore[reportIndexIssue]  # proving the mapping is read-only
+
+
+def test_class_descriptor_is_true_for_a_data_descriptor_on_the_class():
+    assert _class_descriptor(Router, "_core")
+
+
+def test_class_descriptor_is_false_for_a_plain_method_or_a_missing_name():
+    assert not _class_descriptor(Router, "from_config")
+    assert not _class_descriptor(Router, "no_such_member")
+
+
+def test_module_getattr_forwards_old_helpers_to_the_core_module():
+    assert router_module.__getattr__("_stream_pre_content_error") is router_python_core._stream_pre_content_error
+
+
+def test_module_getattr_raises_attribute_error_for_an_unknown_name():
+    with pytest.raises(AttributeError):
+        router_module.__getattr__("no_such_router_helper")

tests/unit/test_router/test_router_helper_utils.py

@@ -1182,7 +1182,7 @@ def test_deployment_callback_respects_cooldown_time(model_list):
         },
     }
 
-    with patch("litellm.router.set_cooldown_deployments") as mock_set:
+    with patch("litellm.router_python_core.set_cooldown_deployments") as mock_set:
         router.deployment_callback_on_failure(
             kwargs=kwargs,
             completion_response=None,
@@ -1554,7 +1554,7 @@ def _rpm_tpm_router(model_id: str) -> Router:
 
 
 def _pin_router_clock(monkeypatch: pytest.MonkeyPatch, pinned: datetime) -> None:
-    monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned)
+    monkeypatch.setattr("litellm.router_python_core.get_utc_datetime", lambda: pinned)
     monkeypatch.setattr(
         "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: pinned
     )
@@ -2398,10 +2398,10 @@ def _routers_seen_in_success_loop_discarding_first(first: Router, second: Router
     for callback in litellm.success_callback:
         if not isinstance(callback, MethodType):
             continue
-        if callback.__self__ is first:
+        if callback.__self__ is first._core:
             first.discard()
             yield first
-        if callback.__self__ is second:
+        if callback.__self__ is second._core:
             yield second
 
 

tests/unit/test_router/test_router_provider_endpoints.py

@@ -53,7 +53,7 @@ def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
 @pytest.fixture
 def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> None:
     pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc)
-    monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned)
+    monkeypatch.setattr("litellm.router_python_core.get_utc_datetime", lambda: pinned)
 
 
 class _RouterLoggingCapture(CustomLogger):

tests/unit/test_router_silent_experiment.py

@@ -620,7 +620,7 @@ def test_sync_shadow_gets_kwargs_snapshot_taken_before_primary_mutates_them(reco
 
     router = Router(model_list=_tagged_primary_model_list())
     with patch(  # test-quality-ok: Router has no thread factory to inject; deferring start is the only deterministic way to expose the race
-        "litellm.router.threading", SimpleNamespace(Thread=_DeferredThread)
+        "litellm.router_python_core.threading", SimpleNamespace(Thread=_DeferredThread)
     ):
         response = router.completion(
             model="primary-model",
@@ -652,7 +652,7 @@ def test_sync_shadow_workers_do_not_share_metadata_with_each_other(recording_log
 
     router = Router(model_list=_streaming_model_list(["shadow-a", "shadow-b"]))
     with patch(  # test-quality-ok: Router has no thread factory to inject; deferring start is the only deterministic way to expose the race
-        "litellm.router.threading", SimpleNamespace(Thread=_DeferredThread)
+        "litellm.router_python_core.threading", SimpleNamespace(Thread=_DeferredThread)
     ):
         router.completion(
             model="primary-model",

tests/unit/test_router_weighted_failover.py

@@ -338,7 +338,7 @@ async def test_maybe_run_weighted_failover_persists_excluded_ids_to_kwargs(monke
     async def _stub_run_async_fallback(*args, **kwargs):
         return "ok"
 
-    monkeypatch.setattr("litellm.router.run_async_fallback", _stub_run_async_fallback)
+    monkeypatch.setattr("litellm.router_python_core.run_async_fallback", _stub_run_async_fallback)
 
     exc = Exception("fail")
     exc.failed_deployment_id = "A"
@@ -807,11 +807,11 @@ async def test_maybe_run_weighted_failover_skips_when_remaining_all_in_cooldown(
         run_async_fallback_called = True
         return "should not reach here"
 
-    monkeypatch.setattr("litellm.router.run_async_fallback", _should_not_be_called)
+    monkeypatch.setattr("litellm.router_python_core.run_async_fallback", _should_not_be_called)
 
     # Patch cooldown so B and C appear in cooldown.
     with patch(
-        "litellm.router.async_get_cooldown_deployments",
+        "litellm.router_python_core.async_get_cooldown_deployments",
         new=AsyncMock(return_value=["B", "C"]),
     ):
         result = await router._maybe_run_weighted_failover(
@@ -867,10 +867,10 @@ async def test_maybe_run_weighted_failover_proceeds_when_one_healthy_remains(
         run_async_fallback_called = True
         return "ok from C"
 
-    monkeypatch.setattr("litellm.router.run_async_fallback", _stub_run_async_fallback)
+    monkeypatch.setattr("litellm.router_python_core.run_async_fallback", _stub_run_async_fallback)
 
     with patch(
-        "litellm.router.async_get_cooldown_deployments",
+        "litellm.router_python_core.async_get_cooldown_deployments",
         new=AsyncMock(return_value=["B"]),
     ):
         result = await router._maybe_run_weighted_failover(
@@ -936,7 +936,7 @@ async def test_failover_falls_through_to_external_fallback_when_remaining_in_coo
 
     # Put B in cooldown so weighted failover can't use it after A fails.
     with patch(
-        "litellm.router.async_get_cooldown_deployments",
+        "litellm.router_python_core.async_get_cooldown_deployments",
         new=AsyncMock(return_value=["B"]),
     ):
         response = await router.acompletion(