diff --git a/litellm/router_core.py b/litellm/router_core.py index 3ac3bf067f1..42a8ba39f0a 100644 --- a/litellm/router_core.py +++ b/litellm/router_core.py @@ -7,7 +7,7 @@ that reaches outside itself only through its host; it does not use ``PythonCore` picked once, when the router is built, from its ``RouterConfig``. """ -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable from litellm._logging import verbose_router_logger @@ -52,6 +52,39 @@ class RouterCore(Protocol): def retire(self) -> None: ... +_REQUEST_METADATA_KEYS: Final = ("metadata", "litellm_metadata") + + +def detached_request( + request_kwargs: Mapping[str, object] | None, +) -> dict[str, object]: # mutable-ok: the selection cores take a dict + """ + A copy of a caller's request for ``select``: selection pops keys, pins tier params and stamps + routing metadata on the request it routes, and none of that may reach the caller's dict (or the + metadata dicts inside it). + """ + if request_kwargs is None: + return {} + return { + key: dict(value) if key in _REQUEST_METADATA_KEYS and isinstance(value, Mapping) else value # pyright: ignore[reportUnknownArgumentType] # a request value + for key, value in request_kwargs.items() + } + + +def organization_deployment_id( + declared_id: str | None, position: int, organization: object, generated: Callable[[], str] +) -> str: + """ + The id of one deployment a ``litellm_params.organization`` list fans out into: each gets its own. + A declared id stays on the first organization's deployment (so lookups by it keep working) and is + suffixed with the organization on the others; without one, ``generated`` hashes that + organization's own params. + """ + if declared_id is None: + return generated() + return declared_id if position == 0 else f"{declared_id}-{organization}" + + def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore: """ The Rust core when the route catalog's router rule allows it and the Rust core serves this diff --git a/litellm/router_deprecated.py b/litellm/router_deprecated.py index eb7d9edda34..9489fcf4bb6 100644 --- a/litellm/router_deprecated.py +++ b/litellm/router_deprecated.py @@ -141,7 +141,10 @@ class _DeprecatedAttribute(Generic[_T]): def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None: _note(self.name, self.replacement) - if self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES: + if self.name == "model_list": + # A whole new list is reindexed, so a replaced id never finds the new list's entries. + _as_router(obj)._assign_model_list(value) # pyright: ignore[reportArgumentType] # the assigned list + elif self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES: _as_router(obj).apply(ChangeSet(assign={self.name: value})) else: setattr(obj, "_core_" + self.name, value) @@ -170,6 +173,15 @@ def _attr(replacement: str, read: Callable[["Router"], _T]) -> _DeprecatedAttrib return _DeprecatedAttribute(replacement, read) +def _select_in_place(router: "Router") -> Callable[..., Coroutine[object, object, object]]: + """ + The core's selection on the caller's own ``request_kwargs``: the old members changed them in + place (for example taking ``_retry_skipped_deployment_ids`` off), where ``select`` works on a copy. + """ + core: Final[object] = router.__dict__.get("_core", router) # pyright: ignore[reportAny] # the facade keeps its core in __dict__ + return core._select_in_place # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # every core has one + + def _async_picker(router: "Router") -> Callable[..., Awaitable[object]]: """ The router's async pick (the old ``async_get_available_deployment`` attribute). A core that @@ -187,7 +199,7 @@ def _async_picker(router: "Router") -> Callable[..., Awaitable[object]]: input: str | list | None = None, # mutable-ok: the old parameter type, unchanged specific_deployment: bool | None = False, ) -> object: - return await router.select( + return await _select_in_place(router)( model, request_kwargs, messages=messages, input=input, specific_deployment=specific_deployment ) @@ -809,7 +821,7 @@ class DeprecatedRouterMembers: health_check_probe: bool = False, ) -> list[dict] | dict: # mutable-ok: the old return type, unchanged _note("async_get_healthy_deployments", "select(model, request_kwargs, kind='healthy')") - return await _as_router(self).select( + return await _select_in_place(_as_router(self))( # pyright: ignore[reportReturnType] # the core's untyped deployment dicts model, request_kwargs, kind="healthy", @@ -830,7 +842,7 @@ class DeprecatedRouterMembers: ): _note("get_available_deployment_for_pass_through", "select(model, kind='pass_through')") return _run_ready( - _as_router(self).select( + _select_in_place(_as_router(self))( model, request_kwargs, kind="pass_through", diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 57fb3c2dafd..8df9e607e96 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -124,6 +124,7 @@ from litellm.litellm_core_utils.clientside_credential_handler import ( ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, coerce_token_limit, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -255,7 +256,11 @@ from litellm.router_api import ( ) from litellm.router_api import RoutingContext as RouterContext from litellm.router_api import deployment_id as _deployment_id +from litellm.router_core import detached_request, organization_deployment_id from litellm.router_deprecated import DeprecatedRouterMembers +from litellm.router_deprecated import ( + _note as _note_deprecated_member, # pyright: ignore[reportPrivateUsage] # the deprecated write notes its use +) from litellm.router_endpoint_table import ( ENDPOINTS, SYNC_DISPATCHES, @@ -369,9 +374,14 @@ from litellm.router_utils.fallback_event_handlers import ( run_async_fallback, ) from litellm.router_utils.global_callback_holders import ( + OUTCOME_CALLBACK_LISTS, + adaptive_routers_on_host, live_check_owners, + on_host, routers_holding_global_callbacks, routers_holding_global_callbacks_lock, + sync_adaptive_response_header_hook, + without_router_callbacks, ) from litellm.router_utils.handle_error import ( async_raise_no_deployment_exception, @@ -1525,6 +1535,11 @@ class PythonCore(DeprecatedRouterMembers): # Settings change only through a changeset, so every core serving this router sees it. self.apply(ChangeSet(assign={name.removeprefix("_core_"): value})) return + if name == "model_list" and "_core_model_list" in self.__dict__: + # ``core.model_list = [...]`` on the core itself: noted and reindexed as through the Router. + _note_deprecated_member("model_list", "registry().all()") + self._assign_model_list(value) # pyright: ignore[reportUnknownMemberType, reportArgumentType] # the assigned list, as given + return if name in _SETTING_ATTRIBUTES: self.__dict__.pop("_settings_snapshot", None) elif name == "_core_model_list" and "_core_model_list" in self.__dict__: @@ -1589,12 +1604,37 @@ class PythonCore(DeprecatedRouterMembers): """ Pick a deployment for ``model`` without calling it (``pick``), list the deployments that are healthy for the request (``healthy``), or pick one for a pass-through route - (``pass_through``). + (``pass_through``). The caller's ``request_kwargs`` are left unchanged: selection works on a copy. """ + return await self._select_in_place( # pyright: ignore[reportUnknownVariableType] # the selection cores return untyped deployment dicts + model, + None if request_kwargs is None else detached_request(request_kwargs), + kind=kind, + messages=messages, + input=input, + specific_deployment=specific_deployment, + parent_otel_span=parent_otel_span, + health_check_probe=health_check_probe, + ) + + async def _select_in_place( # pyright: ignore[reportUnknownParameterType] # the selection cores return untyped deployment dicts + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: pinned and stamped in place + *, + kind: SelectKind = "pick", + messages: list[dict[str, str]] | None = None, # mutable-ok: the selection cores take a list + input: str | list[object] | None = None, # mutable-ok: the selection cores take a list + specific_deployment: bool | None = False, + parent_otel_span: Span | None = None, + health_check_probe: bool = False, + ) -> "list[UntypedDict] | UntypedDict": # mutable-ok: the core's type (list[dict] | dict), as select returns it + """``select`` on the request the router itself routes, which keeps what selection pins on it.""" + request: Final = request_kwargs if request_kwargs is not None else {} if kind == "pick": return await self._core_async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts model=model, - request_kwargs=request_kwargs if request_kwargs is not None else {}, + request_kwargs=request, messages=messages, input=input, specific_deployment=specific_deployment, @@ -1602,7 +1642,7 @@ class PythonCore(DeprecatedRouterMembers): if kind == "healthy": return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts model=model, - request_kwargs=request_kwargs if request_kwargs is not None else {}, + request_kwargs=request, messages=messages, input=input, specific_deployment=specific_deployment, @@ -2065,7 +2105,8 @@ class PythonCore(DeprecatedRouterMembers): f"default_fallbacks must be a list of model names, got '{_fallbacks}'. Use ['{_fallbacks}']" ) if self._core_fallbacks is not None: - self._core_fallbacks.append({"*": _fallbacks}) + # A new list: the configured one may be the process-wide ``litellm.fallbacks``. + self._core_fallbacks = [*self._core_fallbacks, {"*": _fallbacks}] # pyright: ignore[reportUnknownMemberType] # the configured fallback entries else: self._core_fallbacks = [{"*": _fallbacks}] @@ -2224,6 +2265,8 @@ class PythonCore(DeprecatedRouterMembers): live_check_owners.discard(self) self.retire_routing_state() self._release_global_callbacks() + # The shared adaptive response header hook leaves with the host's last adaptive router. + sync_adaptive_response_header_hook(self._host.callbacks, adaptive_routers_on_host(self._host)) def _release_global_callbacks(self) -> None: with _routers_holding_global_callbacks_lock: @@ -2241,7 +2284,7 @@ class PythonCore(DeprecatedRouterMembers): method_owners=(self, *slack_alerting_loggers), ) with _routers_holding_global_callbacks_lock: - survivors: Final = tuple(_routers_holding_global_callbacks) + survivors: Final = on_host(tuple(_routers_holding_global_callbacks), self._host) candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors) for candidate in self._host.callbacks.with_custom_logger_keys(candidates, vacated_keys): self._register_router_selector(candidate) @@ -3189,7 +3232,10 @@ class PythonCore(DeprecatedRouterMembers): self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls] if any( router is not self and any(type(cb) is callback_cls for cb in (router.optional_callbacks or [])) - for router in (*_live_routers, *(owner for owner in live_check_owners if not isinstance(owner, PythonCore))) + for router in on_host( + (*_live_routers, *(owner for owner in live_check_owners if not isinstance(owner, PythonCore))), + self._host, + ) ): return for cb in tuple(self._host.callback_list("callbacks")): @@ -11009,7 +11055,6 @@ class PythonCore(DeprecatedRouterMembers): changes, otherwise a released router keeps recording turns through its hook.""" from litellm.router_strategy.adaptive_router.hooks import ( AdaptiveRouterPostCallHook, - AdaptiveRouterResponseHeaderHook, ) for callback in self._host.callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook): @@ -11017,8 +11062,9 @@ class PythonCore(DeprecatedRouterMembers): for tagged_adaptive_routers in self._core_adaptive_routers.values(): for tagged in tagged_adaptive_routers: self._host.callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)) - if self._core_adaptive_routers: - self._host.callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook()) + sync_adaptive_response_header_hook( + self._host.callbacks, bool(self._core_adaptive_routers) or adaptive_routers_on_host(self._host) + ) def init_adaptive_router_deployment(self, deployment: Deployment) -> None: """ @@ -11161,6 +11207,10 @@ class PythonCore(DeprecatedRouterMembers): self._core_team_model_to_deployment_indices = {} # Reset the team_model index self._core_team_pattern_routers = {} self._core_team_public_model_names = frozenset() + # The replaced list's wildcard routes and names go with it. + self._core_pattern_router = PatternMatchRouter() + self._core_provider_default_deployment_ids.clear() + self._core_deployment_names.clear() # Reset per-strategy router registries so hot-reload doesn't leave # stale routers pointing at the old model_list. self._core_quality_routers = {} @@ -11214,13 +11264,24 @@ class PythonCore(DeprecatedRouterMembers): if _litellm_params.get("organization", None) is not None and isinstance( _litellm_params["organization"], list ): # Addresses https://github.com/BerriAI/litellm/issues/3949 - for org in _litellm_params["organization"]: + for position, org in enumerate(tuple(_litellm_params["organization"])): # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # configured organizations _litellm_params["organization"] = org + # Each organization's deployment has its own id: a declared id stays on the + # first and is suffixed with the organization on the rest, a generated one is + # generated for that organization's params. self._create_deployment( deployment_info=model, _model_name=_model_name, _litellm_params=_litellm_params, - _model_info=_model_info, + _model_info={ + **_model_info, + "id": organization_deployment_id( + declared_id, + position, + org, # pyright: ignore[reportUnknownArgumentType] # a configured organization + lambda: self._core_generate_model_id(_model_name, _litellm_params), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the configured params + ), + }, declared_id=declared_id, duplicate_ids=duplicate_ids, ) @@ -11520,6 +11581,8 @@ class PythonCore(DeprecatedRouterMembers): ) self._core_pattern_router.remove_deployment(model_id) + if model_id in self._core_provider_default_deployment_ids: + self._core_provider_default_deployment_ids.remove(model_id) for team_id in list(self._core_team_pattern_routers.keys()): team_pattern_router = self._core_team_pattern_routers[team_id] team_pattern_router.remove_deployment(model_id) @@ -11642,6 +11705,10 @@ class PythonCore(DeprecatedRouterMembers): ) return None else: + # The failed edit raises, but the deployment it replaced keeps serving. + self._restore_deployment_after_failed_upsert( + previous_deployment=_deployment_on_router, model_id=_deployment_model_id + ) raise e def _restore_deployment_after_failed_upsert(self, previous_deployment: Deployment | None, model_id: str) -> None: @@ -13176,6 +13243,28 @@ class PythonCore(DeprecatedRouterMembers): apply_remaining_usage_headers(additional_headers, remaining_usage) return response + def _assign_model_list(self, model_list: list[Mapping[str, object]]) -> None: # mutable-ok: kept as assigned + """ + ``router.model_list = [...]`` (deprecated): the list is swapped as given and reindexed, so a + replaced deployment's id, name and team entries stop finding the new list's entries. The + registry version moves once. + """ + with self._deployment_index_lock: + self._core_model_list = model_list + self.model_id_to_deployment_index_map = { + str(model_id): idx + for idx, entry in enumerate(model_list) + if (model_id := (as_str_mapping(entry.get("model_info")) or {}).get("id")) is not None + } + self._build_model_name_index(model_list) # pyright: ignore[reportUnknownMemberType] # the untyped index builder + self._core_model_names = { + str(entry["model_name"]) + for entry in model_list + if entry.get("model_name") and (as_str_mapping(entry.get("model_info")) or {}).get("id") is not None + } + self._access_groups_cache = None + self._invalidate_model_group_info_cache() + def _build_model_name_index(self, model_list: list) -> None: """ Build model_name -> deployment indices mapping for O(1) lookups. @@ -13188,6 +13277,10 @@ class PythonCore(DeprecatedRouterMembers): self._core_team_public_model_names = frozenset() for idx, model in enumerate(model_list): + if (as_str_mapping(model.get("model_info")) or {}).get("id") is None: # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # an entry of the untyped list + # An entry without an id (only ``router.model_list = [...]`` keeps one as given) is + # never served, as before that assignment reindexed: routing keys on the id. + continue model_name = model.get("model_name") if model_name: if model_name not in self._core_model_name_to_deployment_indices: @@ -17235,11 +17328,24 @@ class PythonCore(DeprecatedRouterMembers): self._core_prompt_caching_pin_cache.in_memory_cache.flush_cache() def reset(self): - ## clean up on close - self._host.set_callback_list("success_callback", []) - self._host.set_callback_list("_async_success_callback", []) - self._host.set_callback_list("failure_callback", []) - self._host.set_callback_list("_async_failure_callback", []) + ## clean up on close: drop this router's own success/failure callbacks, never another's + slack_alerting_loggers: Final = self._slack_alerting_loggers() + owned: Final = ( # pyright: ignore[reportUnknownVariableType] # the router's callback objects + *self._callback_selectors(), + *self._override_selectors.values(), + *(self.optional_callbacks or ()), # pyright: ignore[reportUnknownMemberType] # the router's checks + *slack_alerting_loggers, + *self._adaptive_router_hooks(), + ) + for name in OUTCOME_CALLBACK_LISTS: + self._host.set_callback_list( + name, + without_router_callbacks( + self._host.callback_list(name), + owned, # pyright: ignore[reportUnknownArgumentType] # see above + method_owners=(self, *slack_alerting_loggers), + ), + ) self.retry_policy = None self.flush_cache() diff --git a/litellm/router_rust_callbacks.py b/litellm/router_rust_callbacks.py index 7f1a09e0ff4..3fd94d89c3a 100644 --- a/litellm/router_rust_callbacks.py +++ b/litellm/router_rust_callbacks.py @@ -28,9 +28,14 @@ from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_strategy.least_busy import LeastBusyLoggingHandler from litellm.router_utils.global_callback_holders import ( + OUTCOME_CALLBACK_LISTS, + adaptive_routers_on_host, live_check_owners, + on_host, routers_holding_global_callbacks, routers_holding_global_callbacks_lock, + sync_adaptive_response_header_hook, + without_router_callbacks, ) from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, @@ -279,7 +284,7 @@ class OptionalChecks: if self.optional_callbacks is None or not any(type(cb) is callback_cls for cb in self.optional_callbacks): return self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls] - others: Final = tuple(owner for owner in live_check_owners if id(owner) != id(self._owner)) + others: Final = tuple(owner for owner in on_host(live_check_owners, self._host) if id(owner) != id(self._owner)) if any(any(type(cb) is callback_cls for cb in (owner.optional_callbacks or ())) for owner in others): return callbacks: Final = _callbacks(self._host) @@ -426,7 +431,7 @@ class OptionalChecks: (*dropped, *(self.optional_callbacks or ()), *slack, *adaptive), method_owners=(self._owner, *slack) ) with routers_holding_global_callbacks_lock: - survivors: Final = tuple(routers_holding_global_callbacks) + survivors: Final = on_host(tuple(routers_holding_global_callbacks), self._host) candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors) # pyright: ignore[reportPrivateUsage] # the holder protocol for candidate in self._host.callbacks.with_custom_logger_keys(candidates, vacated): register_router_selector(self._host, candidate) @@ -521,6 +526,22 @@ class RustOptionalChecks: selectors.callback_selectors(), (*selectors.callback_selectors(), *selectors.override_selectors()) ) + def _release_outcome_callbacks(self) -> None: + """``reset``'s callback half: this core's own entries leave the host's success and failure + lists; every other entry stays.""" + checks: Final = self._opt + slack: Final = checks.slack_alerting_loggers + owned: Final = ( + *self._sel.callback_selectors(), + *self._sel.override_selectors(), + *(checks.optional_callbacks or ()), + *slack, + *checks.own_adaptive_hooks(), + ) + host: Final = checks._host # pyright: ignore[reportPrivateUsage] # the checks' host + for name in OUTCOME_CALLBACK_LISTS: + host.set_callback_list(name, without_router_callbacks(host.callback_list(name), owned, (self, *slack))) + def _retire_callbacks(self) -> None: """``_core_discard``'s callback half: retire the routing state, release the callbacks.""" live_check_owners.discard(self) @@ -528,6 +549,9 @@ class RustOptionalChecks: self._opt.retire_state(selectors) self._opt.release(selectors) self._sel.clear() + # The shared adaptive response header hook leaves with the host's last adaptive router. + host: Final = self._opt._host # pyright: ignore[reportPrivateUsage] # the checks' host + sync_adaptive_response_header_hook(host.callbacks, adaptive_routers_on_host(host)) async def _answer_callback_filters( self, step: Mapping[str, object], request: Mapping[str, object], messages: object diff --git a/litellm/router_rust_core.py b/litellm/router_rust_core.py index 24169395303..468fc2f69cd 100644 --- a/litellm/router_rust_core.py +++ b/litellm/router_rust_core.py @@ -63,6 +63,7 @@ from litellm.router_api import ( ) from litellm.router_attempt import for_requested_model, request_from_call from litellm.router_config import RouterConfig, RouterDynamicConfig, detached +from litellm.router_core import detached_request from litellm.router_python_host import PythonHost from litellm.router_rust_access import caller_fields, selection_refusal from litellm.router_rust_alerting import initialize_alerting @@ -528,8 +529,8 @@ class Resolved: @classmethod def of(cls, config: RouterConfig, dynamic: RouterDynamicConfig) -> "Resolved": get: Final = config.get - # The process list is the router's own, as in the Python router: default_fallbacks are - # appended to it in place. + # The process list is the router's own, as in the Python router; default_fallbacks extend + # a new list, never the process-wide ``litellm.fallbacks``. configured: Final[list[object] | None] = ( # mutable-ok: the router's live list, as Python keeps it list(_as_list(get("fallbacks"))) or (dynamic.fallbacks if dynamic.fallbacks else None) # pyright: ignore[reportAssignmentType] # the process list as given ) @@ -539,10 +540,10 @@ class Resolved: raise TypeError( f"default_fallbacks must be a list of model names, got '{default_fallbacks}'. Use ['{default_fallbacks}']" ) - if default_fallbacks is not None and configured is not None: - configured.append({"*": default_fallbacks}) fallbacks: Final[list[object] | None] = ( # mutable-ok: see above - [{"*": default_fallbacks}] if default_fallbacks is not None and configured is None else configured + configured + if default_fallbacks is None + else [*(configured if configured is not None else ()), {"*": default_fallbacks}] ) content_policy: Final = list(_as_list(get("content_policy_fallbacks"))) or ( list(dynamic.content_policy_fallbacks) if dynamic.content_policy_fallbacks else None @@ -1083,7 +1084,7 @@ class RustCore( self._handle_default_ids: tuple[str, ...] = () # the provider default ids the handle was built with self._hosted.max_parallel_requests_lookup = self._max_parallel_requests_client self._hosted._core_routing_plugins = list(resolved.routing_plugins) # pyright: ignore[reportPrivateUsage] # the core fills its own host registry - self._handle: RouterCoreHandle = self._new_handle(list(self._hosted._core_model_list)) # pyright: ignore[reportPrivateUsage] # the core reads its own host registry + self._handle: RouterCoreHandle = self._new_handle(self._hosted.served_deployments()) self._join_callback_holders() self._checks: Final = OptionalChecks(self, host, self._cache, self._hosted, ctx, config.arguments) self._hosted.checks = self._checks @@ -1345,7 +1346,7 @@ class RustCore( def _rebuild(self) -> None: """A new Rust core for changed static settings, over the same deployments.""" old: Final = self._handle - self._handle = self._new_handle(list(self._hosted._core_model_list)) # pyright: ignore[reportPrivateUsage] # see __init__ + self._handle = self._new_handle(self._hosted.served_deployments()) old.retire() # The new handle builds its indexes from the deployments: written ones are handed again. self._index_sent = None @@ -1357,7 +1358,7 @@ class RustCore( self._tag_filtering_models = None handle: Final[object] = self.__dict__.get("_handle") if isinstance(handle, RouterCoreHandle): - _ = handle.apply({"replace": _plain(list(self._hosted._core_model_list))}) # pyright: ignore[reportPrivateUsage] # see __init__ + _ = handle.apply({"replace": _plain(self._hosted.served_deployments())}) # A wildcard deployment added an id the cooldown check skips. if self._provider_default_ids() != self._handle_default_ids: self._rebuild() @@ -1723,10 +1724,9 @@ class RustCore( self._checks.prompt_caching_pin_cache.in_memory_cache.flush_cache() def reset(self) -> None: - """Empty the host's success and failure callback lists (every entry, not only this core's), - drop the retry policy and flush the caches.""" - for name in ("success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"): - self._host.set_callback_list(name, []) + """Drop this core's own entries from the host's success and failure callback lists (other + code's callbacks stay), drop the retry policy and flush the caches.""" + self._release_outcome_callbacks() self._plain["retry_policy"] = None self._set_retry_policies({"retry_policy": None}) self.flush_cache() @@ -1753,6 +1753,11 @@ class RustCore( if name in PLAIN_SETTING_ATTRIBUTES or name in SETTING_FIELDS or name in ASSIGNABLE_ALIASES: _ = self.apply(ChangeSet(assign={name: value})) elif name == "model_list": + from litellm.router_deprecated import ( + _note, # pyright: ignore[reportPrivateUsage] # the deprecated write notes its use like Python's + ) + + _note("model_list", "registry().all()") self._assign_model_list(_as_list(value)) elif name in _HOSTED_STATE: # A deprecated attribute write of the registry state: it lands where this core keeps it. @@ -1810,7 +1815,31 @@ class RustCore( ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: """Pick a deployment for ``model`` without calling it (``pick``), list the deployments a pick would choose among (``healthy``), or pick one for a pass-through route - (``pass_through``, which picks with the sync strategies, as Python does).""" + (``pass_through``, which picks with the sync strategies, as Python does). The caller's + ``request_kwargs`` are left unchanged: selection works on a copy.""" + return await self._select_in_place( + model, + None if request_kwargs is None else detached_request(request_kwargs), + kind=kind, + specific_deployment=specific_deployment, + messages=messages, + input=input, + health_check_probe=health_check_probe, + ) + + async def _select_in_place( + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: pinned and stamped in place + *, + kind: str = "pick", + specific_deployment: bool | None = None, + messages: object = None, + input: object = None, + health_check_probe: bool = False, + **_: object, # kwargs-ok: the RouterCore select options this core ignores + ) -> Mapping[str, object] | Sequence[Mapping[str, object]]: + """``select`` on the request the core itself routes, which keeps what selection pins on it.""" if kind not in ("pick", "healthy", "pass_through"): raise ValueError(f"unknown select kind {kind!r}") request: Final = request_kwargs if request_kwargs is not None else {} diff --git a/litellm/router_rust_endpoints.py b/litellm/router_rust_endpoints.py index d27e85f514f..d63a9d93436 100644 --- a/litellm/router_rust_endpoints.py +++ b/litellm/router_rust_endpoints.py @@ -1331,7 +1331,7 @@ class RustEndpoints: assert isinstance(scheduler, Scheduler) async def healthy_deployments() -> Sequence[object]: - healthy: Final = await self._core.select(model, dict(request), kind="healthy") + healthy: Final = await self._core._select_in_place(model, dict(request), kind="healthy") # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core return list(healthy) if isinstance(healthy, Sequence) else [healthy] await scheduler.wait_for_turn( @@ -1584,7 +1584,7 @@ class RustEndpoints: metadata_variable_name: Final = get_router_metadata_variable_name(function_name="aretrieve_batch") if model is not None: specific: Final = kwargs.pop("specific_deployment", None) - healthy = await self._core.select( # rebind-ok: rebound as the Python router rebinds it + healthy = await self._core._select_in_place( # pyright: ignore[reportPrivateUsage] # rebind-ok: rebound as the Python router rebinds it model, dict(kwargs), kind="healthy", specific_deployment=bool(specific) or None ) else: @@ -1674,7 +1674,7 @@ class RustEndpoints: ) from litellm.types.utils import LlmProviders - healthy: Final = await self._core.select( + healthy: Final = await self._core._select_in_place( # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core requested, dict(kwargs), kind="healthy", specific_deployment=bool(specific) or None ) deployments: Final = list(healthy) if isinstance(healthy, Sequence) else [healthy] diff --git a/litellm/router_rust_members.py b/litellm/router_rust_members.py index 1aff51513ad..ebf92b94908 100644 --- a/litellm/router_rust_members.py +++ b/litellm/router_rust_members.py @@ -37,7 +37,7 @@ class RustRegistryMembers: return self._registry_state.add_deployment(deployment) def set_model_list(self, model_list: Sequence[object]) -> None: - """Replace every deployment (``set_model_list``); aliases and wildcard routes stay.""" + """Replace every deployment (``set_model_list``) with its wildcard routes and names; aliases stay.""" self._registry_state.set_model_list(model_list) @property @@ -46,11 +46,9 @@ class RustRegistryMembers: return self._registry_state._core_model_list # pyright: ignore[reportPrivateUsage] # the core reads its own host registry def _assign_model_list(self, model_list: Sequence[object]) -> None: - """``router.model_list = [...]``: the list is swapped as given, without reindexing, and the + """``router.model_list = [...]``: the list is swapped as given and reindexed, and the registry version moves once.""" - self._registry_state._core_model_list = model_list # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] # the list as assigned, as the core keeps it - self._registry_state._registry_version += 1 # pyright: ignore[reportPrivateUsage] # see above - self._registry_state._on_change() # pyright: ignore[reportPrivateUsage] # see above + self._registry_state.assign_model_list(model_list) @staticmethod def deployment_is_active_for_environment(deployment: Deployment) -> bool: diff --git a/litellm/router_rust_prerouting.py b/litellm/router_rust_prerouting.py index e57fa2b60ad..bd86c93456a 100644 --- a/litellm/router_rust_prerouting.py +++ b/litellm/router_rust_prerouting.py @@ -484,9 +484,10 @@ class PreRoutingStrategies: def _sync_adaptive_router_hooks(self) -> None: """One post-call hook per registered adaptive router, plus the header hook while any exists.""" - from litellm.router_strategy.adaptive_router.hooks import ( - AdaptiveRouterPostCallHook, - AdaptiveRouterResponseHeaderHook, + from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook + from litellm.router_utils.global_callback_holders import ( + adaptive_routers_on_host, + sync_adaptive_response_header_hook, ) callbacks: Final = self._host.callbacks @@ -494,8 +495,7 @@ class PreRoutingStrategies: callbacks.remove_callback_from_all_lists(callback) for tagged in chain.from_iterable(self.adaptive.values()): callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy)) - if self.adaptive: - callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook()) + sync_adaptive_response_header_hook(callbacks, bool(self.adaptive) or adaptive_routers_on_host(self._host)) # -- the request side -------------------------------------------------------------------------- diff --git a/litellm/router_rust_prompt_management.py b/litellm/router_rust_prompt_management.py index 3fead8698cb..730f2c458fb 100644 --- a/litellm/router_rust_prompt_management.py +++ b/litellm/router_rust_prompt_management.py @@ -71,7 +71,7 @@ async def prompt_management_acompletion( caller_timeout: Final = kwargs.get("timeout") or kwargs.get("request_timeout") caller_stream_timeout: Final = kwargs.get("stream_timeout") specific_deployment: Final = kwargs.pop("specific_deployment", None) - picked: Final = await core.select( + picked: Final = await core._select_in_place( # pyright: ignore[reportPrivateUsage] # the core's own selection, which pins onto the request model, kwargs, # pyright: ignore[reportUnknownArgumentType] # the request kwargs kind="pick", diff --git a/litellm/router_rust_registry.py b/litellm/router_rust_registry.py index fb341fc5ad0..6a9b4fac2d7 100644 --- a/litellm/router_rust_registry.py +++ b/litellm/router_rust_registry.py @@ -67,6 +67,7 @@ from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.responses.bridge_check import keep_registered_mode from litellm.router_api import RouterRegistry, deployment_id +from litellm.router_core import organization_deployment_id from litellm.router_rust_model_info import HostModelInfo from litellm.router_rust_shaping import ROUTER_CLIENTS_TARGET from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model @@ -632,6 +633,10 @@ class HostRegistry(HostModelInfo): self._core_team_model_to_deployment_indices = {} self._core_team_pattern_routers = {} self._core_team_public_model_names = frozenset() + # The replaced list's wildcard routes and names go with it. + self._core_pattern_router = PatternMatchRouter() + self._core_provider_default_deployment_ids.clear() + self._core_deployment_names.clear() if self.strategies is not None: self.strategies.reset() self._provider_unresolved_deployments = () @@ -666,10 +671,27 @@ class HostRegistry(HostModelInfo): if "id" not in info_dict: info_dict["id"] = deployment_id(str(model_name), litellm_params) organization = litellm_params.get("organization") - organizations: Sequence[object] = organization if isinstance(organization, list) else (None,) - for org in organizations: - if org is not None: + if isinstance(organization, list): + # Each organization's deployment has its own id (``organization_deployment_id``). + for position, org in enumerate(tuple(organization)): # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # configured organizations litellm_params["organization"] = org + self._create_deployment( + model, + str(model_name), + litellm_params, + { + **info_dict, + "id": organization_deployment_id( + declared_id, + position, + org, + lambda: deployment_id(str(model_name), litellm_params), + ), + }, + declared_id=declared_id, + duplicate_ids=duplicate_ids, + ) + else: self._create_deployment( model, str(model_name), @@ -865,6 +887,40 @@ class HostRegistry(HostModelInfo): vertex_credentials=credentials, # pyright: ignore[reportArgumentType] # the configured credentials ) + def served_deployments(self) -> list[UntypedEntry]: # mutable-ok: a fresh list for the Rust handle + """The deployments the Rust handle routes over: every entry with an id. Only + ``router.model_list = [...]`` keeps an entry without one, and that entry is never served.""" + return [entry for entry in self._core_model_list if _model_info_of(entry).get("id") is not None] + + def assign_model_list(self, model_list: Sequence[object]) -> None: + """``router.model_list = [...]`` (deprecated, ``_assign_model_list``): the list is swapped as + given and reindexed, and the registry version moves once.""" + entries: Final = [_LITELLM_PARAMS_ADAPTER.validate_python(entry) for entry in model_list] + with self._deployment_index_lock: + self._core_model_list = model_list # pyright: ignore[reportAttributeAccessIssue] # the list as assigned, as the core keeps it + self.model_id_to_deployment_index_map = { + str(model_id): idx + for idx, entry in enumerate(entries) + if (model_id := _model_info_of(entry).get("id")) is not None + } + self._core_model_name_to_deployment_indices = {} + self._core_team_model_to_deployment_indices = {} + self._core_team_public_model_names = frozenset() + # An entry without an id is kept as given but never served, as on the Python core. + served: Final = [ + (idx, entry) for idx, entry in enumerate(entries) if _model_info_of(entry).get("id") is not None + ] + for idx, entry in served: + model_name = entry.get("model_name") + if model_name: + self._core_model_name_to_deployment_indices.setdefault(str(model_name), []).append(idx) + self._update_team_model_index(entry, idx) + self._core_model_names = {str(entry["model_name"]) for _, entry in served if entry.get("model_name")} + self._access_groups_cache = None + self._invalidate_model_group_info_cache() + self._registry_version += 1 + self._on_change() + def _update_team_model_index(self, model: Mapping[str, object], idx: int) -> None: info: Final = _model_info_of(model) team_id: Final = info.get("team_id") @@ -915,6 +971,8 @@ class HostRegistry(HostModelInfo): public_model_name for _, public_model_name in self._core_team_model_to_deployment_indices ) self._core_pattern_router.remove_deployment(model_id) + if model_id in self._core_provider_default_deployment_ids: + self._core_provider_default_deployment_ids.remove(model_id) for team_id in list(self._core_team_pattern_routers): team_router = self._core_team_pattern_routers[team_id] team_router.remove_deployment(model_id) @@ -972,6 +1030,8 @@ class HostRegistry(HostModelInfo): self.strategies.upserted(deployment, on_router) except Exception as e: if not self.ignore_invalid_deployments: + # The failed edit raises, but the deployment it replaced keeps serving. + self._restore_deployment_after_failed_upsert(on_router, model_id) raise verbose_router_logger.warning( "Error upserting deployment %s (id=%s): %s. Dropping it and continuing with other deployments.", diff --git a/litellm/router_utils/global_callback_holders.py b/litellm/router_utils/global_callback_holders.py index 53a7c20cff3..cd837842d2b 100644 --- a/litellm/router_utils/global_callback_holders.py +++ b/litellm/router_utils/global_callback_holders.py @@ -5,8 +5,9 @@ serves them. A router that retires hands the checks it vacates in its host's ``callbacks`` list to a surviving router's same-keyed check (``routers_holding_global_callbacks``), and a router that drops a check at runtime keeps it registered while another live router still has one of that type -(``live_check_owners``). Both rules look at every router in the process, so the sets live here, -outside any one router core. +(``live_check_owners``). The sets hold every router in the process, whatever core serves it, so +they live here, outside any one router core; both rules only ever consult the routers on the same +host (``on_host``), since another host's lists are not this host's. A member of ``routers_holding_global_callbacks`` serves ``_global_callback_candidates()``, ``_core_adaptive_routers`` and ``_sync_adaptive_router_hooks()``; a member of ``live_check_owners`` @@ -15,8 +16,9 @@ serves ``optional_callbacks``. import threading import weakref -from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Final, Protocol +from collections.abc import Iterable, Mapping, Sequence +from types import MethodType +from typing import TYPE_CHECKING, Final, Protocol, TypeVar if TYPE_CHECKING: from litellm.integrations.custom_logger import CustomLogger @@ -40,14 +42,82 @@ class CheckOwner(Protocol): def optional_callbacks(self) -> "Sequence[object] | None": ... +_Router: Final = TypeVar("_Router") + routers_holding_global_callbacks: Final["weakref.WeakSet[GlobalCallbackHolder]"] = weakref.WeakSet() routers_holding_global_callbacks_lock: Final = threading.Lock() live_check_owners: Final["weakref.WeakSet[CheckOwner]"] = weakref.WeakSet() +OUTCOME_CALLBACK_LISTS: Final = ( + "success_callback", + "_async_success_callback", + "failure_callback", + "_async_failure_callback", +) + + +def without_router_callbacks( + callbacks: Iterable[object], owned: Iterable[object], method_owners: Iterable[object] +) -> list[object]: # mutable-ok: a fresh callback list for the host + """``callbacks`` without a router's own entries: the objects in ``owned`` and the bound methods of + ``method_owners``. Everything other code registered stays.""" + owned_ids: Final = frozenset(map(id, owned)) + owner_ids: Final = frozenset(map(id, method_owners)) + return [ + callback + for callback in callbacks + if id(callback) not in owned_ids + and not (isinstance(callback, MethodType) and id(callback.__self__) in owner_ids) + ] + + +def on_host(routers: Iterable[_Router], host: object) -> tuple[_Router, ...]: + """The routers among ``routers`` built on ``host``: a retiring router hands its vacated checks + only to these, and a dropped check stays registered only for these.""" + return tuple(owner for owner in routers if getattr(owner, "_host", None) is host) + + +class _HeaderHookCallbacks(Protocol): + """The host callback registration the adaptive response header hook goes through.""" + + def get_custom_loggers_for_type( + self, callback_type: "type[CustomLogger]" + ) -> "list[CustomLogger]": ... # mutable-ok: the manager's own return type + + def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None: ... + + def add_litellm_callback(self, callback: "CustomLogger") -> None: ... + + +def adaptive_routers_on_host(host: object) -> bool: + """Whether a live router built on ``host`` (request-scoped ones included) has an adaptive router.""" + return any(getattr(owner, "_core_adaptive_routers", None) for owner in on_host(tuple(live_check_owners), host)) + + +def sync_adaptive_response_header_hook(callbacks: _HeaderHookCallbacks, wanted: bool) -> None: + """ + Exactly one ``AdaptiveRouterResponseHeaderHook`` on the host while ``wanted`` (some router on it has + an adaptive router), none otherwise: the hook is stateless and shared by every router on the host, + so it is added once, never per sync, and leaves with the last adaptive router. + """ + from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterResponseHeaderHook + + present: Final = callbacks.get_custom_loggers_for_type(AdaptiveRouterResponseHeaderHook) + for extra in present[1 if wanted else 0 :]: + callbacks.remove_callback_from_all_lists(extra) + if wanted and not present: + callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook()) + + __all__ = [ + "OUTCOME_CALLBACK_LISTS", "CheckOwner", "GlobalCallbackHolder", + "adaptive_routers_on_host", "live_check_owners", + "on_host", "routers_holding_global_callbacks", "routers_holding_global_callbacks_lock", + "sync_adaptive_response_header_hook", + "without_router_callbacks", ] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index da1d99957a7..8d583a0ffb1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4238,12 +4238,19 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"}) +# A deployment's declared context window is its own: written under the shared backend key it would +# outlive the router and narrow every other router's deployment of the same model. +DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS: Final[frozenset[str]] = frozenset( + {"max_input_tokens", "max_output_tokens", "max_tokens"} +) + SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__) - frozenset(CustomPricingLiteLLMParams.model_fields) - DEPLOYMENT_SCOPED_PRICING_FIELDS - DEPLOYMENT_SCOPED_CAPABILITY_FIELDS - DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS + - DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS ) diff --git a/tests/unit/test_router/core/construction_precall_helpers.py b/tests/unit/test_router/core/construction_precall_helpers.py index c83bd37d692..0cf519d06d3 100644 --- a/tests/unit/test_router/core/construction_precall_helpers.py +++ b/tests/unit/test_router/core/construction_precall_helpers.py @@ -59,7 +59,8 @@ def windowed( group: str, deployment_id: str, max_input_tokens: int, **params: object ) -> dict[str, object]: # mutable-ok: a model_list entry """A deployment declaring ``max_input_tokens`` under a model name no other test uses: a deployment's - model_info is written into the process-wide cost map under its model name and id.""" + model_info is written into the process-wide cost map under its id (and its schema fields, though not + its token limits, under its model name).""" model_info: Final = {"max_input_tokens": max_input_tokens, **dict(params.pop("model_info", None) or {})} # pyright: ignore[reportArgumentType] return deployment( group, diff --git a/tests/unit/test_router/core/test_construction_precall_characterization.py b/tests/unit/test_router/core/test_construction_precall_characterization.py index 6256f1021f5..a9c07d65552 100644 --- a/tests/unit/test_router/core/test_construction_precall_characterization.py +++ b/tests/unit/test_router/core/test_construction_precall_characterization.py @@ -320,16 +320,14 @@ def test_default_fallbacks_given_as_a_string_raise_type_error(make_core: MakeCor _one(make_core, default_fallbacks="b") -def test_default_fallbacks_are_appended_to_the_process_fallbacks_list_itself( - make_core: MakeCore, host: ScriptedHost -) -> None: - """Oddity: with no ``fallbacks`` argument the router adopts the process list object and appends the - star entry to it, so the process setting is changed too.""" +def test_default_fallbacks_extend_a_copy_of_the_process_fallbacks_list(make_core: MakeCore, host: ScriptedHost) -> None: + """With no ``fallbacks`` argument the router starts from the process list, but appends the star + entry to its own copy, so the process setting is left alone.""" process_fallbacks: Final[list[object]] = [{"g": ["e"]}] host.set_dynamic(fallbacks=process_fallbacks) core: Final = _one(make_core, default_fallbacks=["c"]) assert core.settings().fallbacks == [{"g": ["e"]}, {"*": ["c"]}] - assert process_fallbacks == [{"g": ["e"]}, {"*": ["c"]}] + assert process_fallbacks == [{"g": ["e"]}] # --- validation -------------------------------------------------------------------------------- @@ -873,11 +871,22 @@ def test_runtime_optional_pre_call_checks_none_changes_nothing(make_core: MakeCo assert host_callback_names(host)["callbacks"] == ["PromptCachingDeploymentCheck"] -def test_runtime_removal_keeps_the_host_callback_while_another_live_router_uses_that_check( +def test_runtime_removal_keeps_the_host_callback_while_another_live_router_on_the_host_uses_that_check( make_core: MakeCore, host: ScriptedHost ) -> None: - """Oddity: the check is left in this host's list because some other live router (here on another - host) still has one of that type.""" + collect_other_routers() + other: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) + core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) + core.apply(ChangeSet(settings={"optional_pre_call_checks": []})) + assert core.optional_callbacks == [] # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + assert all(check in host.lists["callbacks"] for check in other.optional_callbacks) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] + other.retire() + + +def test_runtime_removal_drops_the_host_callback_though_a_router_on_another_host_uses_that_check( + make_core: MakeCore, host: ScriptedHost +) -> None: + """Another host's router keeps its own check on its own host; it does not hold this host's.""" collect_other_routers() other_host: Final = CallbackHost() from tests.unit.test_router.core.scripted_host import python_core @@ -888,7 +897,8 @@ def test_runtime_removal_keeps_the_host_callback_while_another_live_router_uses_ core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) core.apply(ChangeSet(settings={"optional_pre_call_checks": []})) assert core.optional_callbacks == [] # pyright: ignore[reportAttributeAccessIssue] - assert host_callback_names(host)["callbacks"] == ["PromptCachingDeploymentCheck"] + assert host_callback_names(host)["callbacks"] == [] + assert host_callback_names(other_host)["callbacks"] == ["PromptCachingDeploymentCheck"] other.retire() @@ -946,23 +956,35 @@ def test_retiring_one_router_keeps_the_callbacks_of_another_on_the_same_host( assert all(callback.__self__ is second for callback in host.lists["_async_success_callback"]) # pyright: ignore[reportAttributeAccessIssue] -def test_retire_hands_a_vacated_check_to_another_live_router_on_a_different_host( +def test_retire_never_hands_a_vacated_check_to_a_live_router_on_a_different_host( make_core: MakeCore, host: ScriptedHost ) -> None: - """Oddity: retire re-registers the same-keyed check of any other live router in the process into - this host's list, even one built on a different host.""" + """Another host's router keeps its check on its own host; this host's list is left empty.""" collect_other_routers() from tests.unit.test_router.core.scripted_host import python_core + other_host: Final = CallbackHost() other: Final = python_core( - CallbackHost(), model_list=[deployment("g", "o")], optional_pre_call_checks=["prompt_caching"] + other_host, model_list=[deployment("g", "o")], optional_pre_call_checks=["prompt_caching"] ) core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) core.retire() - assert host.lists["callbacks"] == other.optional_callbacks + assert host.lists["callbacks"] == [] + assert other_host.lists["callbacks"] == other.optional_callbacks # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view other.retire() +def test_retire_hands_a_vacated_check_to_another_live_router_on_the_same_host( + make_core: MakeCore, host: ScriptedHost +) -> None: + collect_other_routers() + first: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) + second: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"]) + first.retire() + assert host.lists["callbacks"] == second.optional_callbacks # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + second.retire() + + # --- observe ----------------------------------------------------------------------------- @@ -1131,23 +1153,42 @@ def test_flush_cache_drops_the_response_cache_routing_state_and_prompt_caching_p assert host_callback_names(host) == before -def test_reset_empties_every_success_and_failure_list_and_the_retry_policy( +def test_reset_drops_the_routers_own_success_and_failure_callbacks_and_the_retry_policy( make_core: MakeCore, host: ScriptedHost ) -> None: - """Note: reset clears the host's success/failure lists outright, foreign callbacks included, but - leaves ``callbacks`` and ``input_callback`` alone.""" + """reset drops only this router's entries from the host's success/failure lists (other code's + callbacks stay) and leaves ``callbacks`` and ``input_callback`` alone.""" core: Final = _one(make_core, retry_policy={"RateLimitErrorRetries": 3}, routing_strategy="least-busy") host.lists["success_callback"].append("foreign") + host.lists["_async_failure_callback"].append("foreign-failure") core.health().store.set_cache("k", 1) core.reset() # pyright: ignore[reportAttributeAccessIssue] assert {name: callbacks for name, callbacks in host_callback_names(host).items() if callbacks} == { "callbacks": ["LeastBusyLoggingHandler"], "input_callback": ["LeastBusyLoggingHandler"], + "success_callback": ["str"], + "_async_failure_callback": ["str"], } + assert (host.lists["success_callback"], host.lists["_async_failure_callback"]) == (["foreign"], ["foreign-failure"]) assert core.retry_policy is None # pyright: ignore[reportAttributeAccessIssue] assert core.health().store.get_cache("k") is None +def test_resetting_one_router_keeps_another_routers_success_and_failure_callbacks( + make_core: MakeCore, host: ScriptedHost +) -> None: + collect_other_routers() + first: Final = _one(make_core) + before_first: Final = {name: list(callbacks) for name, callbacks in host.lists.items()} + second: Final = _one(make_core) + first.reset() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + for name in ("success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"): + remaining = host.lists.get(name, []) + assert all(getattr(callback, "__self__", None) is not first for callback in remaining) + theirs = [callback for callback in before_first.get(name, []) if getattr(callback, "__self__", None) is first] + assert len(remaining) == len(theirs) and all(callback.__self__ is second for callback in remaining) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + + # --- attribute writes ---------------------------------------------------------------------- diff --git a/tests/unit/test_router/core/test_construction_precall_characterization_precall.py b/tests/unit/test_router/core/test_construction_precall_characterization_precall.py index bb8d3d30774..568675509d1 100644 --- a/tests/unit/test_router/core/test_construction_precall_characterization_precall.py +++ b/tests/unit/test_router/core/test_construction_precall_characterization_precall.py @@ -291,20 +291,29 @@ async def test_an_anthropic_system_block_is_counted_against_the_context_window(m @pytest.mark.asyncio -async def test_a_deployment_context_window_is_written_into_the_process_cost_map(make_core: MakeCore) -> None: - """Oddity: a deployment's declared ``max_input_tokens`` replaces the process-wide cost-map entry of its - model, so it outlives the router and narrows another router's deployment of the same model.""" +async def test_a_deployment_context_window_stays_out_of_the_models_shared_cost_map_entry( + make_core: MakeCore, +) -> None: + """A deployment's declared ``max_input_tokens`` is its own: the process-wide cost-map entry of its + model keeps the built-in window, so neither a retired nor a live router narrows another router's + deployment of the same model.""" + builtin_window: Final = litellm.model_cost[NO_JSON_MODE_MODEL.removeprefix("openai/")]["max_input_tokens"] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped in the checker's view first: Final = make_core( - model_list=[deployment("g", "declares-five", model=NO_JSON_MODE_MODEL, model_info={"max_input_tokens": 5})] + model_list=[deployment("g", "declares-five", model=NO_JSON_MODE_MODEL, model_info={"max_input_tokens": 5})], + enable_pre_call_checks=True, ) - first.retire() other: Final = python_core( CallbackHost(), model_list=[deployment("g", "declares-nothing", model=NO_JSON_MODE_MODEL)], enable_pre_call_checks=True, ) + for key in (NO_JSON_MODE_MODEL, NO_JSON_MODE_MODEL.removeprefix("openai/")): + assert litellm.model_cost.get(key, {}).get("max_input_tokens") in (None, builtin_window) # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view + assert (await other.select("g", messages=LONG))["model_info"]["id"] == "declares-nothing" with pytest.raises(litellm.ContextWindowExceededError, match="Max Input Tokens=5"): - await other.select("g", messages=SHORT) + await first.select("g", messages=LONG) # pyright: ignore[reportCallIssue] # select takes messages; the checker reads an overload + first.retire() + assert (await other.select("g", messages=LONG))["model_info"]["id"] == "declares-nothing" other.retire() diff --git a/tests/unit/test_router/core/test_deprecated_members.py b/tests/unit/test_router/core/test_deprecated_members.py index b73dda78ecd..cb4a5cd7f6f 100644 --- a/tests/unit/test_router/core/test_deprecated_members.py +++ b/tests/unit/test_router/core/test_deprecated_members.py @@ -663,3 +663,14 @@ def test_cooldown_time_reads_back_as_given(make_core: MakeCore) -> None: assert updated == 3 and type(updated) is int default: Final = as_router(make_core(model_list=[deployment("gpt", "a")])).settings().cooldown_time assert default == 5 and type(default) is int + + +@pytest.mark.asyncio +async def test_the_old_healthy_listing_takes_the_retry_skip_list_off_the_callers_request(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")])) + request: Final[dict[str, object]] = {"_retry_skipped_deployment_ids": ("a",)} + with pytest.warns(DeprecationWarning, match=r"Router\.async_get_healthy_deployments is deprecated"): + healthy: Final = await router.async_get_healthy_deployments("gpt", request) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the old untyped dicts + assert isinstance(healthy, list) + assert [picked_id(entry) for entry in healthy] == ["b"] # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # see above + assert request == {} diff --git a/tests/unit/test_router/core/test_endpoints.py b/tests/unit/test_router/core/test_endpoints.py index ce56ae403ba..fe622df67c1 100644 --- a/tests/unit/test_router/core/test_endpoints.py +++ b/tests/unit/test_router/core/test_endpoints.py @@ -392,6 +392,17 @@ async def test_unknown_model_group_raises_without_calling_the_host( assert host.calls == [] +@pytest.mark.parametrize("endpoint", ["acompletion", "completion"]) +async def test_an_entry_assigned_without_an_id_is_never_called(make_core: MakeCore, host: ScriptedHost, endpoint: str): + """``model_list = [...]`` keeps an entry without ``model_info.id`` as given, but never routes to it.""" + core: Final = _one_group(make_core) + with pytest.warns(DeprecationWarning, match=r"Router\.model_list is deprecated"): # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view + core.model_list = [{"model_name": "b", "litellm_params": {"model": "openai/b"}}] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable + with pytest.raises(litellm.BadRequestError, match="model=b"): + await core.call(endpoint, {"model": "b", "messages": MESSAGES}) + assert host.calls == [] + + @pytest.mark.parametrize("op", ["no_such_endpoint", "acompletions", ""]) async def test_call_with_an_unknown_op_raises_attribute_error_without_calling_the_host( make_core: MakeCore, host: ScriptedHost, op: str diff --git a/tests/unit/test_router/core/test_fallbacks_characterization.py b/tests/unit/test_router/core/test_fallbacks_characterization.py index 333d9d73078..3454656c25e 100644 --- a/tests/unit/test_router/core/test_fallbacks_characterization.py +++ b/tests/unit/test_router/core/test_fallbacks_characterization.py @@ -370,16 +370,18 @@ async def test_order_tiers_run_before_a_plain_list_or_the_default_chain( # --- default fallbacks --------------------------------------------------------------------------- -def test_default_fallbacks_are_appended_to_the_process_fallbacks_list_in_place( +def test_default_fallbacks_extend_the_process_fallbacks_without_changing_the_process_list( make_core: MakeCore, host: ScriptedHost ) -> None: process_fallbacks: list[object] = [{"a": ["b"]}] host.set_dynamic(fallbacks=process_fallbacks) core = build(make_core, default_fallbacks=["c"]) + other = build(make_core, default_fallbacks=["d"]) assert core.settings().fallbacks == [{"a": ["b"]}, {"*": ["c"]}] - assert process_fallbacks == [{"a": ["b"]}, {"*": ["c"]}] + assert other.settings().fallbacks == [{"a": ["b"]}, {"*": ["d"]}] # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view + assert process_fallbacks == [{"a": ["b"]}] def test_default_fallbacks_leave_the_constructor_fallbacks_list_unchanged(make_core: MakeCore) -> None: diff --git a/tests/unit/test_router/core/test_gaps_l11100_strategy.py b/tests/unit/test_router/core/test_gaps_l11100_strategy.py index 2459c577b9c..d08daf5a737 100644 --- a/tests/unit/test_router/core/test_gaps_l11100_strategy.py +++ b/tests/unit/test_router/core/test_gaps_l11100_strategy.py @@ -300,7 +300,7 @@ def test_a_failed_edit_rolls_back_to_the_serving_router_even_above_a_tightened_l assert restored.litellm_params.tags is None -def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_leaves_the_router_removed( +def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_restores_the_router( make_core: MakeCore, ) -> None: limit: Final = {"value": 2} @@ -319,8 +319,8 @@ def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_leaves_the_ ChangeSet(upsert=[Deployment(**complexity_marker("two", "m2", config=HEURISTIC_V2, tags=["edited"]))]) # pyright: ignore[reportArgumentType] ) assert str(raised.value) == HEURISTIC_V2_LIMIT_MESSAGE - assert model_ids(core) == ["s", "b", "m1"] - assert registered_tags(core, "complexity") == {"one": [()]} + assert model_ids(core) == ["s", "b", "m1", "m2"] + assert registered_tags(core, "complexity") == {"one": [()], "two": [()]} # --- the adaptive router ------------------------------------------------------------------------------- @@ -373,10 +373,19 @@ def test_an_adaptive_router_registers_one_post_call_hook_and_a_header_hook( post_call, header = _hook_counts(host) assert post_call == 1 - assert header >= 1 + assert header == 1 -def test_deleting_an_adaptive_router_releases_it_and_its_post_call_hook( +def test_resyncing_an_adaptive_router_keeps_a_single_header_hook(make_core: MakeCore, host: ScriptedHost) -> None: + core: Final = make_core(model_list=_adaptive_list()) + + core.apply(ChangeSet(upsert=[Deployment(**adaptive_marker("ar", "ar1", ["y"]))])) # pyright: ignore[reportArgumentType] + core.apply(ChangeSet(upsert=[Deployment(**adaptive_marker("ar", "ar1", ["x"]))])) # pyright: ignore[reportArgumentType] + + assert _hook_counts(host) == (1, 1) + + +def test_deleting_an_adaptive_router_releases_it_its_post_call_hook_and_the_header_hook( make_core: MakeCore, host: ScriptedHost ) -> None: core: Final = make_core(model_list=_adaptive_list()) @@ -384,7 +393,30 @@ def test_deleting_an_adaptive_router_releases_it_and_its_post_call_hook( core.apply(ChangeSet(delete=["ar1"])) assert registered_tags(core, "adaptive") == {} - assert _hook_counts(host)[0] == 0 + assert _hook_counts(host) == (0, 0) + + +def test_retiring_the_last_adaptive_router_on_the_host_removes_the_header_hook( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=_adaptive_list()) + + core.retire() + + assert _hook_counts(host) == (0, 0) + + +def test_retiring_one_adaptive_router_keeps_the_header_hook_for_another_on_the_host( + make_core: MakeCore, host: ScriptedHost +) -> None: + first: Final = make_core(model_list=_adaptive_list()) + second: Final = make_core(model_list=_adaptive_list()) + + first.retire() + + assert _hook_counts(host)[1] == 1 + second.retire() + assert _hook_counts(host) == (0, 0) def test_a_complexity_router_with_adaptive_enabled_also_registers_an_adaptive_router(make_core: MakeCore) -> None: diff --git a/tests/unit/test_router/core/test_gaps_l14000_selection.py b/tests/unit/test_router/core/test_gaps_l14000_selection.py index 8cff6bcdbe5..700d1f83283 100644 --- a/tests/unit/test_router/core/test_gaps_l14000_selection.py +++ b/tests/unit/test_router/core/test_gaps_l14000_selection.py @@ -315,6 +315,8 @@ async def test_an_access_group_only_caller_reaches_any_deployment_by_its_litellm # --- tier litellm_params ------------------------------------------------------------------------- +# ``select`` works on a copy of its caller's request; these read what selection pins and stamps on the +# request the router routes (and re-routes on retries), through the cores' in-place selection. def _pinned(**tier_params: object) -> dict[str, object]: # mutable-ok: tiers # kwargs-ok: tier litellm_params @@ -331,7 +333,7 @@ async def test_a_tier_param_no_deployment_supports_is_dropped(make_core: MakeCor ] ) request: Final[dict[str, object]] = {} - chosen: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + chosen: Final = await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] assert request["temperature"] == 0.2 assert "logit_bias" not in request @@ -347,7 +349,7 @@ async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore) ] ) request: Final[dict[str, object]] = {"allowed_openai_params": ["logit_bias"]} - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] assert request["logit_bias"] == {"1": 2} @@ -360,7 +362,7 @@ async def test_a_tier_reasoning_effort_removes_the_callers_other_effort_carriers "reasoning": {"effort": "high"}, "max_completion_tokens": 100, } - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] assert request["reasoning_effort"] == "low" assert "thinking" not in request assert request["output_config"] == {"format": "json"} @@ -372,7 +374,7 @@ async def test_a_tier_reasoning_effort_removes_the_callers_other_effort_carriers async def test_a_tier_without_an_output_ceiling_leaves_the_callers_ceiling(make_core: MakeCore) -> None: core: Final = make_core(model_list=_tiered(tiers=_pinned(temperature=0.1))) request: Final[dict[str, object]] = {"max_completion_tokens": 100} - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] assert request["temperature"] == 0.1 assert request["max_completion_tokens"] == 100 assert "max_tokens" not in request @@ -384,12 +386,12 @@ async def test_a_tier_without_an_output_ceiling_leaves_the_callers_ceiling(make_ async def test_a_later_pick_restores_the_callers_ceiling_a_tier_pin_replaced(make_core: MakeCore) -> None: core: Final = make_core(model_list=_tiered(tiers=_pinned(max_tokens=7))) request: Final[dict[str, object]] = {"max_completion_tokens": 100} - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] assert request["max_tokens"] == 7 assert "max_completion_tokens" not in request assert _metadata(request)["_client_output_ceiling"] == {"max_completion_tokens": 100} - chosen: Final = await core.select("smart", request, messages=HARD_MESSAGES) # pyright: ignore[reportCallIssue] + chosen: Final = await core._select_in_place("smart", request, messages=HARD_MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] assert request["max_completion_tokens"] == 100 assert "max_tokens" not in request @@ -401,13 +403,13 @@ async def test_a_second_pick_leaves_the_first_picks_forwarded_marker_params_untr # next attempt, so the tracking key is dropped while the params themselves stay behind. core: Final = make_core(model_list=_tiered(temperature=0.3)) request: Final[dict[str, object]] = {} - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] forwarded: Final = request["_alias_marker_forwarded_params"] assert isinstance(forwarded, tuple) assert "temperature" in forwarded assert request["temperature"] == 0.3 - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] assert "_alias_marker_forwarded_params" not in request assert request["temperature"] == 0.3 @@ -419,7 +421,7 @@ async def test_a_second_pick_leaves_the_first_picks_forwarded_marker_params_untr async def test_the_routing_decision_keeps_its_prompt_signals_by_default(make_core: MakeCore) -> None: core: Final = make_core(model_list=_tiered()) request: Final[dict[str, object]] = {} - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] decision: Final = _metadata(request)["routing_decision"] assert isinstance(decision, Mapping) assert "signals" in decision @@ -429,7 +431,7 @@ async def test_the_routing_decision_keeps_its_prompt_signals_by_default(make_cor async def test_redacted_message_logging_drops_the_prompt_quoting_routing_decision_fields(make_core: MakeCore) -> None: core: Final = make_core(model_list=_tiered()) request: Final[dict[str, object]] = {"standard_callback_dynamic_params": {"turn_off_message_logging": True}} - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] decision: Final = _metadata(request)["routing_decision"] assert isinstance(decision, Mapping) assert "signals" not in decision @@ -440,7 +442,7 @@ async def test_redacted_message_logging_drops_the_prompt_quoting_routing_decisio async def test_routing_metadata_goes_to_litellm_metadata_when_the_request_has_it(make_core: MakeCore) -> None: core: Final = make_core(model_list=_tiered()) request: Final[dict[str, object]] = {"litellm_metadata": {}} - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] assert "routing_decision" in _metadata(request, "litellm_metadata") assert "metadata" not in request diff --git a/tests/unit/test_router/core/test_registry_characterization_construction.py b/tests/unit/test_router/core/test_registry_characterization_construction.py index 7b80f6c3b71..bc312024040 100644 --- a/tests/unit/test_router/core/test_registry_characterization_construction.py +++ b/tests/unit/test_router/core/test_registry_characterization_construction.py @@ -131,8 +131,27 @@ def test_a_list_of_organizations_fans_out_into_one_deployment_per_organization(m entries = core.registry().all() assert [entry["litellm_params"]["organization"] for entry in entries] == ["org-1", "org-2"] - assert len(set(ids(entries))) == 1 + assert len(set(ids(entries))) == 2 assert core.registry().index("model_name") == {"gpt": [0, 1]} + for entry in entries: # pyright: ignore[reportAny] # untyped in the checker's view + found = core.registry().deployment(entry["model_info"]["id"]) # pyright: ignore[reportAny] # untyped in the checker's view + assert found is not None + assert found.litellm_params.organization == entry["litellm_params"]["organization"] + + +def test_organization_deployments_get_the_same_ids_on_every_build(make_core: MakeCore) -> None: + first = make_core(model_list=[_bare("gpt", "openai/x", organization=["org-1", "org-2"])]) + second = make_core(model_list=[_bare("gpt", "openai/x", organization=["org-1", "org-2"])]) + + assert ids(first.registry().all()) == ids(second.registry().all()) + + +def test_a_declared_id_stays_on_the_first_organization_and_is_suffixed_on_the_rest(make_core: MakeCore) -> None: + entry = _bare("gpt", "openai/x", organization=["org-1", "org-2"]) + entry["model_info"] = {"id": "declared"} + core = make_core(model_list=[entry]) + + assert ids(core.registry().all()) == ["declared", "declared-org-2"] def test_deployment_names_list_each_deployments_litellm_model(make_core: MakeCore) -> None: diff --git a/tests/unit/test_router/core/test_registry_characterization_lookups.py b/tests/unit/test_router/core/test_registry_characterization_lookups.py index 19d58ff10d4..be87138fe2c 100644 --- a/tests/unit/test_router/core/test_registry_characterization_lookups.py +++ b/tests/unit/test_router/core/test_registry_characterization_lookups.py @@ -4,6 +4,7 @@ access groups; blocked and unhealthy groups; credentials lookups. """ import time +from typing import Final import pytest @@ -668,15 +669,31 @@ def test_the_deprecated_generate_model_id_is_the_registry_id_hash(make_core: Mak assert generated == deployment_id("g", {"model": "x"}) -def test_assigning_model_list_swaps_the_list_without_reindexing(make_core: MakeCore) -> None: - """Odd but current: the id index still maps ``a`` to position 0, which now holds the new entry.""" +def test_assigning_model_list_swaps_and_reindexes_the_list(make_core: MakeCore) -> None: + """The replaced deployment's id finds nothing; the new entry is found by its own id and name.""" core = make_core(model_list=[deployment("gpt", "a")]) version = core.registry().version - core.model_list = [deployment("n", "n")] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable + with pytest.warns(DeprecationWarning, match=r"Router\.model_list is deprecated"): # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view + core.model_list = [deployment("n", "n")] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable assert core.registry().version == version + 1 - assert core.registry().names(kind="ids") == ["a"] - found = core.registry().deployment("a") + assert core.registry().names(kind="ids") == ["n"] + assert core.registry().deployment("a") is None + found = core.registry().deployment("n") assert found is not None assert found.model_name == "n" + assert ids(core.registry().deployments("n")) == ["n"] + assert core.registry().deployments("gpt") == [] + + +def test_an_entry_assigned_without_an_id_is_not_served(make_core: MakeCore) -> None: + """``model_list = [...]`` keeps entries as given; one without ``model_info.id`` is never routed to.""" + core = make_core(model_list=[deployment("gpt", "a")]) + entry: Final = {"model_name": "b", "litellm_params": {"model": "openai/gpt-4o-mini"}} + + with pytest.warns(DeprecationWarning, match=r"Router\.model_list is deprecated"): # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view + core.model_list = [entry] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable + + assert core.registry().deployments("b") == [] + assert core.registry().names(kind="ids") == [] diff --git a/tests/unit/test_router/core/test_registry_characterization_writes.py b/tests/unit/test_router/core/test_registry_characterization_writes.py index 0558a16fe9d..7f3d6ad5ae4 100644 --- a/tests/unit/test_router/core/test_registry_characterization_writes.py +++ b/tests/unit/test_router/core/test_registry_characterization_writes.py @@ -154,14 +154,20 @@ def test_an_upsert_that_empties_a_group_drops_the_group_name(make_core: MakeCore assert core.registry().deployments("emb") == [] -def test_a_failing_upsert_of_an_existing_deployment_raises_and_leaves_it_removed(make_core: MakeCore) -> None: - """Odd but current: without ignore_invalid_deployments the old deployment is popped and not restored.""" +def test_a_failing_upsert_of_an_existing_deployment_raises_and_restores_the_previous_deployment( + make_core: MakeCore, +) -> None: + """Without ignore_invalid_deployments the failed edit raises, and the deployment it would have + replaced keeps serving its previous configuration.""" core = _core(make_core) with pytest.raises(litellm.BadRequestError): core.apply(ChangeSet(upsert=[typed("gpt", "a", model="notaprovider/x")])) - assert core.registry().names(kind="ids") == ["b", "e", "w", "t1"] + restored = core.registry().deployment("a") + assert restored is not None + assert restored.litellm_params.model == "openai/a" + assert core.registry().names(kind="ids") == ["b", "e", "w", "t1", "a"] def test_a_failing_upsert_under_ignore_invalid_deployments_restores_the_previous_deployment( @@ -241,12 +247,19 @@ def test_deleting_a_wildcard_deployment_stops_it_serving_matching_names(make_cor assert core.registry().deployments("openai/gpt-x") == [] -def test_deleting_a_wildcard_deployment_keeps_it_among_provider_default_deployment_ids(make_core: MakeCore) -> None: - """Odd but current: the provider default id list is append-only.""" +def test_deleting_a_wildcard_deployment_drops_it_from_the_provider_default_deployment_ids(make_core: MakeCore) -> None: core = _core(make_core) core.apply(ChangeSet(delete=["w"])) + assert core.settings().provider_default_deployment_ids == [] + + +def test_deleting_a_plain_deployment_keeps_the_wildcards_provider_default_deployment_id(make_core: MakeCore) -> None: + core = _core(make_core) + + core.apply(ChangeSet(delete=["a"])) + assert core.settings().provider_default_deployment_ids == ["w"] @@ -300,23 +313,31 @@ def test_set_model_list_leaves_the_given_list_unchanged(make_core: MakeCore) -> assert given == [{"model_name": "new", "litellm_params": {"model": "openai/n", "api_key": "k"}}] -def test_set_model_list_keeps_the_old_wildcard_routes(make_core: MakeCore) -> None: - """Odd but current: the wildcard pattern router is not reset, so a replaced wildcard still matches.""" +def test_set_model_list_drops_the_old_wildcard_routes(make_core: MakeCore) -> None: core = _core(make_core) core.set_model_list([deployment("new", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core - assert names_and_ids(core.registry().deployments("openai/gpt-x")) == [("openai/gpt-x", "w")] - assert core.settings().provider_default_deployment_ids == ["w"] + assert core.registry().deployments("openai/gpt-x") == [] + assert core.settings().provider_default_deployment_ids == [] -def test_set_model_list_appends_to_the_deployment_names(make_core: MakeCore) -> None: - """Odd but current: deployment_names keep the replaced deployments' models.""" +def test_set_model_list_serves_the_new_lists_wildcard_routes(make_core: MakeCore) -> None: + core = _core(make_core) + + core.set_model_list([deployment("anthropic/*", "w2", model="anthropic/*")]) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # public on the core + + assert core.registry().deployments("openai/gpt-x") == [] + assert names_and_ids(core.registry().deployments("anthropic/claude-x")) == [("anthropic/claude-x", "w2")] + assert core.settings().provider_default_deployment_ids == ["w2"] + + +def test_set_model_list_replaces_the_deployment_names(make_core: MakeCore) -> None: core = make_core(model_list=[deployment("gpt", "a")]) core.set_model_list([deployment("new", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core - assert core.registry().names(kind="deployment_names") == ["openai/a", "openai/n"] + assert core.registry().names(kind="deployment_names") == ["openai/n"] @pytest.mark.asyncio diff --git a/tests/unit/test_router/core/test_selection_characterization_groups.py b/tests/unit/test_router/core/test_selection_characterization_groups.py index 465118e1c65..476d57f873f 100644 --- a/tests/unit/test_router/core/test_selection_characterization_groups.py +++ b/tests/unit/test_router/core/test_selection_characterization_groups.py @@ -676,7 +676,7 @@ async def test_retry_skipped_deployment_ids_given_as_a_list_are_ignored(make_cor @pytest.mark.asyncio -async def test_select_pops_the_narrowing_keys_from_the_request(make_core: MakeCore) -> None: +async def test_select_honors_the_narrowing_keys_but_leaves_them_on_the_callers_request(make_core: MakeCore) -> None: core: Final = make_core(model_list=list(THREE)) request: Final[dict[str, object]] = { "_target_order": None, @@ -684,6 +684,22 @@ async def test_select_pops_the_narrowing_keys_from_the_request(make_core: MakeCo "_retry_skipped_deployment_ids": ("b",), } assert ids_of(await core.select("g", request, kind="healthy")) == ["c"] + assert request == { + "_target_order": None, + "_excluded_deployment_ids": ["a"], + "_retry_skipped_deployment_ids": ("b",), + } + + +@pytest.mark.asyncio # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # untyped in the checker's view +async def test_in_place_selection_pops_the_narrowing_keys_from_the_routed_request(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(THREE)) + request: Final[dict[str, object]] = { + "_target_order": None, + "_excluded_deployment_ids": ["a"], + "_retry_skipped_deployment_ids": ("b",), + } + assert ids_of(await core._select_in_place("g", request, kind="healthy")) == ["c"] # pyright: ignore[reportAttributeAccessIssue, reportUnknownArgumentType, reportUnknownMemberType] assert request == {} diff --git a/tests/unit/test_router/core/test_selection_characterization_prerouting.py b/tests/unit/test_router/core/test_selection_characterization_prerouting.py index 317fbf4be46..cbe5ebdc2d2 100644 --- a/tests/unit/test_router/core/test_selection_characterization_prerouting.py +++ b/tests/unit/test_router/core/test_selection_characterization_prerouting.py @@ -78,6 +78,19 @@ def _metadata(request: Mapping[str, object]) -> Mapping[str, object]: return metadata +async def _routed_metadata( + core: RouterCore, + host: ScriptedHost, + model: str, + messages: list[dict[str, str]], + **request: object, # kwargs-ok: the request's own keyword payload +) -> Mapping[str, object]: + """The metadata the routed call carried: ``select`` leaves its caller's request alone, so the + routing stamps are read where they land, on the request the router sends.""" + await core.acompletion(model=model, messages=messages, **request) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + return _metadata(host.calls[-1].kwargs) + + # --- the complexity router ------------------------------------------------------------------------ @@ -92,11 +105,11 @@ async def test_a_complexity_router_picks_the_tier_deployment_for_the_prompt( @pytest.mark.asyncio -async def test_a_complexity_router_pick_stamps_the_routing_decision_on_the_request(make_core: MakeCore) -> None: +async def test_a_complexity_router_pick_stamps_the_routing_decision_on_the_routed_request( + make_core: MakeCore, host: ScriptedHost +) -> None: core: Final = make_core(model_list=_tiered()) - request: Final[dict[str, object]] = {} - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] - metadata: Final = _metadata(request) + metadata: Final = await _routed_metadata(core, host, "smart", MESSAGES) decision: Final = metadata["routing_decision"] assert isinstance(decision, Mapping) assert decision["router_model_name"] == "smart" @@ -121,11 +134,11 @@ async def test_completion_through_a_complexity_router_reports_the_router_name_as @pytest.mark.asyncio async def test_an_alias_to_a_complexity_router_routes_through_it(make_core: MakeCore, host: ScriptedHost) -> None: core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"}) - request: Final[dict[str, object]] = {} - chosen: Final = await core.select("al", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + chosen: Final = await core.select("al", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType] assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] - assert _metadata(request)["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue] - await core.acompletion(model="al", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] + metadata: Final = await _routed_metadata(core, host, "al", MESSAGES) + assert metadata["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue] + assert host.calls[-1].deployment_id == "s" assert host.calls[-1].model_group == "al" @@ -137,15 +150,14 @@ async def test_a_complexity_router_takes_precedence_over_plain_deployments_of_it @pytest.mark.asyncio async def test_a_complexity_router_tier_naming_an_unknown_model_falls_back_to_the_default_model( - make_core: MakeCore, + make_core: MakeCore, host: ScriptedHost ) -> None: core: Final = make_core( model_list=[deployment("big", "b"), _complexity_router(tiers={**TIERS, "SIMPLE": "missing"})] ) - request: Final[dict[str, object]] = {} - chosen: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] + chosen: Final = await core.select("smart", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType] assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue] - decision: Final = _metadata(request)["routing_decision"] + decision: Final = (await _routed_metadata(core, host, "smart", MESSAGES))["routing_decision"] assert isinstance(decision, Mapping) assert decision["cause"] == "health_default_fallback" assert decision["routed_model"] == "big" @@ -282,13 +294,25 @@ async def test_tier_litellm_params_override_the_callers_params_on_the_call( @pytest.mark.asyncio -async def test_tier_litellm_params_are_pinned_onto_the_request_by_select(make_core: MakeCore) -> None: +async def test_select_leaves_the_callers_request_without_the_tier_pins(make_core: MakeCore) -> None: core: Final = make_core(model_list=_tiered(tiers=PINNED_TIERS)) request: Final[dict[str, object]] = {"temperature": 0.9, "max_completion_tokens": 100} - await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] - assert request["temperature"] == 0.1 - assert request["max_tokens"] == 7 - assert "max_completion_tokens" not in request + picked: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType] + assert picked["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] # a deployment dict + assert request == {"temperature": 0.9, "max_completion_tokens": 100} + + +@pytest.mark.asyncio # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # untyped in the checker's view +async def test_select_leaves_the_callers_request_and_metadata_without_marker_params_or_stamps( + make_core: MakeCore, +) -> None: + core: Final = make_core(model_list=_tiered(temperature=0.3)) + metadata: Final[dict[str, object]] = {"tags": ["t"]} + request: Final[dict[str, object]] = {"metadata": metadata, "specific_deployment": False} + await core.select("smart", request, messages=HARD_MESSAGES) # pyright: ignore[reportCallIssue] + assert request == {"metadata": {"tags": ["t"]}, "specific_deployment": False} + assert request["metadata"] is metadata + assert metadata == {"tags": ["t"]} @pytest.mark.asyncio @@ -351,13 +375,16 @@ def _session_request( @pytest.mark.asyncio -async def test_a_subagent_request_follows_the_router_its_session_started_on(make_core: MakeCore) -> None: +async def test_a_subagent_request_follows_the_router_its_session_started_on( + make_core: MakeCore, host: ScriptedHost +) -> None: core: Final = make_core(model_list=_tiered()) await core.select("smart", _session_request(), messages=MESSAGES) # pyright: ignore[reportCallIssue] - subagent: Final = _session_request(agent_id="agent-1") - chosen: Final = await core.select("big", subagent, messages=MESSAGES) # pyright: ignore[reportCallIssue] + chosen: Final = await core.select("big", _session_request(agent_id="agent-1"), messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType] assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] - assert _metadata(subagent)["model_group"] == "smart" + metadata: Final = await _routed_metadata(core, host, "big", MESSAGES, **_session_request(agent_id="agent-1")) + assert host.calls[-1].deployment_id == "s" + assert metadata["model_group"] == "smart" @pytest.mark.asyncio @@ -428,24 +455,31 @@ async def test_a_routing_plugin_narrows_the_candidates_to_the_models_it_keeps(ma @pytest.mark.asyncio -async def test_routing_plugin_signals_and_narrowed_candidates_are_stamped_on_the_request(make_core: MakeCore) -> None: +async def test_routing_plugin_signals_and_narrowed_candidates_are_stamped_on_the_routed_request( + make_core: MakeCore, host: ScriptedHost +) -> None: core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/b"}))]) - request: Final[dict[str, object]] = {} - await core.select("g", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] - assert request == { - "metadata": { - "routing_plugin_signals": {"narrowing": {"kept": ["openai/b"]}}, - "_routing_plugin_candidate_models": ["openai/b"], - } - } + metadata: Final = await _routed_metadata(core, host, "g", MESSAGES) + assert metadata["routing_plugin_signals"] == {"narrowing": {"kept": ["openai/b"]}} + assert metadata["_routing_plugin_candidate_models"] == ["openai/b"] @pytest.mark.asyncio -async def test_a_routing_plugin_that_keeps_every_candidate_stamps_only_its_signals(make_core: MakeCore) -> None: +async def test_a_routing_plugin_that_keeps_every_candidate_stamps_only_its_signals( + make_core: MakeCore, host: ScriptedHost +) -> None: core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/a", "openai/b"}))]) + metadata: Final = await _routed_metadata(core, host, "g", MESSAGES) + assert metadata["routing_plugin_signals"] == {"narrowing": {"kept": ["openai/a", "openai/b"]}} + assert "_routing_plugin_candidate_models" not in metadata + + +@pytest.mark.asyncio # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # untyped in the checker's view +async def test_select_leaves_the_callers_request_without_routing_plugin_stamps(make_core: MakeCore) -> None: + core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/b"}))]) request: Final[dict[str, object]] = {} await core.select("g", request, messages=MESSAGES) # pyright: ignore[reportCallIssue] - assert request == {"metadata": {"routing_plugin_signals": {"narrowing": {"kept": ["openai/a", "openai/b"]}}}} + assert request == {} @pytest.mark.asyncio diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 91f0b2339c7..b92e6f105eb 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -2081,7 +2081,8 @@ def test_add_deployment_does_not_leak_custom_metadata_to_shared_backend_key(): def test_shared_backend_model_info_keeps_schema_fields_and_drops_the_rest(): """Unit test of the whitelist helper: cost-map schema fields survive, - custom pricing overrides and per-deployment metadata do not. + custom pricing overrides, per-deployment metadata and per-deployment token + limits do not. """ from litellm.types.utils import shared_backend_model_info @@ -2106,7 +2107,6 @@ def test_shared_backend_model_info_keeps_schema_fields_and_drops_the_rest(): assert filtered == { "mode": "chat", "litellm_provider": "openai", - "max_tokens": 128000, "supports_vision": True, "supported_endpoints": ["/v1/responses"], "use_openai_responses_path": True, @@ -3052,7 +3052,7 @@ def test_register_deployment_in_model_cost_writes_both_key_families(): """ A deployment contributes its full model_info under its unique id and the cost-map subset under the shared backend key, and the shared key must not - pick up the deployment's private metadata. + pick up the deployment's private metadata or its declared token limits. """ model_keys = { "both-families-id": copy.deepcopy(litellm.model_cost.get("both-families-id")), @@ -3068,7 +3068,8 @@ def test_register_deployment_in_model_cost_writes_both_key_families(): assert litellm.model_cost["both-families-id"]["max_input_tokens"] == 999 shared = litellm.model_cost["hosted_vllm/both-families-backend"] - assert shared["max_input_tokens"] == 999 + assert shared["litellm_provider"] == "hosted_vllm" + assert "max_input_tokens" not in shared assert "id" not in shared finally: _restore_model_cost_entries(model_keys)