fix(router): stop router state leaking across callers, routers and hosts

select works on a copy of the caller's request; default_fallbacks extend a
copy of litellm.fallbacks; deployment token limits stay off the shared
cost-map key; retire and runtime check removal only consult routers on the
same host; reset drops only the router's own success/failure callbacks; a
failed upsert restores the replaced deployment; set_model_list and delete
reset wildcard routes, provider default ids and deployment names;
model_list assignment reindexes and warns; organization-list deployments get
their own ids; the adaptive response header hook is kept to one per host and
removed with the last adaptive router. Both cores.
This commit is contained in:
narmstrong-berry 2026-11-02 02:52:28 +00:00
parent cfba1d3ac6
commit afd1e98692
26 changed files with 712 additions and 156 deletions

View file

@ -7,7 +7,7 @@ that reaches outside itself only through its host; it does not use ``PythonCore`
picked once, when the router is built, from its ``RouterConfig``.
"""
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
from litellm._logging import verbose_router_logger
@ -52,6 +52,39 @@ class RouterCore(Protocol):
def retire(self) -> None: ...
_REQUEST_METADATA_KEYS: Final = ("metadata", "litellm_metadata")
def detached_request(
request_kwargs: Mapping[str, object] | None,
) -> dict[str, object]: # mutable-ok: the selection cores take a dict
"""
A copy of a caller's request for ``select``: selection pops keys, pins tier params and stamps
routing metadata on the request it routes, and none of that may reach the caller's dict (or the
metadata dicts inside it).
"""
if request_kwargs is None:
return {}
return {
key: dict(value) if key in _REQUEST_METADATA_KEYS and isinstance(value, Mapping) else value # pyright: ignore[reportUnknownArgumentType] # a request value
for key, value in request_kwargs.items()
}
def organization_deployment_id(
declared_id: str | None, position: int, organization: object, generated: Callable[[], str]
) -> str:
"""
The id of one deployment a ``litellm_params.organization`` list fans out into: each gets its own.
A declared id stays on the first organization's deployment (so lookups by it keep working) and is
suffixed with the organization on the others; without one, ``generated`` hashes that
organization's own params.
"""
if declared_id is None:
return generated()
return declared_id if position == 0 else f"{declared_id}-{organization}"
def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore:
"""
The Rust core when the route catalog's router rule allows it and the Rust core serves this

View file

@ -141,7 +141,10 @@ class _DeprecatedAttribute(Generic[_T]):
def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None:
_note(self.name, self.replacement)
if self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES:
if self.name == "model_list":
# A whole new list is reindexed, so a replaced id never finds the new list's entries.
_as_router(obj)._assign_model_list(value) # pyright: ignore[reportArgumentType] # the assigned list
elif self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES:
_as_router(obj).apply(ChangeSet(assign={self.name: value}))
else:
setattr(obj, "_core_" + self.name, value)
@ -170,6 +173,15 @@ def _attr(replacement: str, read: Callable[["Router"], _T]) -> _DeprecatedAttrib
return _DeprecatedAttribute(replacement, read)
def _select_in_place(router: "Router") -> Callable[..., Coroutine[object, object, object]]:
"""
The core's selection on the caller's own ``request_kwargs``: the old members changed them in
place (for example taking ``_retry_skipped_deployment_ids`` off), where ``select`` works on a copy.
"""
core: Final[object] = router.__dict__.get("_core", router) # pyright: ignore[reportAny] # the facade keeps its core in __dict__
return core._select_in_place # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # every core has one
def _async_picker(router: "Router") -> Callable[..., Awaitable[object]]:
"""
The router's async pick (the old ``async_get_available_deployment`` attribute). A core that
@ -187,7 +199,7 @@ def _async_picker(router: "Router") -> Callable[..., Awaitable[object]]:
input: str | list | None = None, # mutable-ok: the old parameter type, unchanged
specific_deployment: bool | None = False,
) -> object:
return await router.select(
return await _select_in_place(router)(
model, request_kwargs, messages=messages, input=input, specific_deployment=specific_deployment
)
@ -809,7 +821,7 @@ class DeprecatedRouterMembers:
health_check_probe: bool = False,
) -> list[dict] | dict: # mutable-ok: the old return type, unchanged
_note("async_get_healthy_deployments", "select(model, request_kwargs, kind='healthy')")
return await _as_router(self).select(
return await _select_in_place(_as_router(self))( # pyright: ignore[reportReturnType] # the core's untyped deployment dicts
model,
request_kwargs,
kind="healthy",
@ -830,7 +842,7 @@ class DeprecatedRouterMembers:
):
_note("get_available_deployment_for_pass_through", "select(model, kind='pass_through')")
return _run_ready(
_as_router(self).select(
_select_in_place(_as_router(self))(
model,
request_kwargs,
kind="pass_through",

View file

@ -124,6 +124,7 @@ from litellm.litellm_core_utils.clientside_credential_handler import (
)
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
coerce_token_limit,
get_litellm_metadata_from_kwargs,
get_metadata_variable_name_from_kwargs,
@ -255,7 +256,11 @@ from litellm.router_api import (
)
from litellm.router_api import RoutingContext as RouterContext
from litellm.router_api import deployment_id as _deployment_id
from litellm.router_core import detached_request, organization_deployment_id
from litellm.router_deprecated import DeprecatedRouterMembers
from litellm.router_deprecated import (
_note as _note_deprecated_member, # pyright: ignore[reportPrivateUsage] # the deprecated write notes its use
)
from litellm.router_endpoint_table import (
ENDPOINTS,
SYNC_DISPATCHES,
@ -369,9 +374,14 @@ from litellm.router_utils.fallback_event_handlers import (
run_async_fallback,
)
from litellm.router_utils.global_callback_holders import (
OUTCOME_CALLBACK_LISTS,
adaptive_routers_on_host,
live_check_owners,
on_host,
routers_holding_global_callbacks,
routers_holding_global_callbacks_lock,
sync_adaptive_response_header_hook,
without_router_callbacks,
)
from litellm.router_utils.handle_error import (
async_raise_no_deployment_exception,
@ -1525,6 +1535,11 @@ class PythonCore(DeprecatedRouterMembers):
# Settings change only through a changeset, so every core serving this router sees it.
self.apply(ChangeSet(assign={name.removeprefix("_core_"): value}))
return
if name == "model_list" and "_core_model_list" in self.__dict__:
# ``core.model_list = [...]`` on the core itself: noted and reindexed as through the Router.
_note_deprecated_member("model_list", "registry().all()")
self._assign_model_list(value) # pyright: ignore[reportUnknownMemberType, reportArgumentType] # the assigned list, as given
return
if name in _SETTING_ATTRIBUTES:
self.__dict__.pop("_settings_snapshot", None)
elif name == "_core_model_list" and "_core_model_list" in self.__dict__:
@ -1589,12 +1604,37 @@ class PythonCore(DeprecatedRouterMembers):
"""
Pick a deployment for ``model`` without calling it (``pick``), list the deployments that
are healthy for the request (``healthy``), or pick one for a pass-through route
(``pass_through``).
(``pass_through``). The caller's ``request_kwargs`` are left unchanged: selection works on a copy.
"""
return await self._select_in_place( # pyright: ignore[reportUnknownVariableType] # the selection cores return untyped deployment dicts
model,
None if request_kwargs is None else detached_request(request_kwargs),
kind=kind,
messages=messages,
input=input,
specific_deployment=specific_deployment,
parent_otel_span=parent_otel_span,
health_check_probe=health_check_probe,
)
async def _select_in_place( # pyright: ignore[reportUnknownParameterType] # the selection cores return untyped deployment dicts
self,
model: str,
request_kwargs: dict[str, object] | None = None, # mutable-ok: pinned and stamped in place
*,
kind: SelectKind = "pick",
messages: list[dict[str, str]] | None = None, # mutable-ok: the selection cores take a list
input: str | list[object] | None = None, # mutable-ok: the selection cores take a list
specific_deployment: bool | None = False,
parent_otel_span: Span | None = None,
health_check_probe: bool = False,
) -> "list[UntypedDict] | UntypedDict": # mutable-ok: the core's type (list[dict] | dict), as select returns it
"""``select`` on the request the router itself routes, which keeps what selection pins on it."""
request: Final = request_kwargs if request_kwargs is not None else {}
if kind == "pick":
return await self._core_async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts
model=model,
request_kwargs=request_kwargs if request_kwargs is not None else {},
request_kwargs=request,
messages=messages,
input=input,
specific_deployment=specific_deployment,
@ -1602,7 +1642,7 @@ class PythonCore(DeprecatedRouterMembers):
if kind == "healthy":
return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts
model=model,
request_kwargs=request_kwargs if request_kwargs is not None else {},
request_kwargs=request,
messages=messages,
input=input,
specific_deployment=specific_deployment,
@ -2065,7 +2105,8 @@ class PythonCore(DeprecatedRouterMembers):
f"default_fallbacks must be a list of model names, got '{_fallbacks}'. Use ['{_fallbacks}']"
)
if self._core_fallbacks is not None:
self._core_fallbacks.append({"*": _fallbacks})
# A new list: the configured one may be the process-wide ``litellm.fallbacks``.
self._core_fallbacks = [*self._core_fallbacks, {"*": _fallbacks}] # pyright: ignore[reportUnknownMemberType] # the configured fallback entries
else:
self._core_fallbacks = [{"*": _fallbacks}]
@ -2224,6 +2265,8 @@ class PythonCore(DeprecatedRouterMembers):
live_check_owners.discard(self)
self.retire_routing_state()
self._release_global_callbacks()
# The shared adaptive response header hook leaves with the host's last adaptive router.
sync_adaptive_response_header_hook(self._host.callbacks, adaptive_routers_on_host(self._host))
def _release_global_callbacks(self) -> None:
with _routers_holding_global_callbacks_lock:
@ -2241,7 +2284,7 @@ class PythonCore(DeprecatedRouterMembers):
method_owners=(self, *slack_alerting_loggers),
)
with _routers_holding_global_callbacks_lock:
survivors: Final = tuple(_routers_holding_global_callbacks)
survivors: Final = on_host(tuple(_routers_holding_global_callbacks), self._host)
candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors)
for candidate in self._host.callbacks.with_custom_logger_keys(candidates, vacated_keys):
self._register_router_selector(candidate)
@ -3189,7 +3232,10 @@ class PythonCore(DeprecatedRouterMembers):
self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls]
if any(
router is not self and any(type(cb) is callback_cls for cb in (router.optional_callbacks or []))
for router in (*_live_routers, *(owner for owner in live_check_owners if not isinstance(owner, PythonCore)))
for router in on_host(
(*_live_routers, *(owner for owner in live_check_owners if not isinstance(owner, PythonCore))),
self._host,
)
):
return
for cb in tuple(self._host.callback_list("callbacks")):
@ -11009,7 +11055,6 @@ class PythonCore(DeprecatedRouterMembers):
changes, otherwise a released router keeps recording turns through its hook."""
from litellm.router_strategy.adaptive_router.hooks import (
AdaptiveRouterPostCallHook,
AdaptiveRouterResponseHeaderHook,
)
for callback in self._host.callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook):
@ -11017,8 +11062,9 @@ class PythonCore(DeprecatedRouterMembers):
for tagged_adaptive_routers in self._core_adaptive_routers.values():
for tagged in tagged_adaptive_routers:
self._host.callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy))
if self._core_adaptive_routers:
self._host.callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook())
sync_adaptive_response_header_hook(
self._host.callbacks, bool(self._core_adaptive_routers) or adaptive_routers_on_host(self._host)
)
def init_adaptive_router_deployment(self, deployment: Deployment) -> None:
"""
@ -11161,6 +11207,10 @@ class PythonCore(DeprecatedRouterMembers):
self._core_team_model_to_deployment_indices = {} # Reset the team_model index
self._core_team_pattern_routers = {}
self._core_team_public_model_names = frozenset()
# The replaced list's wildcard routes and names go with it.
self._core_pattern_router = PatternMatchRouter()
self._core_provider_default_deployment_ids.clear()
self._core_deployment_names.clear()
# Reset per-strategy router registries so hot-reload doesn't leave
# stale routers pointing at the old model_list.
self._core_quality_routers = {}
@ -11214,13 +11264,24 @@ class PythonCore(DeprecatedRouterMembers):
if _litellm_params.get("organization", None) is not None and isinstance(
_litellm_params["organization"], list
): # Addresses https://github.com/BerriAI/litellm/issues/3949
for org in _litellm_params["organization"]:
for position, org in enumerate(tuple(_litellm_params["organization"])): # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # configured organizations
_litellm_params["organization"] = org
# Each organization's deployment has its own id: a declared id stays on the
# first and is suffixed with the organization on the rest, a generated one is
# generated for that organization's params.
self._create_deployment(
deployment_info=model,
_model_name=_model_name,
_litellm_params=_litellm_params,
_model_info=_model_info,
_model_info={
**_model_info,
"id": organization_deployment_id(
declared_id,
position,
org, # pyright: ignore[reportUnknownArgumentType] # a configured organization
lambda: self._core_generate_model_id(_model_name, _litellm_params), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the configured params
),
},
declared_id=declared_id,
duplicate_ids=duplicate_ids,
)
@ -11520,6 +11581,8 @@ class PythonCore(DeprecatedRouterMembers):
)
self._core_pattern_router.remove_deployment(model_id)
if model_id in self._core_provider_default_deployment_ids:
self._core_provider_default_deployment_ids.remove(model_id)
for team_id in list(self._core_team_pattern_routers.keys()):
team_pattern_router = self._core_team_pattern_routers[team_id]
team_pattern_router.remove_deployment(model_id)
@ -11642,6 +11705,10 @@ class PythonCore(DeprecatedRouterMembers):
)
return None
else:
# The failed edit raises, but the deployment it replaced keeps serving.
self._restore_deployment_after_failed_upsert(
previous_deployment=_deployment_on_router, model_id=_deployment_model_id
)
raise e
def _restore_deployment_after_failed_upsert(self, previous_deployment: Deployment | None, model_id: str) -> None:
@ -13176,6 +13243,28 @@ class PythonCore(DeprecatedRouterMembers):
apply_remaining_usage_headers(additional_headers, remaining_usage)
return response
def _assign_model_list(self, model_list: list[Mapping[str, object]]) -> None: # mutable-ok: kept as assigned
"""
``router.model_list = [...]`` (deprecated): the list is swapped as given and reindexed, so a
replaced deployment's id, name and team entries stop finding the new list's entries. The
registry version moves once.
"""
with self._deployment_index_lock:
self._core_model_list = model_list
self.model_id_to_deployment_index_map = {
str(model_id): idx
for idx, entry in enumerate(model_list)
if (model_id := (as_str_mapping(entry.get("model_info")) or {}).get("id")) is not None
}
self._build_model_name_index(model_list) # pyright: ignore[reportUnknownMemberType] # the untyped index builder
self._core_model_names = {
str(entry["model_name"])
for entry in model_list
if entry.get("model_name") and (as_str_mapping(entry.get("model_info")) or {}).get("id") is not None
}
self._access_groups_cache = None
self._invalidate_model_group_info_cache()
def _build_model_name_index(self, model_list: list) -> None:
"""
Build model_name -> deployment indices mapping for O(1) lookups.
@ -13188,6 +13277,10 @@ class PythonCore(DeprecatedRouterMembers):
self._core_team_public_model_names = frozenset()
for idx, model in enumerate(model_list):
if (as_str_mapping(model.get("model_info")) or {}).get("id") is None: # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # an entry of the untyped list
# An entry without an id (only ``router.model_list = [...]`` keeps one as given) is
# never served, as before that assignment reindexed: routing keys on the id.
continue
model_name = model.get("model_name")
if model_name:
if model_name not in self._core_model_name_to_deployment_indices:
@ -17235,11 +17328,24 @@ class PythonCore(DeprecatedRouterMembers):
self._core_prompt_caching_pin_cache.in_memory_cache.flush_cache()
def reset(self):
## clean up on close
self._host.set_callback_list("success_callback", [])
self._host.set_callback_list("_async_success_callback", [])
self._host.set_callback_list("failure_callback", [])
self._host.set_callback_list("_async_failure_callback", [])
## clean up on close: drop this router's own success/failure callbacks, never another's
slack_alerting_loggers: Final = self._slack_alerting_loggers()
owned: Final = ( # pyright: ignore[reportUnknownVariableType] # the router's callback objects
*self._callback_selectors(),
*self._override_selectors.values(),
*(self.optional_callbacks or ()), # pyright: ignore[reportUnknownMemberType] # the router's checks
*slack_alerting_loggers,
*self._adaptive_router_hooks(),
)
for name in OUTCOME_CALLBACK_LISTS:
self._host.set_callback_list(
name,
without_router_callbacks(
self._host.callback_list(name),
owned, # pyright: ignore[reportUnknownArgumentType] # see above
method_owners=(self, *slack_alerting_loggers),
),
)
self.retry_policy = None
self.flush_cache()

View file

@ -28,9 +28,14 @@ from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
from litellm.router_utils.global_callback_holders import (
OUTCOME_CALLBACK_LISTS,
adaptive_routers_on_host,
live_check_owners,
on_host,
routers_holding_global_callbacks,
routers_holding_global_callbacks_lock,
sync_adaptive_response_header_hook,
without_router_callbacks,
)
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
DeploymentAffinityCheck,
@ -279,7 +284,7 @@ class OptionalChecks:
if self.optional_callbacks is None or not any(type(cb) is callback_cls for cb in self.optional_callbacks):
return
self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls]
others: Final = tuple(owner for owner in live_check_owners if id(owner) != id(self._owner))
others: Final = tuple(owner for owner in on_host(live_check_owners, self._host) if id(owner) != id(self._owner))
if any(any(type(cb) is callback_cls for cb in (owner.optional_callbacks or ())) for owner in others):
return
callbacks: Final = _callbacks(self._host)
@ -426,7 +431,7 @@ class OptionalChecks:
(*dropped, *(self.optional_callbacks or ()), *slack, *adaptive), method_owners=(self._owner, *slack)
)
with routers_holding_global_callbacks_lock:
survivors: Final = tuple(routers_holding_global_callbacks)
survivors: Final = on_host(tuple(routers_holding_global_callbacks), self._host)
candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors) # pyright: ignore[reportPrivateUsage] # the holder protocol
for candidate in self._host.callbacks.with_custom_logger_keys(candidates, vacated):
register_router_selector(self._host, candidate)
@ -521,6 +526,22 @@ class RustOptionalChecks:
selectors.callback_selectors(), (*selectors.callback_selectors(), *selectors.override_selectors())
)
def _release_outcome_callbacks(self) -> None:
"""``reset``'s callback half: this core's own entries leave the host's success and failure
lists; every other entry stays."""
checks: Final = self._opt
slack: Final = checks.slack_alerting_loggers
owned: Final = (
*self._sel.callback_selectors(),
*self._sel.override_selectors(),
*(checks.optional_callbacks or ()),
*slack,
*checks.own_adaptive_hooks(),
)
host: Final = checks._host # pyright: ignore[reportPrivateUsage] # the checks' host
for name in OUTCOME_CALLBACK_LISTS:
host.set_callback_list(name, without_router_callbacks(host.callback_list(name), owned, (self, *slack)))
def _retire_callbacks(self) -> None:
"""``_core_discard``'s callback half: retire the routing state, release the callbacks."""
live_check_owners.discard(self)
@ -528,6 +549,9 @@ class RustOptionalChecks:
self._opt.retire_state(selectors)
self._opt.release(selectors)
self._sel.clear()
# The shared adaptive response header hook leaves with the host's last adaptive router.
host: Final = self._opt._host # pyright: ignore[reportPrivateUsage] # the checks' host
sync_adaptive_response_header_hook(host.callbacks, adaptive_routers_on_host(host))
async def _answer_callback_filters(
self, step: Mapping[str, object], request: Mapping[str, object], messages: object

View file

@ -63,6 +63,7 @@ from litellm.router_api import (
)
from litellm.router_attempt import for_requested_model, request_from_call
from litellm.router_config import RouterConfig, RouterDynamicConfig, detached
from litellm.router_core import detached_request
from litellm.router_python_host import PythonHost
from litellm.router_rust_access import caller_fields, selection_refusal
from litellm.router_rust_alerting import initialize_alerting
@ -528,8 +529,8 @@ class Resolved:
@classmethod
def of(cls, config: RouterConfig, dynamic: RouterDynamicConfig) -> "Resolved":
get: Final = config.get
# The process list is the router's own, as in the Python router: default_fallbacks are
# appended to it in place.
# The process list is the router's own, as in the Python router; default_fallbacks extend
# a new list, never the process-wide ``litellm.fallbacks``.
configured: Final[list[object] | None] = ( # mutable-ok: the router's live list, as Python keeps it
list(_as_list(get("fallbacks"))) or (dynamic.fallbacks if dynamic.fallbacks else None) # pyright: ignore[reportAssignmentType] # the process list as given
)
@ -539,10 +540,10 @@ class Resolved:
raise TypeError(
f"default_fallbacks must be a list of model names, got '{default_fallbacks}'. Use ['{default_fallbacks}']"
)
if default_fallbacks is not None and configured is not None:
configured.append({"*": default_fallbacks})
fallbacks: Final[list[object] | None] = ( # mutable-ok: see above
[{"*": default_fallbacks}] if default_fallbacks is not None and configured is None else configured
configured
if default_fallbacks is None
else [*(configured if configured is not None else ()), {"*": default_fallbacks}]
)
content_policy: Final = list(_as_list(get("content_policy_fallbacks"))) or (
list(dynamic.content_policy_fallbacks) if dynamic.content_policy_fallbacks else None
@ -1083,7 +1084,7 @@ class RustCore(
self._handle_default_ids: tuple[str, ...] = () # the provider default ids the handle was built with
self._hosted.max_parallel_requests_lookup = self._max_parallel_requests_client
self._hosted._core_routing_plugins = list(resolved.routing_plugins) # pyright: ignore[reportPrivateUsage] # the core fills its own host registry
self._handle: RouterCoreHandle = self._new_handle(list(self._hosted._core_model_list)) # pyright: ignore[reportPrivateUsage] # the core reads its own host registry
self._handle: RouterCoreHandle = self._new_handle(self._hosted.served_deployments())
self._join_callback_holders()
self._checks: Final = OptionalChecks(self, host, self._cache, self._hosted, ctx, config.arguments)
self._hosted.checks = self._checks
@ -1345,7 +1346,7 @@ class RustCore(
def _rebuild(self) -> None:
"""A new Rust core for changed static settings, over the same deployments."""
old: Final = self._handle
self._handle = self._new_handle(list(self._hosted._core_model_list)) # pyright: ignore[reportPrivateUsage] # see __init__
self._handle = self._new_handle(self._hosted.served_deployments())
old.retire()
# The new handle builds its indexes from the deployments: written ones are handed again.
self._index_sent = None
@ -1357,7 +1358,7 @@ class RustCore(
self._tag_filtering_models = None
handle: Final[object] = self.__dict__.get("_handle")
if isinstance(handle, RouterCoreHandle):
_ = handle.apply({"replace": _plain(list(self._hosted._core_model_list))}) # pyright: ignore[reportPrivateUsage] # see __init__
_ = handle.apply({"replace": _plain(self._hosted.served_deployments())})
# A wildcard deployment added an id the cooldown check skips.
if self._provider_default_ids() != self._handle_default_ids:
self._rebuild()
@ -1723,10 +1724,9 @@ class RustCore(
self._checks.prompt_caching_pin_cache.in_memory_cache.flush_cache()
def reset(self) -> None:
"""Empty the host's success and failure callback lists (every entry, not only this core's),
drop the retry policy and flush the caches."""
for name in ("success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"):
self._host.set_callback_list(name, [])
"""Drop this core's own entries from the host's success and failure callback lists (other
code's callbacks stay), drop the retry policy and flush the caches."""
self._release_outcome_callbacks()
self._plain["retry_policy"] = None
self._set_retry_policies({"retry_policy": None})
self.flush_cache()
@ -1753,6 +1753,11 @@ class RustCore(
if name in PLAIN_SETTING_ATTRIBUTES or name in SETTING_FIELDS or name in ASSIGNABLE_ALIASES:
_ = self.apply(ChangeSet(assign={name: value}))
elif name == "model_list":
from litellm.router_deprecated import (
_note, # pyright: ignore[reportPrivateUsage] # the deprecated write notes its use like Python's
)
_note("model_list", "registry().all()")
self._assign_model_list(_as_list(value))
elif name in _HOSTED_STATE:
# A deprecated attribute write of the registry state: it lands where this core keeps it.
@ -1810,7 +1815,31 @@ class RustCore(
) -> Mapping[str, object] | Sequence[Mapping[str, object]]:
"""Pick a deployment for ``model`` without calling it (``pick``), list the deployments a
pick would choose among (``healthy``), or pick one for a pass-through route
(``pass_through``, which picks with the sync strategies, as Python does)."""
(``pass_through``, which picks with the sync strategies, as Python does). The caller's
``request_kwargs`` are left unchanged: selection works on a copy."""
return await self._select_in_place(
model,
None if request_kwargs is None else detached_request(request_kwargs),
kind=kind,
specific_deployment=specific_deployment,
messages=messages,
input=input,
health_check_probe=health_check_probe,
)
async def _select_in_place(
self,
model: str,
request_kwargs: dict[str, object] | None = None, # mutable-ok: pinned and stamped in place
*,
kind: str = "pick",
specific_deployment: bool | None = None,
messages: object = None,
input: object = None,
health_check_probe: bool = False,
**_: object, # kwargs-ok: the RouterCore select options this core ignores
) -> Mapping[str, object] | Sequence[Mapping[str, object]]:
"""``select`` on the request the core itself routes, which keeps what selection pins on it."""
if kind not in ("pick", "healthy", "pass_through"):
raise ValueError(f"unknown select kind {kind!r}")
request: Final = request_kwargs if request_kwargs is not None else {}

View file

@ -1331,7 +1331,7 @@ class RustEndpoints:
assert isinstance(scheduler, Scheduler)
async def healthy_deployments() -> Sequence[object]:
healthy: Final = await self._core.select(model, dict(request), kind="healthy")
healthy: Final = await self._core._select_in_place(model, dict(request), kind="healthy") # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core
return list(healthy) if isinstance(healthy, Sequence) else [healthy]
await scheduler.wait_for_turn(
@ -1584,7 +1584,7 @@ class RustEndpoints:
metadata_variable_name: Final = get_router_metadata_variable_name(function_name="aretrieve_batch")
if model is not None:
specific: Final = kwargs.pop("specific_deployment", None)
healthy = await self._core.select( # rebind-ok: rebound as the Python router rebinds it
healthy = await self._core._select_in_place( # pyright: ignore[reportPrivateUsage] # rebind-ok: rebound as the Python router rebinds it
model, dict(kwargs), kind="healthy", specific_deployment=bool(specific) or None
)
else:
@ -1674,7 +1674,7 @@ class RustEndpoints:
)
from litellm.types.utils import LlmProviders
healthy: Final = await self._core.select(
healthy: Final = await self._core._select_in_place( # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core
requested, dict(kwargs), kind="healthy", specific_deployment=bool(specific) or None
)
deployments: Final = list(healthy) if isinstance(healthy, Sequence) else [healthy]

View file

@ -37,7 +37,7 @@ class RustRegistryMembers:
return self._registry_state.add_deployment(deployment)
def set_model_list(self, model_list: Sequence[object]) -> None:
"""Replace every deployment (``set_model_list``); aliases and wildcard routes stay."""
"""Replace every deployment (``set_model_list``) with its wildcard routes and names; aliases stay."""
self._registry_state.set_model_list(model_list)
@property
@ -46,11 +46,9 @@ class RustRegistryMembers:
return self._registry_state._core_model_list # pyright: ignore[reportPrivateUsage] # the core reads its own host registry
def _assign_model_list(self, model_list: Sequence[object]) -> None:
"""``router.model_list = [...]``: the list is swapped as given, without reindexing, and the
"""``router.model_list = [...]``: the list is swapped as given and reindexed, and the
registry version moves once."""
self._registry_state._core_model_list = model_list # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] # the list as assigned, as the core keeps it
self._registry_state._registry_version += 1 # pyright: ignore[reportPrivateUsage] # see above
self._registry_state._on_change() # pyright: ignore[reportPrivateUsage] # see above
self._registry_state.assign_model_list(model_list)
@staticmethod
def deployment_is_active_for_environment(deployment: Deployment) -> bool:

View file

@ -484,9 +484,10 @@ class PreRoutingStrategies:
def _sync_adaptive_router_hooks(self) -> None:
"""One post-call hook per registered adaptive router, plus the header hook while any exists."""
from litellm.router_strategy.adaptive_router.hooks import (
AdaptiveRouterPostCallHook,
AdaptiveRouterResponseHeaderHook,
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
from litellm.router_utils.global_callback_holders import (
adaptive_routers_on_host,
sync_adaptive_response_header_hook,
)
callbacks: Final = self._host.callbacks
@ -494,8 +495,7 @@ class PreRoutingStrategies:
callbacks.remove_callback_from_all_lists(callback)
for tagged in chain.from_iterable(self.adaptive.values()):
callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy))
if self.adaptive:
callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook())
sync_adaptive_response_header_hook(callbacks, bool(self.adaptive) or adaptive_routers_on_host(self._host))
# -- the request side --------------------------------------------------------------------------

View file

@ -71,7 +71,7 @@ async def prompt_management_acompletion(
caller_timeout: Final = kwargs.get("timeout") or kwargs.get("request_timeout")
caller_stream_timeout: Final = kwargs.get("stream_timeout")
specific_deployment: Final = kwargs.pop("specific_deployment", None)
picked: Final = await core.select(
picked: Final = await core._select_in_place( # pyright: ignore[reportPrivateUsage] # the core's own selection, which pins onto the request
model,
kwargs, # pyright: ignore[reportUnknownArgumentType] # the request kwargs
kind="pick",

View file

@ -67,6 +67,7 @@ from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS
from litellm.responses.bridge_check import keep_registered_mode
from litellm.router_api import RouterRegistry, deployment_id
from litellm.router_core import organization_deployment_id
from litellm.router_rust_model_info import HostModelInfo
from litellm.router_rust_shaping import ROUTER_CLIENTS_TARGET
from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model
@ -632,6 +633,10 @@ class HostRegistry(HostModelInfo):
self._core_team_model_to_deployment_indices = {}
self._core_team_pattern_routers = {}
self._core_team_public_model_names = frozenset()
# The replaced list's wildcard routes and names go with it.
self._core_pattern_router = PatternMatchRouter()
self._core_provider_default_deployment_ids.clear()
self._core_deployment_names.clear()
if self.strategies is not None:
self.strategies.reset()
self._provider_unresolved_deployments = ()
@ -666,10 +671,27 @@ class HostRegistry(HostModelInfo):
if "id" not in info_dict:
info_dict["id"] = deployment_id(str(model_name), litellm_params)
organization = litellm_params.get("organization")
organizations: Sequence[object] = organization if isinstance(organization, list) else (None,)
for org in organizations:
if org is not None:
if isinstance(organization, list):
# Each organization's deployment has its own id (``organization_deployment_id``).
for position, org in enumerate(tuple(organization)): # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # configured organizations
litellm_params["organization"] = org
self._create_deployment(
model,
str(model_name),
litellm_params,
{
**info_dict,
"id": organization_deployment_id(
declared_id,
position,
org,
lambda: deployment_id(str(model_name), litellm_params),
),
},
declared_id=declared_id,
duplicate_ids=duplicate_ids,
)
else:
self._create_deployment(
model,
str(model_name),
@ -865,6 +887,40 @@ class HostRegistry(HostModelInfo):
vertex_credentials=credentials, # pyright: ignore[reportArgumentType] # the configured credentials
)
def served_deployments(self) -> list[UntypedEntry]: # mutable-ok: a fresh list for the Rust handle
"""The deployments the Rust handle routes over: every entry with an id. Only
``router.model_list = [...]`` keeps an entry without one, and that entry is never served."""
return [entry for entry in self._core_model_list if _model_info_of(entry).get("id") is not None]
def assign_model_list(self, model_list: Sequence[object]) -> None:
"""``router.model_list = [...]`` (deprecated, ``_assign_model_list``): the list is swapped as
given and reindexed, and the registry version moves once."""
entries: Final = [_LITELLM_PARAMS_ADAPTER.validate_python(entry) for entry in model_list]
with self._deployment_index_lock:
self._core_model_list = model_list # pyright: ignore[reportAttributeAccessIssue] # the list as assigned, as the core keeps it
self.model_id_to_deployment_index_map = {
str(model_id): idx
for idx, entry in enumerate(entries)
if (model_id := _model_info_of(entry).get("id")) is not None
}
self._core_model_name_to_deployment_indices = {}
self._core_team_model_to_deployment_indices = {}
self._core_team_public_model_names = frozenset()
# An entry without an id is kept as given but never served, as on the Python core.
served: Final = [
(idx, entry) for idx, entry in enumerate(entries) if _model_info_of(entry).get("id") is not None
]
for idx, entry in served:
model_name = entry.get("model_name")
if model_name:
self._core_model_name_to_deployment_indices.setdefault(str(model_name), []).append(idx)
self._update_team_model_index(entry, idx)
self._core_model_names = {str(entry["model_name"]) for _, entry in served if entry.get("model_name")}
self._access_groups_cache = None
self._invalidate_model_group_info_cache()
self._registry_version += 1
self._on_change()
def _update_team_model_index(self, model: Mapping[str, object], idx: int) -> None:
info: Final = _model_info_of(model)
team_id: Final = info.get("team_id")
@ -915,6 +971,8 @@ class HostRegistry(HostModelInfo):
public_model_name for _, public_model_name in self._core_team_model_to_deployment_indices
)
self._core_pattern_router.remove_deployment(model_id)
if model_id in self._core_provider_default_deployment_ids:
self._core_provider_default_deployment_ids.remove(model_id)
for team_id in list(self._core_team_pattern_routers):
team_router = self._core_team_pattern_routers[team_id]
team_router.remove_deployment(model_id)
@ -972,6 +1030,8 @@ class HostRegistry(HostModelInfo):
self.strategies.upserted(deployment, on_router)
except Exception as e:
if not self.ignore_invalid_deployments:
# The failed edit raises, but the deployment it replaced keeps serving.
self._restore_deployment_after_failed_upsert(on_router, model_id)
raise
verbose_router_logger.warning(
"Error upserting deployment %s (id=%s): %s. Dropping it and continuing with other deployments.",

View file

@ -5,8 +5,9 @@ serves them.
A router that retires hands the checks it vacates in its host's ``callbacks`` list to a surviving
router's same-keyed check (``routers_holding_global_callbacks``), and a router that drops a check at
runtime keeps it registered while another live router still has one of that type
(``live_check_owners``). Both rules look at every router in the process, so the sets live here,
outside any one router core.
(``live_check_owners``). The sets hold every router in the process, whatever core serves it, so
they live here, outside any one router core; both rules only ever consult the routers on the same
host (``on_host``), since another host's lists are not this host's.
A member of ``routers_holding_global_callbacks`` serves ``_global_callback_candidates()``,
``_core_adaptive_routers`` and ``_sync_adaptive_router_hooks()``; a member of ``live_check_owners``
@ -15,8 +16,9 @@ serves ``optional_callbacks``.
import threading
import weakref
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final, Protocol
from collections.abc import Iterable, Mapping, Sequence
from types import MethodType
from typing import TYPE_CHECKING, Final, Protocol, TypeVar
if TYPE_CHECKING:
from litellm.integrations.custom_logger import CustomLogger
@ -40,14 +42,82 @@ class CheckOwner(Protocol):
def optional_callbacks(self) -> "Sequence[object] | None": ...
_Router: Final = TypeVar("_Router")
routers_holding_global_callbacks: Final["weakref.WeakSet[GlobalCallbackHolder]"] = weakref.WeakSet()
routers_holding_global_callbacks_lock: Final = threading.Lock()
live_check_owners: Final["weakref.WeakSet[CheckOwner]"] = weakref.WeakSet()
OUTCOME_CALLBACK_LISTS: Final = (
"success_callback",
"_async_success_callback",
"failure_callback",
"_async_failure_callback",
)
def without_router_callbacks(
callbacks: Iterable[object], owned: Iterable[object], method_owners: Iterable[object]
) -> list[object]: # mutable-ok: a fresh callback list for the host
"""``callbacks`` without a router's own entries: the objects in ``owned`` and the bound methods of
``method_owners``. Everything other code registered stays."""
owned_ids: Final = frozenset(map(id, owned))
owner_ids: Final = frozenset(map(id, method_owners))
return [
callback
for callback in callbacks
if id(callback) not in owned_ids
and not (isinstance(callback, MethodType) and id(callback.__self__) in owner_ids)
]
def on_host(routers: Iterable[_Router], host: object) -> tuple[_Router, ...]:
"""The routers among ``routers`` built on ``host``: a retiring router hands its vacated checks
only to these, and a dropped check stays registered only for these."""
return tuple(owner for owner in routers if getattr(owner, "_host", None) is host)
class _HeaderHookCallbacks(Protocol):
"""The host callback registration the adaptive response header hook goes through."""
def get_custom_loggers_for_type(
self, callback_type: "type[CustomLogger]"
) -> "list[CustomLogger]": ... # mutable-ok: the manager's own return type
def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None: ...
def add_litellm_callback(self, callback: "CustomLogger") -> None: ...
def adaptive_routers_on_host(host: object) -> bool:
"""Whether a live router built on ``host`` (request-scoped ones included) has an adaptive router."""
return any(getattr(owner, "_core_adaptive_routers", None) for owner in on_host(tuple(live_check_owners), host))
def sync_adaptive_response_header_hook(callbacks: _HeaderHookCallbacks, wanted: bool) -> None:
"""
Exactly one ``AdaptiveRouterResponseHeaderHook`` on the host while ``wanted`` (some router on it has
an adaptive router), none otherwise: the hook is stateless and shared by every router on the host,
so it is added once, never per sync, and leaves with the last adaptive router.
"""
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterResponseHeaderHook
present: Final = callbacks.get_custom_loggers_for_type(AdaptiveRouterResponseHeaderHook)
for extra in present[1 if wanted else 0 :]:
callbacks.remove_callback_from_all_lists(extra)
if wanted and not present:
callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook())
__all__ = [
"OUTCOME_CALLBACK_LISTS",
"CheckOwner",
"GlobalCallbackHolder",
"adaptive_routers_on_host",
"live_check_owners",
"on_host",
"routers_holding_global_callbacks",
"routers_holding_global_callbacks_lock",
"sync_adaptive_response_header_hook",
"without_router_callbacks",
]

View file

@ -4238,12 +4238,19 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"})
# A deployment's declared context window is its own: written under the shared backend key it would
# outlive the router and narrow every other router's deployment of the same model.
DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS: Final[frozenset[str]] = frozenset(
{"max_input_tokens", "max_output_tokens", "max_tokens"}
)
SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = (
frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__)
- frozenset(CustomPricingLiteLLMParams.model_fields)
- DEPLOYMENT_SCOPED_PRICING_FIELDS
- DEPLOYMENT_SCOPED_CAPABILITY_FIELDS
- DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS
- DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS
)

View file

@ -59,7 +59,8 @@ def windowed(
group: str, deployment_id: str, max_input_tokens: int, **params: object
) -> dict[str, object]: # mutable-ok: a model_list entry
"""A deployment declaring ``max_input_tokens`` under a model name no other test uses: a deployment's
model_info is written into the process-wide cost map under its model name and id."""
model_info is written into the process-wide cost map under its id (and its schema fields, though not
its token limits, under its model name)."""
model_info: Final = {"max_input_tokens": max_input_tokens, **dict(params.pop("model_info", None) or {})} # pyright: ignore[reportArgumentType]
return deployment(
group,

View file

@ -320,16 +320,14 @@ def test_default_fallbacks_given_as_a_string_raise_type_error(make_core: MakeCor
_one(make_core, default_fallbacks="b")
def test_default_fallbacks_are_appended_to_the_process_fallbacks_list_itself(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Oddity: with no ``fallbacks`` argument the router adopts the process list object and appends the
star entry to it, so the process setting is changed too."""
def test_default_fallbacks_extend_a_copy_of_the_process_fallbacks_list(make_core: MakeCore, host: ScriptedHost) -> None:
"""With no ``fallbacks`` argument the router starts from the process list, but appends the star
entry to its own copy, so the process setting is left alone."""
process_fallbacks: Final[list[object]] = [{"g": ["e"]}]
host.set_dynamic(fallbacks=process_fallbacks)
core: Final = _one(make_core, default_fallbacks=["c"])
assert core.settings().fallbacks == [{"g": ["e"]}, {"*": ["c"]}]
assert process_fallbacks == [{"g": ["e"]}, {"*": ["c"]}]
assert process_fallbacks == [{"g": ["e"]}]
# --- validation --------------------------------------------------------------------------------
@ -873,11 +871,22 @@ def test_runtime_optional_pre_call_checks_none_changes_nothing(make_core: MakeCo
assert host_callback_names(host)["callbacks"] == ["PromptCachingDeploymentCheck"]
def test_runtime_removal_keeps_the_host_callback_while_another_live_router_uses_that_check(
def test_runtime_removal_keeps_the_host_callback_while_another_live_router_on_the_host_uses_that_check(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Oddity: the check is left in this host's list because some other live router (here on another
host) still has one of that type."""
collect_other_routers()
other: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
core.apply(ChangeSet(settings={"optional_pre_call_checks": []}))
assert core.optional_callbacks == [] # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
assert all(check in host.lists["callbacks"] for check in other.optional_callbacks) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
other.retire()
def test_runtime_removal_drops_the_host_callback_though_a_router_on_another_host_uses_that_check(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Another host's router keeps its own check on its own host; it does not hold this host's."""
collect_other_routers()
other_host: Final = CallbackHost()
from tests.unit.test_router.core.scripted_host import python_core
@ -888,7 +897,8 @@ def test_runtime_removal_keeps_the_host_callback_while_another_live_router_uses_
core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
core.apply(ChangeSet(settings={"optional_pre_call_checks": []}))
assert core.optional_callbacks == [] # pyright: ignore[reportAttributeAccessIssue]
assert host_callback_names(host)["callbacks"] == ["PromptCachingDeploymentCheck"]
assert host_callback_names(host)["callbacks"] == []
assert host_callback_names(other_host)["callbacks"] == ["PromptCachingDeploymentCheck"]
other.retire()
@ -946,23 +956,35 @@ def test_retiring_one_router_keeps_the_callbacks_of_another_on_the_same_host(
assert all(callback.__self__ is second for callback in host.lists["_async_success_callback"]) # pyright: ignore[reportAttributeAccessIssue]
def test_retire_hands_a_vacated_check_to_another_live_router_on_a_different_host(
def test_retire_never_hands_a_vacated_check_to_a_live_router_on_a_different_host(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Oddity: retire re-registers the same-keyed check of any other live router in the process into
this host's list, even one built on a different host."""
"""Another host's router keeps its check on its own host; this host's list is left empty."""
collect_other_routers()
from tests.unit.test_router.core.scripted_host import python_core
other_host: Final = CallbackHost()
other: Final = python_core(
CallbackHost(), model_list=[deployment("g", "o")], optional_pre_call_checks=["prompt_caching"]
other_host, model_list=[deployment("g", "o")], optional_pre_call_checks=["prompt_caching"]
)
core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
core.retire()
assert host.lists["callbacks"] == other.optional_callbacks
assert host.lists["callbacks"] == []
assert other_host.lists["callbacks"] == other.optional_callbacks # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
other.retire()
def test_retire_hands_a_vacated_check_to_another_live_router_on_the_same_host(
make_core: MakeCore, host: ScriptedHost
) -> None:
collect_other_routers()
first: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
second: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
first.retire()
assert host.lists["callbacks"] == second.optional_callbacks # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
second.retire()
# --- observe -----------------------------------------------------------------------------
@ -1131,23 +1153,42 @@ def test_flush_cache_drops_the_response_cache_routing_state_and_prompt_caching_p
assert host_callback_names(host) == before
def test_reset_empties_every_success_and_failure_list_and_the_retry_policy(
def test_reset_drops_the_routers_own_success_and_failure_callbacks_and_the_retry_policy(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Note: reset clears the host's success/failure lists outright, foreign callbacks included, but
leaves ``callbacks`` and ``input_callback`` alone."""
"""reset drops only this router's entries from the host's success/failure lists (other code's
callbacks stay) and leaves ``callbacks`` and ``input_callback`` alone."""
core: Final = _one(make_core, retry_policy={"RateLimitErrorRetries": 3}, routing_strategy="least-busy")
host.lists["success_callback"].append("foreign")
host.lists["_async_failure_callback"].append("foreign-failure")
core.health().store.set_cache("k", 1)
core.reset() # pyright: ignore[reportAttributeAccessIssue]
assert {name: callbacks for name, callbacks in host_callback_names(host).items() if callbacks} == {
"callbacks": ["LeastBusyLoggingHandler"],
"input_callback": ["LeastBusyLoggingHandler"],
"success_callback": ["str"],
"_async_failure_callback": ["str"],
}
assert (host.lists["success_callback"], host.lists["_async_failure_callback"]) == (["foreign"], ["foreign-failure"])
assert core.retry_policy is None # pyright: ignore[reportAttributeAccessIssue]
assert core.health().store.get_cache("k") is None
def test_resetting_one_router_keeps_another_routers_success_and_failure_callbacks(
make_core: MakeCore, host: ScriptedHost
) -> None:
collect_other_routers()
first: Final = _one(make_core)
before_first: Final = {name: list(callbacks) for name, callbacks in host.lists.items()}
second: Final = _one(make_core)
first.reset() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
for name in ("success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"):
remaining = host.lists.get(name, [])
assert all(getattr(callback, "__self__", None) is not first for callback in remaining)
theirs = [callback for callback in before_first.get(name, []) if getattr(callback, "__self__", None) is first]
assert len(remaining) == len(theirs) and all(callback.__self__ is second for callback in remaining) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
# --- attribute writes ----------------------------------------------------------------------

View file

@ -291,20 +291,29 @@ async def test_an_anthropic_system_block_is_counted_against_the_context_window(m
@pytest.mark.asyncio
async def test_a_deployment_context_window_is_written_into_the_process_cost_map(make_core: MakeCore) -> None:
"""Oddity: a deployment's declared ``max_input_tokens`` replaces the process-wide cost-map entry of its
model, so it outlives the router and narrows another router's deployment of the same model."""
async def test_a_deployment_context_window_stays_out_of_the_models_shared_cost_map_entry(
make_core: MakeCore,
) -> None:
"""A deployment's declared ``max_input_tokens`` is its own: the process-wide cost-map entry of its
model keeps the built-in window, so neither a retired nor a live router narrows another router's
deployment of the same model."""
builtin_window: Final = litellm.model_cost[NO_JSON_MODE_MODEL.removeprefix("openai/")]["max_input_tokens"] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped in the checker's view
first: Final = make_core(
model_list=[deployment("g", "declares-five", model=NO_JSON_MODE_MODEL, model_info={"max_input_tokens": 5})]
model_list=[deployment("g", "declares-five", model=NO_JSON_MODE_MODEL, model_info={"max_input_tokens": 5})],
enable_pre_call_checks=True,
)
first.retire()
other: Final = python_core(
CallbackHost(),
model_list=[deployment("g", "declares-nothing", model=NO_JSON_MODE_MODEL)],
enable_pre_call_checks=True,
)
for key in (NO_JSON_MODE_MODEL, NO_JSON_MODE_MODEL.removeprefix("openai/")):
assert litellm.model_cost.get(key, {}).get("max_input_tokens") in (None, builtin_window) # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
assert (await other.select("g", messages=LONG))["model_info"]["id"] == "declares-nothing"
with pytest.raises(litellm.ContextWindowExceededError, match="Max Input Tokens=5"):
await other.select("g", messages=SHORT)
await first.select("g", messages=LONG) # pyright: ignore[reportCallIssue] # select takes messages; the checker reads an overload
first.retire()
assert (await other.select("g", messages=LONG))["model_info"]["id"] == "declares-nothing"
other.retire()

View file

@ -663,3 +663,14 @@ def test_cooldown_time_reads_back_as_given(make_core: MakeCore) -> None:
assert updated == 3 and type(updated) is int
default: Final = as_router(make_core(model_list=[deployment("gpt", "a")])).settings().cooldown_time
assert default == 5 and type(default) is int
@pytest.mark.asyncio
async def test_the_old_healthy_listing_takes_the_retry_skip_list_off_the_callers_request(make_core: MakeCore) -> None:
router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")]))
request: Final[dict[str, object]] = {"_retry_skipped_deployment_ids": ("a",)}
with pytest.warns(DeprecationWarning, match=r"Router\.async_get_healthy_deployments is deprecated"):
healthy: Final = await router.async_get_healthy_deployments("gpt", request) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the old untyped dicts
assert isinstance(healthy, list)
assert [picked_id(entry) for entry in healthy] == ["b"] # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # see above
assert request == {}

View file

@ -392,6 +392,17 @@ async def test_unknown_model_group_raises_without_calling_the_host(
assert host.calls == []
@pytest.mark.parametrize("endpoint", ["acompletion", "completion"])
async def test_an_entry_assigned_without_an_id_is_never_called(make_core: MakeCore, host: ScriptedHost, endpoint: str):
"""``model_list = [...]`` keeps an entry without ``model_info.id`` as given, but never routes to it."""
core: Final = _one_group(make_core)
with pytest.warns(DeprecationWarning, match=r"Router\.model_list is deprecated"): # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
core.model_list = [{"model_name": "b", "litellm_params": {"model": "openai/b"}}] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable
with pytest.raises(litellm.BadRequestError, match="model=b"):
await core.call(endpoint, {"model": "b", "messages": MESSAGES})
assert host.calls == []
@pytest.mark.parametrize("op", ["no_such_endpoint", "acompletions", ""])
async def test_call_with_an_unknown_op_raises_attribute_error_without_calling_the_host(
make_core: MakeCore, host: ScriptedHost, op: str

View file

@ -370,16 +370,18 @@ async def test_order_tiers_run_before_a_plain_list_or_the_default_chain(
# --- default fallbacks ---------------------------------------------------------------------------
def test_default_fallbacks_are_appended_to_the_process_fallbacks_list_in_place(
def test_default_fallbacks_extend_the_process_fallbacks_without_changing_the_process_list(
make_core: MakeCore, host: ScriptedHost
) -> None:
process_fallbacks: list[object] = [{"a": ["b"]}]
host.set_dynamic(fallbacks=process_fallbacks)
core = build(make_core, default_fallbacks=["c"])
other = build(make_core, default_fallbacks=["d"])
assert core.settings().fallbacks == [{"a": ["b"]}, {"*": ["c"]}]
assert process_fallbacks == [{"a": ["b"]}, {"*": ["c"]}]
assert other.settings().fallbacks == [{"a": ["b"]}, {"*": ["d"]}] # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
assert process_fallbacks == [{"a": ["b"]}]
def test_default_fallbacks_leave_the_constructor_fallbacks_list_unchanged(make_core: MakeCore) -> None:

View file

@ -300,7 +300,7 @@ def test_a_failed_edit_rolls_back_to_the_serving_router_even_above_a_tightened_l
assert restored.litellm_params.tags is None
def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_leaves_the_router_removed(
def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_restores_the_router(
make_core: MakeCore,
) -> None:
limit: Final = {"value": 2}
@ -319,8 +319,8 @@ def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_leaves_the_
ChangeSet(upsert=[Deployment(**complexity_marker("two", "m2", config=HEURISTIC_V2, tags=["edited"]))]) # pyright: ignore[reportArgumentType]
)
assert str(raised.value) == HEURISTIC_V2_LIMIT_MESSAGE
assert model_ids(core) == ["s", "b", "m1"]
assert registered_tags(core, "complexity") == {"one": [()]}
assert model_ids(core) == ["s", "b", "m1", "m2"]
assert registered_tags(core, "complexity") == {"one": [()], "two": [()]}
# --- the adaptive router -------------------------------------------------------------------------------
@ -373,10 +373,19 @@ def test_an_adaptive_router_registers_one_post_call_hook_and_a_header_hook(
post_call, header = _hook_counts(host)
assert post_call == 1
assert header >= 1
assert header == 1
def test_deleting_an_adaptive_router_releases_it_and_its_post_call_hook(
def test_resyncing_an_adaptive_router_keeps_a_single_header_hook(make_core: MakeCore, host: ScriptedHost) -> None:
core: Final = make_core(model_list=_adaptive_list())
core.apply(ChangeSet(upsert=[Deployment(**adaptive_marker("ar", "ar1", ["y"]))])) # pyright: ignore[reportArgumentType]
core.apply(ChangeSet(upsert=[Deployment(**adaptive_marker("ar", "ar1", ["x"]))])) # pyright: ignore[reportArgumentType]
assert _hook_counts(host) == (1, 1)
def test_deleting_an_adaptive_router_releases_it_its_post_call_hook_and_the_header_hook(
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(model_list=_adaptive_list())
@ -384,7 +393,30 @@ def test_deleting_an_adaptive_router_releases_it_and_its_post_call_hook(
core.apply(ChangeSet(delete=["ar1"]))
assert registered_tags(core, "adaptive") == {}
assert _hook_counts(host)[0] == 0
assert _hook_counts(host) == (0, 0)
def test_retiring_the_last_adaptive_router_on_the_host_removes_the_header_hook(
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(model_list=_adaptive_list())
core.retire()
assert _hook_counts(host) == (0, 0)
def test_retiring_one_adaptive_router_keeps_the_header_hook_for_another_on_the_host(
make_core: MakeCore, host: ScriptedHost
) -> None:
first: Final = make_core(model_list=_adaptive_list())
second: Final = make_core(model_list=_adaptive_list())
first.retire()
assert _hook_counts(host)[1] == 1
second.retire()
assert _hook_counts(host) == (0, 0)
def test_a_complexity_router_with_adaptive_enabled_also_registers_an_adaptive_router(make_core: MakeCore) -> None:

View file

@ -315,6 +315,8 @@ async def test_an_access_group_only_caller_reaches_any_deployment_by_its_litellm
# --- tier litellm_params -------------------------------------------------------------------------
# ``select`` works on a copy of its caller's request; these read what selection pins and stamps on the
# request the router routes (and re-routes on retries), through the cores' in-place selection.
def _pinned(**tier_params: object) -> dict[str, object]: # mutable-ok: tiers # kwargs-ok: tier litellm_params
@ -331,7 +333,7 @@ async def test_a_tier_param_no_deployment_supports_is_dropped(make_core: MakeCor
]
)
request: Final[dict[str, object]] = {}
chosen: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
chosen: Final = await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue]
assert request["temperature"] == 0.2
assert "logit_bias" not in request
@ -347,7 +349,7 @@ async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore)
]
)
request: Final[dict[str, object]] = {"allowed_openai_params": ["logit_bias"]}
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
assert request["logit_bias"] == {"1": 2}
@ -360,7 +362,7 @@ async def test_a_tier_reasoning_effort_removes_the_callers_other_effort_carriers
"reasoning": {"effort": "high"},
"max_completion_tokens": 100,
}
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
assert request["reasoning_effort"] == "low"
assert "thinking" not in request
assert request["output_config"] == {"format": "json"}
@ -372,7 +374,7 @@ async def test_a_tier_reasoning_effort_removes_the_callers_other_effort_carriers
async def test_a_tier_without_an_output_ceiling_leaves_the_callers_ceiling(make_core: MakeCore) -> None:
core: Final = make_core(model_list=_tiered(tiers=_pinned(temperature=0.1)))
request: Final[dict[str, object]] = {"max_completion_tokens": 100}
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
assert request["temperature"] == 0.1
assert request["max_completion_tokens"] == 100
assert "max_tokens" not in request
@ -384,12 +386,12 @@ async def test_a_tier_without_an_output_ceiling_leaves_the_callers_ceiling(make_
async def test_a_later_pick_restores_the_callers_ceiling_a_tier_pin_replaced(make_core: MakeCore) -> None:
core: Final = make_core(model_list=_tiered(tiers=_pinned(max_tokens=7)))
request: Final[dict[str, object]] = {"max_completion_tokens": 100}
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
assert request["max_tokens"] == 7
assert "max_completion_tokens" not in request
assert _metadata(request)["_client_output_ceiling"] == {"max_completion_tokens": 100}
chosen: Final = await core.select("smart", request, messages=HARD_MESSAGES) # pyright: ignore[reportCallIssue]
chosen: Final = await core._select_in_place("smart", request, messages=HARD_MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue]
assert request["max_completion_tokens"] == 100
assert "max_tokens" not in request
@ -401,13 +403,13 @@ async def test_a_second_pick_leaves_the_first_picks_forwarded_marker_params_untr
# next attempt, so the tracking key is dropped while the params themselves stay behind.
core: Final = make_core(model_list=_tiered(temperature=0.3))
request: Final[dict[str, object]] = {}
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
forwarded: Final = request["_alias_marker_forwarded_params"]
assert isinstance(forwarded, tuple)
assert "temperature" in forwarded
assert request["temperature"] == 0.3
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
assert "_alias_marker_forwarded_params" not in request
assert request["temperature"] == 0.3
@ -419,7 +421,7 @@ async def test_a_second_pick_leaves_the_first_picks_forwarded_marker_params_untr
async def test_the_routing_decision_keeps_its_prompt_signals_by_default(make_core: MakeCore) -> None:
core: Final = make_core(model_list=_tiered())
request: Final[dict[str, object]] = {}
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
decision: Final = _metadata(request)["routing_decision"]
assert isinstance(decision, Mapping)
assert "signals" in decision
@ -429,7 +431,7 @@ async def test_the_routing_decision_keeps_its_prompt_signals_by_default(make_cor
async def test_redacted_message_logging_drops_the_prompt_quoting_routing_decision_fields(make_core: MakeCore) -> None:
core: Final = make_core(model_list=_tiered())
request: Final[dict[str, object]] = {"standard_callback_dynamic_params": {"turn_off_message_logging": True}}
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
decision: Final = _metadata(request)["routing_decision"]
assert isinstance(decision, Mapping)
assert "signals" not in decision
@ -440,7 +442,7 @@ async def test_redacted_message_logging_drops_the_prompt_quoting_routing_decisio
async def test_routing_metadata_goes_to_litellm_metadata_when_the_request_has_it(make_core: MakeCore) -> None:
core: Final = make_core(model_list=_tiered())
request: Final[dict[str, object]] = {"litellm_metadata": {}}
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
assert "routing_decision" in _metadata(request, "litellm_metadata")
assert "metadata" not in request

View file

@ -131,8 +131,27 @@ def test_a_list_of_organizations_fans_out_into_one_deployment_per_organization(m
entries = core.registry().all()
assert [entry["litellm_params"]["organization"] for entry in entries] == ["org-1", "org-2"]
assert len(set(ids(entries))) == 1
assert len(set(ids(entries))) == 2
assert core.registry().index("model_name") == {"gpt": [0, 1]}
for entry in entries: # pyright: ignore[reportAny] # untyped in the checker's view
found = core.registry().deployment(entry["model_info"]["id"]) # pyright: ignore[reportAny] # untyped in the checker's view
assert found is not None
assert found.litellm_params.organization == entry["litellm_params"]["organization"]
def test_organization_deployments_get_the_same_ids_on_every_build(make_core: MakeCore) -> None:
first = make_core(model_list=[_bare("gpt", "openai/x", organization=["org-1", "org-2"])])
second = make_core(model_list=[_bare("gpt", "openai/x", organization=["org-1", "org-2"])])
assert ids(first.registry().all()) == ids(second.registry().all())
def test_a_declared_id_stays_on_the_first_organization_and_is_suffixed_on_the_rest(make_core: MakeCore) -> None:
entry = _bare("gpt", "openai/x", organization=["org-1", "org-2"])
entry["model_info"] = {"id": "declared"}
core = make_core(model_list=[entry])
assert ids(core.registry().all()) == ["declared", "declared-org-2"]
def test_deployment_names_list_each_deployments_litellm_model(make_core: MakeCore) -> None:

View file

@ -4,6 +4,7 @@ access groups; blocked and unhealthy groups; credentials lookups.
"""
import time
from typing import Final
import pytest
@ -668,15 +669,31 @@ def test_the_deprecated_generate_model_id_is_the_registry_id_hash(make_core: Mak
assert generated == deployment_id("g", {"model": "x"})
def test_assigning_model_list_swaps_the_list_without_reindexing(make_core: MakeCore) -> None:
"""Odd but current: the id index still maps ``a`` to position 0, which now holds the new entry."""
def test_assigning_model_list_swaps_and_reindexes_the_list(make_core: MakeCore) -> None:
"""The replaced deployment's id finds nothing; the new entry is found by its own id and name."""
core = make_core(model_list=[deployment("gpt", "a")])
version = core.registry().version
core.model_list = [deployment("n", "n")] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable
with pytest.warns(DeprecationWarning, match=r"Router\.model_list is deprecated"): # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
core.model_list = [deployment("n", "n")] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable
assert core.registry().version == version + 1
assert core.registry().names(kind="ids") == ["a"]
found = core.registry().deployment("a")
assert core.registry().names(kind="ids") == ["n"]
assert core.registry().deployment("a") is None
found = core.registry().deployment("n")
assert found is not None
assert found.model_name == "n"
assert ids(core.registry().deployments("n")) == ["n"]
assert core.registry().deployments("gpt") == []
def test_an_entry_assigned_without_an_id_is_not_served(make_core: MakeCore) -> None:
"""``model_list = [...]`` keeps entries as given; one without ``model_info.id`` is never routed to."""
core = make_core(model_list=[deployment("gpt", "a")])
entry: Final = {"model_name": "b", "litellm_params": {"model": "openai/gpt-4o-mini"}}
with pytest.warns(DeprecationWarning, match=r"Router\.model_list is deprecated"): # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
core.model_list = [entry] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable
assert core.registry().deployments("b") == []
assert core.registry().names(kind="ids") == []

View file

@ -154,14 +154,20 @@ def test_an_upsert_that_empties_a_group_drops_the_group_name(make_core: MakeCore
assert core.registry().deployments("emb") == []
def test_a_failing_upsert_of_an_existing_deployment_raises_and_leaves_it_removed(make_core: MakeCore) -> None:
"""Odd but current: without ignore_invalid_deployments the old deployment is popped and not restored."""
def test_a_failing_upsert_of_an_existing_deployment_raises_and_restores_the_previous_deployment(
make_core: MakeCore,
) -> None:
"""Without ignore_invalid_deployments the failed edit raises, and the deployment it would have
replaced keeps serving its previous configuration."""
core = _core(make_core)
with pytest.raises(litellm.BadRequestError):
core.apply(ChangeSet(upsert=[typed("gpt", "a", model="notaprovider/x")]))
assert core.registry().names(kind="ids") == ["b", "e", "w", "t1"]
restored = core.registry().deployment("a")
assert restored is not None
assert restored.litellm_params.model == "openai/a"
assert core.registry().names(kind="ids") == ["b", "e", "w", "t1", "a"]
def test_a_failing_upsert_under_ignore_invalid_deployments_restores_the_previous_deployment(
@ -241,12 +247,19 @@ def test_deleting_a_wildcard_deployment_stops_it_serving_matching_names(make_cor
assert core.registry().deployments("openai/gpt-x") == []
def test_deleting_a_wildcard_deployment_keeps_it_among_provider_default_deployment_ids(make_core: MakeCore) -> None:
"""Odd but current: the provider default id list is append-only."""
def test_deleting_a_wildcard_deployment_drops_it_from_the_provider_default_deployment_ids(make_core: MakeCore) -> None:
core = _core(make_core)
core.apply(ChangeSet(delete=["w"]))
assert core.settings().provider_default_deployment_ids == []
def test_deleting_a_plain_deployment_keeps_the_wildcards_provider_default_deployment_id(make_core: MakeCore) -> None:
core = _core(make_core)
core.apply(ChangeSet(delete=["a"]))
assert core.settings().provider_default_deployment_ids == ["w"]
@ -300,23 +313,31 @@ def test_set_model_list_leaves_the_given_list_unchanged(make_core: MakeCore) ->
assert given == [{"model_name": "new", "litellm_params": {"model": "openai/n", "api_key": "k"}}]
def test_set_model_list_keeps_the_old_wildcard_routes(make_core: MakeCore) -> None:
"""Odd but current: the wildcard pattern router is not reset, so a replaced wildcard still matches."""
def test_set_model_list_drops_the_old_wildcard_routes(make_core: MakeCore) -> None:
core = _core(make_core)
core.set_model_list([deployment("new", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core
assert names_and_ids(core.registry().deployments("openai/gpt-x")) == [("openai/gpt-x", "w")]
assert core.settings().provider_default_deployment_ids == ["w"]
assert core.registry().deployments("openai/gpt-x") == []
assert core.settings().provider_default_deployment_ids == []
def test_set_model_list_appends_to_the_deployment_names(make_core: MakeCore) -> None:
"""Odd but current: deployment_names keep the replaced deployments' models."""
def test_set_model_list_serves_the_new_lists_wildcard_routes(make_core: MakeCore) -> None:
core = _core(make_core)
core.set_model_list([deployment("anthropic/*", "w2", model="anthropic/*")]) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # public on the core
assert core.registry().deployments("openai/gpt-x") == []
assert names_and_ids(core.registry().deployments("anthropic/claude-x")) == [("anthropic/claude-x", "w2")]
assert core.settings().provider_default_deployment_ids == ["w2"]
def test_set_model_list_replaces_the_deployment_names(make_core: MakeCore) -> None:
core = make_core(model_list=[deployment("gpt", "a")])
core.set_model_list([deployment("new", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core
assert core.registry().names(kind="deployment_names") == ["openai/a", "openai/n"]
assert core.registry().names(kind="deployment_names") == ["openai/n"]
@pytest.mark.asyncio

View file

@ -676,7 +676,7 @@ async def test_retry_skipped_deployment_ids_given_as_a_list_are_ignored(make_cor
@pytest.mark.asyncio
async def test_select_pops_the_narrowing_keys_from_the_request(make_core: MakeCore) -> None:
async def test_select_honors_the_narrowing_keys_but_leaves_them_on_the_callers_request(make_core: MakeCore) -> None:
core: Final = make_core(model_list=list(THREE))
request: Final[dict[str, object]] = {
"_target_order": None,
@ -684,6 +684,22 @@ async def test_select_pops_the_narrowing_keys_from_the_request(make_core: MakeCo
"_retry_skipped_deployment_ids": ("b",),
}
assert ids_of(await core.select("g", request, kind="healthy")) == ["c"]
assert request == {
"_target_order": None,
"_excluded_deployment_ids": ["a"],
"_retry_skipped_deployment_ids": ("b",),
}
@pytest.mark.asyncio # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # untyped in the checker's view
async def test_in_place_selection_pops_the_narrowing_keys_from_the_routed_request(make_core: MakeCore) -> None:
core: Final = make_core(model_list=list(THREE))
request: Final[dict[str, object]] = {
"_target_order": None,
"_excluded_deployment_ids": ["a"],
"_retry_skipped_deployment_ids": ("b",),
}
assert ids_of(await core._select_in_place("g", request, kind="healthy")) == ["c"] # pyright: ignore[reportAttributeAccessIssue, reportUnknownArgumentType, reportUnknownMemberType]
assert request == {}

View file

@ -78,6 +78,19 @@ def _metadata(request: Mapping[str, object]) -> Mapping[str, object]:
return metadata
async def _routed_metadata(
core: RouterCore,
host: ScriptedHost,
model: str,
messages: list[dict[str, str]],
**request: object, # kwargs-ok: the request's own keyword payload
) -> Mapping[str, object]:
"""The metadata the routed call carried: ``select`` leaves its caller's request alone, so the
routing stamps are read where they land, on the request the router sends."""
await core.acompletion(model=model, messages=messages, **request) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
return _metadata(host.calls[-1].kwargs)
# --- the complexity router ------------------------------------------------------------------------
@ -92,11 +105,11 @@ async def test_a_complexity_router_picks_the_tier_deployment_for_the_prompt(
@pytest.mark.asyncio
async def test_a_complexity_router_pick_stamps_the_routing_decision_on_the_request(make_core: MakeCore) -> None:
async def test_a_complexity_router_pick_stamps_the_routing_decision_on_the_routed_request(
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(model_list=_tiered())
request: Final[dict[str, object]] = {}
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
metadata: Final = _metadata(request)
metadata: Final = await _routed_metadata(core, host, "smart", MESSAGES)
decision: Final = metadata["routing_decision"]
assert isinstance(decision, Mapping)
assert decision["router_model_name"] == "smart"
@ -121,11 +134,11 @@ async def test_completion_through_a_complexity_router_reports_the_router_name_as
@pytest.mark.asyncio
async def test_an_alias_to_a_complexity_router_routes_through_it(make_core: MakeCore, host: ScriptedHost) -> None:
core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"})
request: Final[dict[str, object]] = {}
chosen: Final = await core.select("al", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
chosen: Final = await core.select("al", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType]
assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue]
assert _metadata(request)["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue]
await core.acompletion(model="al", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
metadata: Final = await _routed_metadata(core, host, "al", MESSAGES)
assert metadata["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue]
assert host.calls[-1].deployment_id == "s"
assert host.calls[-1].model_group == "al"
@ -137,15 +150,14 @@ async def test_a_complexity_router_takes_precedence_over_plain_deployments_of_it
@pytest.mark.asyncio
async def test_a_complexity_router_tier_naming_an_unknown_model_falls_back_to_the_default_model(
make_core: MakeCore,
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(
model_list=[deployment("big", "b"), _complexity_router(tiers={**TIERS, "SIMPLE": "missing"})]
)
request: Final[dict[str, object]] = {}
chosen: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
chosen: Final = await core.select("smart", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType]
assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue]
decision: Final = _metadata(request)["routing_decision"]
decision: Final = (await _routed_metadata(core, host, "smart", MESSAGES))["routing_decision"]
assert isinstance(decision, Mapping)
assert decision["cause"] == "health_default_fallback"
assert decision["routed_model"] == "big"
@ -282,13 +294,25 @@ async def test_tier_litellm_params_override_the_callers_params_on_the_call(
@pytest.mark.asyncio
async def test_tier_litellm_params_are_pinned_onto_the_request_by_select(make_core: MakeCore) -> None:
async def test_select_leaves_the_callers_request_without_the_tier_pins(make_core: MakeCore) -> None:
core: Final = make_core(model_list=_tiered(tiers=PINNED_TIERS))
request: Final[dict[str, object]] = {"temperature": 0.9, "max_completion_tokens": 100}
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
assert request["temperature"] == 0.1
assert request["max_tokens"] == 7
assert "max_completion_tokens" not in request
picked: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType]
assert picked["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] # a deployment dict
assert request == {"temperature": 0.9, "max_completion_tokens": 100}
@pytest.mark.asyncio # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # untyped in the checker's view
async def test_select_leaves_the_callers_request_and_metadata_without_marker_params_or_stamps(
make_core: MakeCore,
) -> None:
core: Final = make_core(model_list=_tiered(temperature=0.3))
metadata: Final[dict[str, object]] = {"tags": ["t"]}
request: Final[dict[str, object]] = {"metadata": metadata, "specific_deployment": False}
await core.select("smart", request, messages=HARD_MESSAGES) # pyright: ignore[reportCallIssue]
assert request == {"metadata": {"tags": ["t"]}, "specific_deployment": False}
assert request["metadata"] is metadata
assert metadata == {"tags": ["t"]}
@pytest.mark.asyncio
@ -351,13 +375,16 @@ def _session_request(
@pytest.mark.asyncio
async def test_a_subagent_request_follows_the_router_its_session_started_on(make_core: MakeCore) -> None:
async def test_a_subagent_request_follows_the_router_its_session_started_on(
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(model_list=_tiered())
await core.select("smart", _session_request(), messages=MESSAGES) # pyright: ignore[reportCallIssue]
subagent: Final = _session_request(agent_id="agent-1")
chosen: Final = await core.select("big", subagent, messages=MESSAGES) # pyright: ignore[reportCallIssue]
chosen: Final = await core.select("big", _session_request(agent_id="agent-1"), messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType]
assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue]
assert _metadata(subagent)["model_group"] == "smart"
metadata: Final = await _routed_metadata(core, host, "big", MESSAGES, **_session_request(agent_id="agent-1"))
assert host.calls[-1].deployment_id == "s"
assert metadata["model_group"] == "smart"
@pytest.mark.asyncio
@ -428,24 +455,31 @@ async def test_a_routing_plugin_narrows_the_candidates_to_the_models_it_keeps(ma
@pytest.mark.asyncio
async def test_routing_plugin_signals_and_narrowed_candidates_are_stamped_on_the_request(make_core: MakeCore) -> None:
async def test_routing_plugin_signals_and_narrowed_candidates_are_stamped_on_the_routed_request(
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/b"}))])
request: Final[dict[str, object]] = {}
await core.select("g", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
assert request == {
"metadata": {
"routing_plugin_signals": {"narrowing": {"kept": ["openai/b"]}},
"_routing_plugin_candidate_models": ["openai/b"],
}
}
metadata: Final = await _routed_metadata(core, host, "g", MESSAGES)
assert metadata["routing_plugin_signals"] == {"narrowing": {"kept": ["openai/b"]}}
assert metadata["_routing_plugin_candidate_models"] == ["openai/b"]
@pytest.mark.asyncio
async def test_a_routing_plugin_that_keeps_every_candidate_stamps_only_its_signals(make_core: MakeCore) -> None:
async def test_a_routing_plugin_that_keeps_every_candidate_stamps_only_its_signals(
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/a", "openai/b"}))])
metadata: Final = await _routed_metadata(core, host, "g", MESSAGES)
assert metadata["routing_plugin_signals"] == {"narrowing": {"kept": ["openai/a", "openai/b"]}}
assert "_routing_plugin_candidate_models" not in metadata
@pytest.mark.asyncio # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # untyped in the checker's view
async def test_select_leaves_the_callers_request_without_routing_plugin_stamps(make_core: MakeCore) -> None:
core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/b"}))])
request: Final[dict[str, object]] = {}
await core.select("g", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
assert request == {"metadata": {"routing_plugin_signals": {"narrowing": {"kept": ["openai/a", "openai/b"]}}}}
assert request == {}
@pytest.mark.asyncio

View file

@ -2081,7 +2081,8 @@ def test_add_deployment_does_not_leak_custom_metadata_to_shared_backend_key():
def test_shared_backend_model_info_keeps_schema_fields_and_drops_the_rest():
"""Unit test of the whitelist helper: cost-map schema fields survive,
custom pricing overrides and per-deployment metadata do not.
custom pricing overrides, per-deployment metadata and per-deployment token
limits do not.
"""
from litellm.types.utils import shared_backend_model_info
@ -2106,7 +2107,6 @@ def test_shared_backend_model_info_keeps_schema_fields_and_drops_the_rest():
assert filtered == {
"mode": "chat",
"litellm_provider": "openai",
"max_tokens": 128000,
"supports_vision": True,
"supported_endpoints": ["/v1/responses"],
"use_openai_responses_path": True,
@ -3052,7 +3052,7 @@ def test_register_deployment_in_model_cost_writes_both_key_families():
"""
A deployment contributes its full model_info under its unique id and the
cost-map subset under the shared backend key, and the shared key must not
pick up the deployment's private metadata.
pick up the deployment's private metadata or its declared token limits.
"""
model_keys = {
"both-families-id": copy.deepcopy(litellm.model_cost.get("both-families-id")),
@ -3068,7 +3068,8 @@ def test_register_deployment_in_model_cost_writes_both_key_families():
assert litellm.model_cost["both-families-id"]["max_input_tokens"] == 999
shared = litellm.model_cost["hosted_vllm/both-families-backend"]
assert shared["max_input_tokens"] == 999
assert shared["litellm_provider"] == "hosted_vllm"
assert "max_input_tokens" not in shared
assert "id" not in shared
finally:
_restore_model_cost_entries(model_keys)