mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): stop router state leaking across callers, routers and hosts
select works on a copy of the caller's request; default_fallbacks extend a copy of litellm.fallbacks; deployment token limits stay off the shared cost-map key; retire and runtime check removal only consult routers on the same host; reset drops only the router's own success/failure callbacks; a failed upsert restores the replaced deployment; set_model_list and delete reset wildcard routes, provider default ids and deployment names; model_list assignment reindexes and warns; organization-list deployments get their own ids; the adaptive response header hook is kept to one per host and removed with the last adaptive router. Both cores.
This commit is contained in:
parent
cfba1d3ac6
commit
afd1e98692
26 changed files with 712 additions and 156 deletions
|
|
@ -7,7 +7,7 @@ that reaches outside itself only through its host; it does not use ``PythonCore`
|
|||
picked once, when the router is built, from its ``RouterConfig``.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
|
|
@ -52,6 +52,39 @@ class RouterCore(Protocol):
|
|||
def retire(self) -> None: ...
|
||||
|
||||
|
||||
_REQUEST_METADATA_KEYS: Final = ("metadata", "litellm_metadata")
|
||||
|
||||
|
||||
def detached_request(
|
||||
request_kwargs: Mapping[str, object] | None,
|
||||
) -> dict[str, object]: # mutable-ok: the selection cores take a dict
|
||||
"""
|
||||
A copy of a caller's request for ``select``: selection pops keys, pins tier params and stamps
|
||||
routing metadata on the request it routes, and none of that may reach the caller's dict (or the
|
||||
metadata dicts inside it).
|
||||
"""
|
||||
if request_kwargs is None:
|
||||
return {}
|
||||
return {
|
||||
key: dict(value) if key in _REQUEST_METADATA_KEYS and isinstance(value, Mapping) else value # pyright: ignore[reportUnknownArgumentType] # a request value
|
||||
for key, value in request_kwargs.items()
|
||||
}
|
||||
|
||||
|
||||
def organization_deployment_id(
|
||||
declared_id: str | None, position: int, organization: object, generated: Callable[[], str]
|
||||
) -> str:
|
||||
"""
|
||||
The id of one deployment a ``litellm_params.organization`` list fans out into: each gets its own.
|
||||
A declared id stays on the first organization's deployment (so lookups by it keep working) and is
|
||||
suffixed with the organization on the others; without one, ``generated`` hashes that
|
||||
organization's own params.
|
||||
"""
|
||||
if declared_id is None:
|
||||
return generated()
|
||||
return declared_id if position == 0 else f"{declared_id}-{organization}"
|
||||
|
||||
|
||||
def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore:
|
||||
"""
|
||||
The Rust core when the route catalog's router rule allows it and the Rust core serves this
|
||||
|
|
|
|||
|
|
@ -141,7 +141,10 @@ class _DeprecatedAttribute(Generic[_T]):
|
|||
|
||||
def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None:
|
||||
_note(self.name, self.replacement)
|
||||
if self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES:
|
||||
if self.name == "model_list":
|
||||
# A whole new list is reindexed, so a replaced id never finds the new list's entries.
|
||||
_as_router(obj)._assign_model_list(value) # pyright: ignore[reportArgumentType] # the assigned list
|
||||
elif self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES:
|
||||
_as_router(obj).apply(ChangeSet(assign={self.name: value}))
|
||||
else:
|
||||
setattr(obj, "_core_" + self.name, value)
|
||||
|
|
@ -170,6 +173,15 @@ def _attr(replacement: str, read: Callable[["Router"], _T]) -> _DeprecatedAttrib
|
|||
return _DeprecatedAttribute(replacement, read)
|
||||
|
||||
|
||||
def _select_in_place(router: "Router") -> Callable[..., Coroutine[object, object, object]]:
|
||||
"""
|
||||
The core's selection on the caller's own ``request_kwargs``: the old members changed them in
|
||||
place (for example taking ``_retry_skipped_deployment_ids`` off), where ``select`` works on a copy.
|
||||
"""
|
||||
core: Final[object] = router.__dict__.get("_core", router) # pyright: ignore[reportAny] # the facade keeps its core in __dict__
|
||||
return core._select_in_place # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] # every core has one
|
||||
|
||||
|
||||
def _async_picker(router: "Router") -> Callable[..., Awaitable[object]]:
|
||||
"""
|
||||
The router's async pick (the old ``async_get_available_deployment`` attribute). A core that
|
||||
|
|
@ -187,7 +199,7 @@ def _async_picker(router: "Router") -> Callable[..., Awaitable[object]]:
|
|||
input: str | list | None = None, # mutable-ok: the old parameter type, unchanged
|
||||
specific_deployment: bool | None = False,
|
||||
) -> object:
|
||||
return await router.select(
|
||||
return await _select_in_place(router)(
|
||||
model, request_kwargs, messages=messages, input=input, specific_deployment=specific_deployment
|
||||
)
|
||||
|
||||
|
|
@ -809,7 +821,7 @@ class DeprecatedRouterMembers:
|
|||
health_check_probe: bool = False,
|
||||
) -> list[dict] | dict: # mutable-ok: the old return type, unchanged
|
||||
_note("async_get_healthy_deployments", "select(model, request_kwargs, kind='healthy')")
|
||||
return await _as_router(self).select(
|
||||
return await _select_in_place(_as_router(self))( # pyright: ignore[reportReturnType] # the core's untyped deployment dicts
|
||||
model,
|
||||
request_kwargs,
|
||||
kind="healthy",
|
||||
|
|
@ -830,7 +842,7 @@ class DeprecatedRouterMembers:
|
|||
):
|
||||
_note("get_available_deployment_for_pass_through", "select(model, kind='pass_through')")
|
||||
return _run_ready(
|
||||
_as_router(self).select(
|
||||
_select_in_place(_as_router(self))(
|
||||
model,
|
||||
request_kwargs,
|
||||
kind="pass_through",
|
||||
|
|
|
|||
|
|
@ -124,6 +124,7 @@ from litellm.litellm_core_utils.clientside_credential_handler import (
|
|||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
coerce_token_limit,
|
||||
get_litellm_metadata_from_kwargs,
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
|
|
@ -255,7 +256,11 @@ from litellm.router_api import (
|
|||
)
|
||||
from litellm.router_api import RoutingContext as RouterContext
|
||||
from litellm.router_api import deployment_id as _deployment_id
|
||||
from litellm.router_core import detached_request, organization_deployment_id
|
||||
from litellm.router_deprecated import DeprecatedRouterMembers
|
||||
from litellm.router_deprecated import (
|
||||
_note as _note_deprecated_member, # pyright: ignore[reportPrivateUsage] # the deprecated write notes its use
|
||||
)
|
||||
from litellm.router_endpoint_table import (
|
||||
ENDPOINTS,
|
||||
SYNC_DISPATCHES,
|
||||
|
|
@ -369,9 +374,14 @@ from litellm.router_utils.fallback_event_handlers import (
|
|||
run_async_fallback,
|
||||
)
|
||||
from litellm.router_utils.global_callback_holders import (
|
||||
OUTCOME_CALLBACK_LISTS,
|
||||
adaptive_routers_on_host,
|
||||
live_check_owners,
|
||||
on_host,
|
||||
routers_holding_global_callbacks,
|
||||
routers_holding_global_callbacks_lock,
|
||||
sync_adaptive_response_header_hook,
|
||||
without_router_callbacks,
|
||||
)
|
||||
from litellm.router_utils.handle_error import (
|
||||
async_raise_no_deployment_exception,
|
||||
|
|
@ -1525,6 +1535,11 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
# Settings change only through a changeset, so every core serving this router sees it.
|
||||
self.apply(ChangeSet(assign={name.removeprefix("_core_"): value}))
|
||||
return
|
||||
if name == "model_list" and "_core_model_list" in self.__dict__:
|
||||
# ``core.model_list = [...]`` on the core itself: noted and reindexed as through the Router.
|
||||
_note_deprecated_member("model_list", "registry().all()")
|
||||
self._assign_model_list(value) # pyright: ignore[reportUnknownMemberType, reportArgumentType] # the assigned list, as given
|
||||
return
|
||||
if name in _SETTING_ATTRIBUTES:
|
||||
self.__dict__.pop("_settings_snapshot", None)
|
||||
elif name == "_core_model_list" and "_core_model_list" in self.__dict__:
|
||||
|
|
@ -1589,12 +1604,37 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
"""
|
||||
Pick a deployment for ``model`` without calling it (``pick``), list the deployments that
|
||||
are healthy for the request (``healthy``), or pick one for a pass-through route
|
||||
(``pass_through``).
|
||||
(``pass_through``). The caller's ``request_kwargs`` are left unchanged: selection works on a copy.
|
||||
"""
|
||||
return await self._select_in_place( # pyright: ignore[reportUnknownVariableType] # the selection cores return untyped deployment dicts
|
||||
model,
|
||||
None if request_kwargs is None else detached_request(request_kwargs),
|
||||
kind=kind,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
parent_otel_span=parent_otel_span,
|
||||
health_check_probe=health_check_probe,
|
||||
)
|
||||
|
||||
async def _select_in_place( # pyright: ignore[reportUnknownParameterType] # the selection cores return untyped deployment dicts
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict[str, object] | None = None, # mutable-ok: pinned and stamped in place
|
||||
*,
|
||||
kind: SelectKind = "pick",
|
||||
messages: list[dict[str, str]] | None = None, # mutable-ok: the selection cores take a list
|
||||
input: str | list[object] | None = None, # mutable-ok: the selection cores take a list
|
||||
specific_deployment: bool | None = False,
|
||||
parent_otel_span: Span | None = None,
|
||||
health_check_probe: bool = False,
|
||||
) -> "list[UntypedDict] | UntypedDict": # mutable-ok: the core's type (list[dict] | dict), as select returns it
|
||||
"""``select`` on the request the router itself routes, which keeps what selection pins on it."""
|
||||
request: Final = request_kwargs if request_kwargs is not None else {}
|
||||
if kind == "pick":
|
||||
return await self._core_async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts
|
||||
model=model,
|
||||
request_kwargs=request_kwargs if request_kwargs is not None else {},
|
||||
request_kwargs=request,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
|
|
@ -1602,7 +1642,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
if kind == "healthy":
|
||||
return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts
|
||||
model=model,
|
||||
request_kwargs=request_kwargs if request_kwargs is not None else {},
|
||||
request_kwargs=request,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
|
|
@ -2065,7 +2105,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
f"default_fallbacks must be a list of model names, got '{_fallbacks}'. Use ['{_fallbacks}']"
|
||||
)
|
||||
if self._core_fallbacks is not None:
|
||||
self._core_fallbacks.append({"*": _fallbacks})
|
||||
# A new list: the configured one may be the process-wide ``litellm.fallbacks``.
|
||||
self._core_fallbacks = [*self._core_fallbacks, {"*": _fallbacks}] # pyright: ignore[reportUnknownMemberType] # the configured fallback entries
|
||||
else:
|
||||
self._core_fallbacks = [{"*": _fallbacks}]
|
||||
|
||||
|
|
@ -2224,6 +2265,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
live_check_owners.discard(self)
|
||||
self.retire_routing_state()
|
||||
self._release_global_callbacks()
|
||||
# The shared adaptive response header hook leaves with the host's last adaptive router.
|
||||
sync_adaptive_response_header_hook(self._host.callbacks, adaptive_routers_on_host(self._host))
|
||||
|
||||
def _release_global_callbacks(self) -> None:
|
||||
with _routers_holding_global_callbacks_lock:
|
||||
|
|
@ -2241,7 +2284,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
method_owners=(self, *slack_alerting_loggers),
|
||||
)
|
||||
with _routers_holding_global_callbacks_lock:
|
||||
survivors: Final = tuple(_routers_holding_global_callbacks)
|
||||
survivors: Final = on_host(tuple(_routers_holding_global_callbacks), self._host)
|
||||
candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors)
|
||||
for candidate in self._host.callbacks.with_custom_logger_keys(candidates, vacated_keys):
|
||||
self._register_router_selector(candidate)
|
||||
|
|
@ -3189,7 +3232,10 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls]
|
||||
if any(
|
||||
router is not self and any(type(cb) is callback_cls for cb in (router.optional_callbacks or []))
|
||||
for router in (*_live_routers, *(owner for owner in live_check_owners if not isinstance(owner, PythonCore)))
|
||||
for router in on_host(
|
||||
(*_live_routers, *(owner for owner in live_check_owners if not isinstance(owner, PythonCore))),
|
||||
self._host,
|
||||
)
|
||||
):
|
||||
return
|
||||
for cb in tuple(self._host.callback_list("callbacks")):
|
||||
|
|
@ -11009,7 +11055,6 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
changes, otherwise a released router keeps recording turns through its hook."""
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
AdaptiveRouterResponseHeaderHook,
|
||||
)
|
||||
|
||||
for callback in self._host.callbacks.get_custom_loggers_for_type(AdaptiveRouterPostCallHook):
|
||||
|
|
@ -11017,8 +11062,9 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
for tagged_adaptive_routers in self._core_adaptive_routers.values():
|
||||
for tagged in tagged_adaptive_routers:
|
||||
self._host.callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy))
|
||||
if self._core_adaptive_routers:
|
||||
self._host.callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook())
|
||||
sync_adaptive_response_header_hook(
|
||||
self._host.callbacks, bool(self._core_adaptive_routers) or adaptive_routers_on_host(self._host)
|
||||
)
|
||||
|
||||
def init_adaptive_router_deployment(self, deployment: Deployment) -> None:
|
||||
"""
|
||||
|
|
@ -11161,6 +11207,10 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self._core_team_model_to_deployment_indices = {} # Reset the team_model index
|
||||
self._core_team_pattern_routers = {}
|
||||
self._core_team_public_model_names = frozenset()
|
||||
# The replaced list's wildcard routes and names go with it.
|
||||
self._core_pattern_router = PatternMatchRouter()
|
||||
self._core_provider_default_deployment_ids.clear()
|
||||
self._core_deployment_names.clear()
|
||||
# Reset per-strategy router registries so hot-reload doesn't leave
|
||||
# stale routers pointing at the old model_list.
|
||||
self._core_quality_routers = {}
|
||||
|
|
@ -11214,13 +11264,24 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
if _litellm_params.get("organization", None) is not None and isinstance(
|
||||
_litellm_params["organization"], list
|
||||
): # Addresses https://github.com/BerriAI/litellm/issues/3949
|
||||
for org in _litellm_params["organization"]:
|
||||
for position, org in enumerate(tuple(_litellm_params["organization"])): # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] # configured organizations
|
||||
_litellm_params["organization"] = org
|
||||
# Each organization's deployment has its own id: a declared id stays on the
|
||||
# first and is suffixed with the organization on the rest, a generated one is
|
||||
# generated for that organization's params.
|
||||
self._create_deployment(
|
||||
deployment_info=model,
|
||||
_model_name=_model_name,
|
||||
_litellm_params=_litellm_params,
|
||||
_model_info=_model_info,
|
||||
_model_info={
|
||||
**_model_info,
|
||||
"id": organization_deployment_id(
|
||||
declared_id,
|
||||
position,
|
||||
org, # pyright: ignore[reportUnknownArgumentType] # a configured organization
|
||||
lambda: self._core_generate_model_id(_model_name, _litellm_params), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # the configured params
|
||||
),
|
||||
},
|
||||
declared_id=declared_id,
|
||||
duplicate_ids=duplicate_ids,
|
||||
)
|
||||
|
|
@ -11520,6 +11581,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
|
||||
self._core_pattern_router.remove_deployment(model_id)
|
||||
if model_id in self._core_provider_default_deployment_ids:
|
||||
self._core_provider_default_deployment_ids.remove(model_id)
|
||||
for team_id in list(self._core_team_pattern_routers.keys()):
|
||||
team_pattern_router = self._core_team_pattern_routers[team_id]
|
||||
team_pattern_router.remove_deployment(model_id)
|
||||
|
|
@ -11642,6 +11705,10 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
return None
|
||||
else:
|
||||
# The failed edit raises, but the deployment it replaced keeps serving.
|
||||
self._restore_deployment_after_failed_upsert(
|
||||
previous_deployment=_deployment_on_router, model_id=_deployment_model_id
|
||||
)
|
||||
raise e
|
||||
|
||||
def _restore_deployment_after_failed_upsert(self, previous_deployment: Deployment | None, model_id: str) -> None:
|
||||
|
|
@ -13176,6 +13243,28 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
apply_remaining_usage_headers(additional_headers, remaining_usage)
|
||||
return response
|
||||
|
||||
def _assign_model_list(self, model_list: list[Mapping[str, object]]) -> None: # mutable-ok: kept as assigned
|
||||
"""
|
||||
``router.model_list = [...]`` (deprecated): the list is swapped as given and reindexed, so a
|
||||
replaced deployment's id, name and team entries stop finding the new list's entries. The
|
||||
registry version moves once.
|
||||
"""
|
||||
with self._deployment_index_lock:
|
||||
self._core_model_list = model_list
|
||||
self.model_id_to_deployment_index_map = {
|
||||
str(model_id): idx
|
||||
for idx, entry in enumerate(model_list)
|
||||
if (model_id := (as_str_mapping(entry.get("model_info")) or {}).get("id")) is not None
|
||||
}
|
||||
self._build_model_name_index(model_list) # pyright: ignore[reportUnknownMemberType] # the untyped index builder
|
||||
self._core_model_names = {
|
||||
str(entry["model_name"])
|
||||
for entry in model_list
|
||||
if entry.get("model_name") and (as_str_mapping(entry.get("model_info")) or {}).get("id") is not None
|
||||
}
|
||||
self._access_groups_cache = None
|
||||
self._invalidate_model_group_info_cache()
|
||||
|
||||
def _build_model_name_index(self, model_list: list) -> None:
|
||||
"""
|
||||
Build model_name -> deployment indices mapping for O(1) lookups.
|
||||
|
|
@ -13188,6 +13277,10 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self._core_team_public_model_names = frozenset()
|
||||
|
||||
for idx, model in enumerate(model_list):
|
||||
if (as_str_mapping(model.get("model_info")) or {}).get("id") is None: # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # an entry of the untyped list
|
||||
# An entry without an id (only ``router.model_list = [...]`` keeps one as given) is
|
||||
# never served, as before that assignment reindexed: routing keys on the id.
|
||||
continue
|
||||
model_name = model.get("model_name")
|
||||
if model_name:
|
||||
if model_name not in self._core_model_name_to_deployment_indices:
|
||||
|
|
@ -17235,11 +17328,24 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self._core_prompt_caching_pin_cache.in_memory_cache.flush_cache()
|
||||
|
||||
def reset(self):
|
||||
## clean up on close
|
||||
self._host.set_callback_list("success_callback", [])
|
||||
self._host.set_callback_list("_async_success_callback", [])
|
||||
self._host.set_callback_list("failure_callback", [])
|
||||
self._host.set_callback_list("_async_failure_callback", [])
|
||||
## clean up on close: drop this router's own success/failure callbacks, never another's
|
||||
slack_alerting_loggers: Final = self._slack_alerting_loggers()
|
||||
owned: Final = ( # pyright: ignore[reportUnknownVariableType] # the router's callback objects
|
||||
*self._callback_selectors(),
|
||||
*self._override_selectors.values(),
|
||||
*(self.optional_callbacks or ()), # pyright: ignore[reportUnknownMemberType] # the router's checks
|
||||
*slack_alerting_loggers,
|
||||
*self._adaptive_router_hooks(),
|
||||
)
|
||||
for name in OUTCOME_CALLBACK_LISTS:
|
||||
self._host.set_callback_list(
|
||||
name,
|
||||
without_router_callbacks(
|
||||
self._host.callback_list(name),
|
||||
owned, # pyright: ignore[reportUnknownArgumentType] # see above
|
||||
method_owners=(self, *slack_alerting_loggers),
|
||||
),
|
||||
)
|
||||
self.retry_policy = None
|
||||
self.flush_cache()
|
||||
|
||||
|
|
|
|||
|
|
@ -28,9 +28,14 @@ from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy
|
|||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
|
||||
from litellm.router_utils.global_callback_holders import (
|
||||
OUTCOME_CALLBACK_LISTS,
|
||||
adaptive_routers_on_host,
|
||||
live_check_owners,
|
||||
on_host,
|
||||
routers_holding_global_callbacks,
|
||||
routers_holding_global_callbacks_lock,
|
||||
sync_adaptive_response_header_hook,
|
||||
without_router_callbacks,
|
||||
)
|
||||
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
|
||||
DeploymentAffinityCheck,
|
||||
|
|
@ -279,7 +284,7 @@ class OptionalChecks:
|
|||
if self.optional_callbacks is None or not any(type(cb) is callback_cls for cb in self.optional_callbacks):
|
||||
return
|
||||
self.optional_callbacks = [cb for cb in self.optional_callbacks if type(cb) is not callback_cls]
|
||||
others: Final = tuple(owner for owner in live_check_owners if id(owner) != id(self._owner))
|
||||
others: Final = tuple(owner for owner in on_host(live_check_owners, self._host) if id(owner) != id(self._owner))
|
||||
if any(any(type(cb) is callback_cls for cb in (owner.optional_callbacks or ())) for owner in others):
|
||||
return
|
||||
callbacks: Final = _callbacks(self._host)
|
||||
|
|
@ -426,7 +431,7 @@ class OptionalChecks:
|
|||
(*dropped, *(self.optional_callbacks or ()), *slack, *adaptive), method_owners=(self._owner, *slack)
|
||||
)
|
||||
with routers_holding_global_callbacks_lock:
|
||||
survivors: Final = tuple(routers_holding_global_callbacks)
|
||||
survivors: Final = on_host(tuple(routers_holding_global_callbacks), self._host)
|
||||
candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors) # pyright: ignore[reportPrivateUsage] # the holder protocol
|
||||
for candidate in self._host.callbacks.with_custom_logger_keys(candidates, vacated):
|
||||
register_router_selector(self._host, candidate)
|
||||
|
|
@ -521,6 +526,22 @@ class RustOptionalChecks:
|
|||
selectors.callback_selectors(), (*selectors.callback_selectors(), *selectors.override_selectors())
|
||||
)
|
||||
|
||||
def _release_outcome_callbacks(self) -> None:
|
||||
"""``reset``'s callback half: this core's own entries leave the host's success and failure
|
||||
lists; every other entry stays."""
|
||||
checks: Final = self._opt
|
||||
slack: Final = checks.slack_alerting_loggers
|
||||
owned: Final = (
|
||||
*self._sel.callback_selectors(),
|
||||
*self._sel.override_selectors(),
|
||||
*(checks.optional_callbacks or ()),
|
||||
*slack,
|
||||
*checks.own_adaptive_hooks(),
|
||||
)
|
||||
host: Final = checks._host # pyright: ignore[reportPrivateUsage] # the checks' host
|
||||
for name in OUTCOME_CALLBACK_LISTS:
|
||||
host.set_callback_list(name, without_router_callbacks(host.callback_list(name), owned, (self, *slack)))
|
||||
|
||||
def _retire_callbacks(self) -> None:
|
||||
"""``_core_discard``'s callback half: retire the routing state, release the callbacks."""
|
||||
live_check_owners.discard(self)
|
||||
|
|
@ -528,6 +549,9 @@ class RustOptionalChecks:
|
|||
self._opt.retire_state(selectors)
|
||||
self._opt.release(selectors)
|
||||
self._sel.clear()
|
||||
# The shared adaptive response header hook leaves with the host's last adaptive router.
|
||||
host: Final = self._opt._host # pyright: ignore[reportPrivateUsage] # the checks' host
|
||||
sync_adaptive_response_header_hook(host.callbacks, adaptive_routers_on_host(host))
|
||||
|
||||
async def _answer_callback_filters(
|
||||
self, step: Mapping[str, object], request: Mapping[str, object], messages: object
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ from litellm.router_api import (
|
|||
)
|
||||
from litellm.router_attempt import for_requested_model, request_from_call
|
||||
from litellm.router_config import RouterConfig, RouterDynamicConfig, detached
|
||||
from litellm.router_core import detached_request
|
||||
from litellm.router_python_host import PythonHost
|
||||
from litellm.router_rust_access import caller_fields, selection_refusal
|
||||
from litellm.router_rust_alerting import initialize_alerting
|
||||
|
|
@ -528,8 +529,8 @@ class Resolved:
|
|||
@classmethod
|
||||
def of(cls, config: RouterConfig, dynamic: RouterDynamicConfig) -> "Resolved":
|
||||
get: Final = config.get
|
||||
# The process list is the router's own, as in the Python router: default_fallbacks are
|
||||
# appended to it in place.
|
||||
# The process list is the router's own, as in the Python router; default_fallbacks extend
|
||||
# a new list, never the process-wide ``litellm.fallbacks``.
|
||||
configured: Final[list[object] | None] = ( # mutable-ok: the router's live list, as Python keeps it
|
||||
list(_as_list(get("fallbacks"))) or (dynamic.fallbacks if dynamic.fallbacks else None) # pyright: ignore[reportAssignmentType] # the process list as given
|
||||
)
|
||||
|
|
@ -539,10 +540,10 @@ class Resolved:
|
|||
raise TypeError(
|
||||
f"default_fallbacks must be a list of model names, got '{default_fallbacks}'. Use ['{default_fallbacks}']"
|
||||
)
|
||||
if default_fallbacks is not None and configured is not None:
|
||||
configured.append({"*": default_fallbacks})
|
||||
fallbacks: Final[list[object] | None] = ( # mutable-ok: see above
|
||||
[{"*": default_fallbacks}] if default_fallbacks is not None and configured is None else configured
|
||||
configured
|
||||
if default_fallbacks is None
|
||||
else [*(configured if configured is not None else ()), {"*": default_fallbacks}]
|
||||
)
|
||||
content_policy: Final = list(_as_list(get("content_policy_fallbacks"))) or (
|
||||
list(dynamic.content_policy_fallbacks) if dynamic.content_policy_fallbacks else None
|
||||
|
|
@ -1083,7 +1084,7 @@ class RustCore(
|
|||
self._handle_default_ids: tuple[str, ...] = () # the provider default ids the handle was built with
|
||||
self._hosted.max_parallel_requests_lookup = self._max_parallel_requests_client
|
||||
self._hosted._core_routing_plugins = list(resolved.routing_plugins) # pyright: ignore[reportPrivateUsage] # the core fills its own host registry
|
||||
self._handle: RouterCoreHandle = self._new_handle(list(self._hosted._core_model_list)) # pyright: ignore[reportPrivateUsage] # the core reads its own host registry
|
||||
self._handle: RouterCoreHandle = self._new_handle(self._hosted.served_deployments())
|
||||
self._join_callback_holders()
|
||||
self._checks: Final = OptionalChecks(self, host, self._cache, self._hosted, ctx, config.arguments)
|
||||
self._hosted.checks = self._checks
|
||||
|
|
@ -1345,7 +1346,7 @@ class RustCore(
|
|||
def _rebuild(self) -> None:
|
||||
"""A new Rust core for changed static settings, over the same deployments."""
|
||||
old: Final = self._handle
|
||||
self._handle = self._new_handle(list(self._hosted._core_model_list)) # pyright: ignore[reportPrivateUsage] # see __init__
|
||||
self._handle = self._new_handle(self._hosted.served_deployments())
|
||||
old.retire()
|
||||
# The new handle builds its indexes from the deployments: written ones are handed again.
|
||||
self._index_sent = None
|
||||
|
|
@ -1357,7 +1358,7 @@ class RustCore(
|
|||
self._tag_filtering_models = None
|
||||
handle: Final[object] = self.__dict__.get("_handle")
|
||||
if isinstance(handle, RouterCoreHandle):
|
||||
_ = handle.apply({"replace": _plain(list(self._hosted._core_model_list))}) # pyright: ignore[reportPrivateUsage] # see __init__
|
||||
_ = handle.apply({"replace": _plain(self._hosted.served_deployments())})
|
||||
# A wildcard deployment added an id the cooldown check skips.
|
||||
if self._provider_default_ids() != self._handle_default_ids:
|
||||
self._rebuild()
|
||||
|
|
@ -1723,10 +1724,9 @@ class RustCore(
|
|||
self._checks.prompt_caching_pin_cache.in_memory_cache.flush_cache()
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Empty the host's success and failure callback lists (every entry, not only this core's),
|
||||
drop the retry policy and flush the caches."""
|
||||
for name in ("success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"):
|
||||
self._host.set_callback_list(name, [])
|
||||
"""Drop this core's own entries from the host's success and failure callback lists (other
|
||||
code's callbacks stay), drop the retry policy and flush the caches."""
|
||||
self._release_outcome_callbacks()
|
||||
self._plain["retry_policy"] = None
|
||||
self._set_retry_policies({"retry_policy": None})
|
||||
self.flush_cache()
|
||||
|
|
@ -1753,6 +1753,11 @@ class RustCore(
|
|||
if name in PLAIN_SETTING_ATTRIBUTES or name in SETTING_FIELDS or name in ASSIGNABLE_ALIASES:
|
||||
_ = self.apply(ChangeSet(assign={name: value}))
|
||||
elif name == "model_list":
|
||||
from litellm.router_deprecated import (
|
||||
_note, # pyright: ignore[reportPrivateUsage] # the deprecated write notes its use like Python's
|
||||
)
|
||||
|
||||
_note("model_list", "registry().all()")
|
||||
self._assign_model_list(_as_list(value))
|
||||
elif name in _HOSTED_STATE:
|
||||
# A deprecated attribute write of the registry state: it lands where this core keeps it.
|
||||
|
|
@ -1810,7 +1815,31 @@ class RustCore(
|
|||
) -> Mapping[str, object] | Sequence[Mapping[str, object]]:
|
||||
"""Pick a deployment for ``model`` without calling it (``pick``), list the deployments a
|
||||
pick would choose among (``healthy``), or pick one for a pass-through route
|
||||
(``pass_through``, which picks with the sync strategies, as Python does)."""
|
||||
(``pass_through``, which picks with the sync strategies, as Python does). The caller's
|
||||
``request_kwargs`` are left unchanged: selection works on a copy."""
|
||||
return await self._select_in_place(
|
||||
model,
|
||||
None if request_kwargs is None else detached_request(request_kwargs),
|
||||
kind=kind,
|
||||
specific_deployment=specific_deployment,
|
||||
messages=messages,
|
||||
input=input,
|
||||
health_check_probe=health_check_probe,
|
||||
)
|
||||
|
||||
async def _select_in_place(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict[str, object] | None = None, # mutable-ok: pinned and stamped in place
|
||||
*,
|
||||
kind: str = "pick",
|
||||
specific_deployment: bool | None = None,
|
||||
messages: object = None,
|
||||
input: object = None,
|
||||
health_check_probe: bool = False,
|
||||
**_: object, # kwargs-ok: the RouterCore select options this core ignores
|
||||
) -> Mapping[str, object] | Sequence[Mapping[str, object]]:
|
||||
"""``select`` on the request the core itself routes, which keeps what selection pins on it."""
|
||||
if kind not in ("pick", "healthy", "pass_through"):
|
||||
raise ValueError(f"unknown select kind {kind!r}")
|
||||
request: Final = request_kwargs if request_kwargs is not None else {}
|
||||
|
|
|
|||
|
|
@ -1331,7 +1331,7 @@ class RustEndpoints:
|
|||
assert isinstance(scheduler, Scheduler)
|
||||
|
||||
async def healthy_deployments() -> Sequence[object]:
|
||||
healthy: Final = await self._core.select(model, dict(request), kind="healthy")
|
||||
healthy: Final = await self._core._select_in_place(model, dict(request), kind="healthy") # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core
|
||||
return list(healthy) if isinstance(healthy, Sequence) else [healthy]
|
||||
|
||||
await scheduler.wait_for_turn(
|
||||
|
|
@ -1584,7 +1584,7 @@ class RustEndpoints:
|
|||
metadata_variable_name: Final = get_router_metadata_variable_name(function_name="aretrieve_batch")
|
||||
if model is not None:
|
||||
specific: Final = kwargs.pop("specific_deployment", None)
|
||||
healthy = await self._core.select( # rebind-ok: rebound as the Python router rebinds it
|
||||
healthy = await self._core._select_in_place( # pyright: ignore[reportPrivateUsage] # rebind-ok: rebound as the Python router rebinds it
|
||||
model, dict(kwargs), kind="healthy", specific_deployment=bool(specific) or None
|
||||
)
|
||||
else:
|
||||
|
|
@ -1674,7 +1674,7 @@ class RustEndpoints:
|
|||
)
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
healthy: Final = await self._core.select(
|
||||
healthy: Final = await self._core._select_in_place( # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core
|
||||
requested, dict(kwargs), kind="healthy", specific_deployment=bool(specific) or None
|
||||
)
|
||||
deployments: Final = list(healthy) if isinstance(healthy, Sequence) else [healthy]
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ class RustRegistryMembers:
|
|||
return self._registry_state.add_deployment(deployment)
|
||||
|
||||
def set_model_list(self, model_list: Sequence[object]) -> None:
|
||||
"""Replace every deployment (``set_model_list``); aliases and wildcard routes stay."""
|
||||
"""Replace every deployment (``set_model_list``) with its wildcard routes and names; aliases stay."""
|
||||
self._registry_state.set_model_list(model_list)
|
||||
|
||||
@property
|
||||
|
|
@ -46,11 +46,9 @@ class RustRegistryMembers:
|
|||
return self._registry_state._core_model_list # pyright: ignore[reportPrivateUsage] # the core reads its own host registry
|
||||
|
||||
def _assign_model_list(self, model_list: Sequence[object]) -> None:
|
||||
"""``router.model_list = [...]``: the list is swapped as given, without reindexing, and the
|
||||
"""``router.model_list = [...]``: the list is swapped as given and reindexed, and the
|
||||
registry version moves once."""
|
||||
self._registry_state._core_model_list = model_list # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue] # the list as assigned, as the core keeps it
|
||||
self._registry_state._registry_version += 1 # pyright: ignore[reportPrivateUsage] # see above
|
||||
self._registry_state._on_change() # pyright: ignore[reportPrivateUsage] # see above
|
||||
self._registry_state.assign_model_list(model_list)
|
||||
|
||||
@staticmethod
|
||||
def deployment_is_active_for_environment(deployment: Deployment) -> bool:
|
||||
|
|
|
|||
|
|
@ -484,9 +484,10 @@ class PreRoutingStrategies:
|
|||
|
||||
def _sync_adaptive_router_hooks(self) -> None:
|
||||
"""One post-call hook per registered adaptive router, plus the header hook while any exists."""
|
||||
from litellm.router_strategy.adaptive_router.hooks import (
|
||||
AdaptiveRouterPostCallHook,
|
||||
AdaptiveRouterResponseHeaderHook,
|
||||
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
|
||||
from litellm.router_utils.global_callback_holders import (
|
||||
adaptive_routers_on_host,
|
||||
sync_adaptive_response_header_hook,
|
||||
)
|
||||
|
||||
callbacks: Final = self._host.callbacks
|
||||
|
|
@ -494,8 +495,7 @@ class PreRoutingStrategies:
|
|||
callbacks.remove_callback_from_all_lists(callback)
|
||||
for tagged in chain.from_iterable(self.adaptive.values()):
|
||||
callbacks.add_litellm_callback(AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy))
|
||||
if self.adaptive:
|
||||
callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook())
|
||||
sync_adaptive_response_header_hook(callbacks, bool(self.adaptive) or adaptive_routers_on_host(self._host))
|
||||
|
||||
|
||||
# -- the request side --------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ async def prompt_management_acompletion(
|
|||
caller_timeout: Final = kwargs.get("timeout") or kwargs.get("request_timeout")
|
||||
caller_stream_timeout: Final = kwargs.get("stream_timeout")
|
||||
specific_deployment: Final = kwargs.pop("specific_deployment", None)
|
||||
picked: Final = await core.select(
|
||||
picked: Final = await core._select_in_place( # pyright: ignore[reportPrivateUsage] # the core's own selection, which pins onto the request
|
||||
model,
|
||||
kwargs, # pyright: ignore[reportUnknownArgumentType] # the request kwargs
|
||||
kind="pick",
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
|||
from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS
|
||||
from litellm.responses.bridge_check import keep_registered_mode
|
||||
from litellm.router_api import RouterRegistry, deployment_id
|
||||
from litellm.router_core import organization_deployment_id
|
||||
from litellm.router_rust_model_info import HostModelInfo
|
||||
from litellm.router_rust_shaping import ROUTER_CLIENTS_TARGET
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model
|
||||
|
|
@ -632,6 +633,10 @@ class HostRegistry(HostModelInfo):
|
|||
self._core_team_model_to_deployment_indices = {}
|
||||
self._core_team_pattern_routers = {}
|
||||
self._core_team_public_model_names = frozenset()
|
||||
# The replaced list's wildcard routes and names go with it.
|
||||
self._core_pattern_router = PatternMatchRouter()
|
||||
self._core_provider_default_deployment_ids.clear()
|
||||
self._core_deployment_names.clear()
|
||||
if self.strategies is not None:
|
||||
self.strategies.reset()
|
||||
self._provider_unresolved_deployments = ()
|
||||
|
|
@ -666,10 +671,27 @@ class HostRegistry(HostModelInfo):
|
|||
if "id" not in info_dict:
|
||||
info_dict["id"] = deployment_id(str(model_name), litellm_params)
|
||||
organization = litellm_params.get("organization")
|
||||
organizations: Sequence[object] = organization if isinstance(organization, list) else (None,)
|
||||
for org in organizations:
|
||||
if org is not None:
|
||||
if isinstance(organization, list):
|
||||
# Each organization's deployment has its own id (``organization_deployment_id``).
|
||||
for position, org in enumerate(tuple(organization)): # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # configured organizations
|
||||
litellm_params["organization"] = org
|
||||
self._create_deployment(
|
||||
model,
|
||||
str(model_name),
|
||||
litellm_params,
|
||||
{
|
||||
**info_dict,
|
||||
"id": organization_deployment_id(
|
||||
declared_id,
|
||||
position,
|
||||
org,
|
||||
lambda: deployment_id(str(model_name), litellm_params),
|
||||
),
|
||||
},
|
||||
declared_id=declared_id,
|
||||
duplicate_ids=duplicate_ids,
|
||||
)
|
||||
else:
|
||||
self._create_deployment(
|
||||
model,
|
||||
str(model_name),
|
||||
|
|
@ -865,6 +887,40 @@ class HostRegistry(HostModelInfo):
|
|||
vertex_credentials=credentials, # pyright: ignore[reportArgumentType] # the configured credentials
|
||||
)
|
||||
|
||||
def served_deployments(self) -> list[UntypedEntry]: # mutable-ok: a fresh list for the Rust handle
|
||||
"""The deployments the Rust handle routes over: every entry with an id. Only
|
||||
``router.model_list = [...]`` keeps an entry without one, and that entry is never served."""
|
||||
return [entry for entry in self._core_model_list if _model_info_of(entry).get("id") is not None]
|
||||
|
||||
def assign_model_list(self, model_list: Sequence[object]) -> None:
|
||||
"""``router.model_list = [...]`` (deprecated, ``_assign_model_list``): the list is swapped as
|
||||
given and reindexed, and the registry version moves once."""
|
||||
entries: Final = [_LITELLM_PARAMS_ADAPTER.validate_python(entry) for entry in model_list]
|
||||
with self._deployment_index_lock:
|
||||
self._core_model_list = model_list # pyright: ignore[reportAttributeAccessIssue] # the list as assigned, as the core keeps it
|
||||
self.model_id_to_deployment_index_map = {
|
||||
str(model_id): idx
|
||||
for idx, entry in enumerate(entries)
|
||||
if (model_id := _model_info_of(entry).get("id")) is not None
|
||||
}
|
||||
self._core_model_name_to_deployment_indices = {}
|
||||
self._core_team_model_to_deployment_indices = {}
|
||||
self._core_team_public_model_names = frozenset()
|
||||
# An entry without an id is kept as given but never served, as on the Python core.
|
||||
served: Final = [
|
||||
(idx, entry) for idx, entry in enumerate(entries) if _model_info_of(entry).get("id") is not None
|
||||
]
|
||||
for idx, entry in served:
|
||||
model_name = entry.get("model_name")
|
||||
if model_name:
|
||||
self._core_model_name_to_deployment_indices.setdefault(str(model_name), []).append(idx)
|
||||
self._update_team_model_index(entry, idx)
|
||||
self._core_model_names = {str(entry["model_name"]) for _, entry in served if entry.get("model_name")}
|
||||
self._access_groups_cache = None
|
||||
self._invalidate_model_group_info_cache()
|
||||
self._registry_version += 1
|
||||
self._on_change()
|
||||
|
||||
def _update_team_model_index(self, model: Mapping[str, object], idx: int) -> None:
|
||||
info: Final = _model_info_of(model)
|
||||
team_id: Final = info.get("team_id")
|
||||
|
|
@ -915,6 +971,8 @@ class HostRegistry(HostModelInfo):
|
|||
public_model_name for _, public_model_name in self._core_team_model_to_deployment_indices
|
||||
)
|
||||
self._core_pattern_router.remove_deployment(model_id)
|
||||
if model_id in self._core_provider_default_deployment_ids:
|
||||
self._core_provider_default_deployment_ids.remove(model_id)
|
||||
for team_id in list(self._core_team_pattern_routers):
|
||||
team_router = self._core_team_pattern_routers[team_id]
|
||||
team_router.remove_deployment(model_id)
|
||||
|
|
@ -972,6 +1030,8 @@ class HostRegistry(HostModelInfo):
|
|||
self.strategies.upserted(deployment, on_router)
|
||||
except Exception as e:
|
||||
if not self.ignore_invalid_deployments:
|
||||
# The failed edit raises, but the deployment it replaced keeps serving.
|
||||
self._restore_deployment_after_failed_upsert(on_router, model_id)
|
||||
raise
|
||||
verbose_router_logger.warning(
|
||||
"Error upserting deployment %s (id=%s): %s. Dropping it and continuing with other deployments.",
|
||||
|
|
|
|||
|
|
@ -5,8 +5,9 @@ serves them.
|
|||
A router that retires hands the checks it vacates in its host's ``callbacks`` list to a surviving
|
||||
router's same-keyed check (``routers_holding_global_callbacks``), and a router that drops a check at
|
||||
runtime keeps it registered while another live router still has one of that type
|
||||
(``live_check_owners``). Both rules look at every router in the process, so the sets live here,
|
||||
outside any one router core.
|
||||
(``live_check_owners``). The sets hold every router in the process, whatever core serves it, so
|
||||
they live here, outside any one router core; both rules only ever consult the routers on the same
|
||||
host (``on_host``), since another host's lists are not this host's.
|
||||
|
||||
A member of ``routers_holding_global_callbacks`` serves ``_global_callback_candidates()``,
|
||||
``_core_adaptive_routers`` and ``_sync_adaptive_router_hooks()``; a member of ``live_check_owners``
|
||||
|
|
@ -15,8 +16,9 @@ serves ``optional_callbacks``.
|
|||
|
||||
import threading
|
||||
import weakref
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from types import MethodType
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeVar
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -40,14 +42,82 @@ class CheckOwner(Protocol):
|
|||
def optional_callbacks(self) -> "Sequence[object] | None": ...
|
||||
|
||||
|
||||
_Router: Final = TypeVar("_Router")
|
||||
|
||||
routers_holding_global_callbacks: Final["weakref.WeakSet[GlobalCallbackHolder]"] = weakref.WeakSet()
|
||||
routers_holding_global_callbacks_lock: Final = threading.Lock()
|
||||
live_check_owners: Final["weakref.WeakSet[CheckOwner]"] = weakref.WeakSet()
|
||||
|
||||
OUTCOME_CALLBACK_LISTS: Final = (
|
||||
"success_callback",
|
||||
"_async_success_callback",
|
||||
"failure_callback",
|
||||
"_async_failure_callback",
|
||||
)
|
||||
|
||||
|
||||
def without_router_callbacks(
|
||||
callbacks: Iterable[object], owned: Iterable[object], method_owners: Iterable[object]
|
||||
) -> list[object]: # mutable-ok: a fresh callback list for the host
|
||||
"""``callbacks`` without a router's own entries: the objects in ``owned`` and the bound methods of
|
||||
``method_owners``. Everything other code registered stays."""
|
||||
owned_ids: Final = frozenset(map(id, owned))
|
||||
owner_ids: Final = frozenset(map(id, method_owners))
|
||||
return [
|
||||
callback
|
||||
for callback in callbacks
|
||||
if id(callback) not in owned_ids
|
||||
and not (isinstance(callback, MethodType) and id(callback.__self__) in owner_ids)
|
||||
]
|
||||
|
||||
|
||||
def on_host(routers: Iterable[_Router], host: object) -> tuple[_Router, ...]:
|
||||
"""The routers among ``routers`` built on ``host``: a retiring router hands its vacated checks
|
||||
only to these, and a dropped check stays registered only for these."""
|
||||
return tuple(owner for owner in routers if getattr(owner, "_host", None) is host)
|
||||
|
||||
|
||||
class _HeaderHookCallbacks(Protocol):
|
||||
"""The host callback registration the adaptive response header hook goes through."""
|
||||
|
||||
def get_custom_loggers_for_type(
|
||||
self, callback_type: "type[CustomLogger]"
|
||||
) -> "list[CustomLogger]": ... # mutable-ok: the manager's own return type
|
||||
|
||||
def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None: ...
|
||||
|
||||
def add_litellm_callback(self, callback: "CustomLogger") -> None: ...
|
||||
|
||||
|
||||
def adaptive_routers_on_host(host: object) -> bool:
|
||||
"""Whether a live router built on ``host`` (request-scoped ones included) has an adaptive router."""
|
||||
return any(getattr(owner, "_core_adaptive_routers", None) for owner in on_host(tuple(live_check_owners), host))
|
||||
|
||||
|
||||
def sync_adaptive_response_header_hook(callbacks: _HeaderHookCallbacks, wanted: bool) -> None:
|
||||
"""
|
||||
Exactly one ``AdaptiveRouterResponseHeaderHook`` on the host while ``wanted`` (some router on it has
|
||||
an adaptive router), none otherwise: the hook is stateless and shared by every router on the host,
|
||||
so it is added once, never per sync, and leaves with the last adaptive router.
|
||||
"""
|
||||
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterResponseHeaderHook
|
||||
|
||||
present: Final = callbacks.get_custom_loggers_for_type(AdaptiveRouterResponseHeaderHook)
|
||||
for extra in present[1 if wanted else 0 :]:
|
||||
callbacks.remove_callback_from_all_lists(extra)
|
||||
if wanted and not present:
|
||||
callbacks.add_litellm_callback(AdaptiveRouterResponseHeaderHook())
|
||||
|
||||
|
||||
__all__ = [
|
||||
"OUTCOME_CALLBACK_LISTS",
|
||||
"CheckOwner",
|
||||
"GlobalCallbackHolder",
|
||||
"adaptive_routers_on_host",
|
||||
"live_check_owners",
|
||||
"on_host",
|
||||
"routers_holding_global_callbacks",
|
||||
"routers_holding_global_callbacks_lock",
|
||||
"sync_adaptive_response_header_hook",
|
||||
"without_router_callbacks",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -4238,12 +4238,19 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
|
||||
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"})
|
||||
|
||||
# A deployment's declared context window is its own: written under the shared backend key it would
|
||||
# outlive the router and narrow every other router's deployment of the same model.
|
||||
DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS: Final[frozenset[str]] = frozenset(
|
||||
{"max_input_tokens", "max_output_tokens", "max_tokens"}
|
||||
)
|
||||
|
||||
SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = (
|
||||
frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__)
|
||||
- frozenset(CustomPricingLiteLLMParams.model_fields)
|
||||
- DEPLOYMENT_SCOPED_PRICING_FIELDS
|
||||
- DEPLOYMENT_SCOPED_CAPABILITY_FIELDS
|
||||
- DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS
|
||||
- DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -59,7 +59,8 @@ def windowed(
|
|||
group: str, deployment_id: str, max_input_tokens: int, **params: object
|
||||
) -> dict[str, object]: # mutable-ok: a model_list entry
|
||||
"""A deployment declaring ``max_input_tokens`` under a model name no other test uses: a deployment's
|
||||
model_info is written into the process-wide cost map under its model name and id."""
|
||||
model_info is written into the process-wide cost map under its id (and its schema fields, though not
|
||||
its token limits, under its model name)."""
|
||||
model_info: Final = {"max_input_tokens": max_input_tokens, **dict(params.pop("model_info", None) or {})} # pyright: ignore[reportArgumentType]
|
||||
return deployment(
|
||||
group,
|
||||
|
|
|
|||
|
|
@ -320,16 +320,14 @@ def test_default_fallbacks_given_as_a_string_raise_type_error(make_core: MakeCor
|
|||
_one(make_core, default_fallbacks="b")
|
||||
|
||||
|
||||
def test_default_fallbacks_are_appended_to_the_process_fallbacks_list_itself(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""Oddity: with no ``fallbacks`` argument the router adopts the process list object and appends the
|
||||
star entry to it, so the process setting is changed too."""
|
||||
def test_default_fallbacks_extend_a_copy_of_the_process_fallbacks_list(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
"""With no ``fallbacks`` argument the router starts from the process list, but appends the star
|
||||
entry to its own copy, so the process setting is left alone."""
|
||||
process_fallbacks: Final[list[object]] = [{"g": ["e"]}]
|
||||
host.set_dynamic(fallbacks=process_fallbacks)
|
||||
core: Final = _one(make_core, default_fallbacks=["c"])
|
||||
assert core.settings().fallbacks == [{"g": ["e"]}, {"*": ["c"]}]
|
||||
assert process_fallbacks == [{"g": ["e"]}, {"*": ["c"]}]
|
||||
assert process_fallbacks == [{"g": ["e"]}]
|
||||
|
||||
|
||||
# --- validation --------------------------------------------------------------------------------
|
||||
|
|
@ -873,11 +871,22 @@ def test_runtime_optional_pre_call_checks_none_changes_nothing(make_core: MakeCo
|
|||
assert host_callback_names(host)["callbacks"] == ["PromptCachingDeploymentCheck"]
|
||||
|
||||
|
||||
def test_runtime_removal_keeps_the_host_callback_while_another_live_router_uses_that_check(
|
||||
def test_runtime_removal_keeps_the_host_callback_while_another_live_router_on_the_host_uses_that_check(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""Oddity: the check is left in this host's list because some other live router (here on another
|
||||
host) still has one of that type."""
|
||||
collect_other_routers()
|
||||
other: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
|
||||
core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
|
||||
core.apply(ChangeSet(settings={"optional_pre_call_checks": []}))
|
||||
assert core.optional_callbacks == [] # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
assert all(check in host.lists["callbacks"] for check in other.optional_callbacks) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
|
||||
other.retire()
|
||||
|
||||
|
||||
def test_runtime_removal_drops_the_host_callback_though_a_router_on_another_host_uses_that_check(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""Another host's router keeps its own check on its own host; it does not hold this host's."""
|
||||
collect_other_routers()
|
||||
other_host: Final = CallbackHost()
|
||||
from tests.unit.test_router.core.scripted_host import python_core
|
||||
|
|
@ -888,7 +897,8 @@ def test_runtime_removal_keeps_the_host_callback_while_another_live_router_uses_
|
|||
core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
|
||||
core.apply(ChangeSet(settings={"optional_pre_call_checks": []}))
|
||||
assert core.optional_callbacks == [] # pyright: ignore[reportAttributeAccessIssue]
|
||||
assert host_callback_names(host)["callbacks"] == ["PromptCachingDeploymentCheck"]
|
||||
assert host_callback_names(host)["callbacks"] == []
|
||||
assert host_callback_names(other_host)["callbacks"] == ["PromptCachingDeploymentCheck"]
|
||||
other.retire()
|
||||
|
||||
|
||||
|
|
@ -946,23 +956,35 @@ def test_retiring_one_router_keeps_the_callbacks_of_another_on_the_same_host(
|
|||
assert all(callback.__self__ is second for callback in host.lists["_async_success_callback"]) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
|
||||
def test_retire_hands_a_vacated_check_to_another_live_router_on_a_different_host(
|
||||
def test_retire_never_hands_a_vacated_check_to_a_live_router_on_a_different_host(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""Oddity: retire re-registers the same-keyed check of any other live router in the process into
|
||||
this host's list, even one built on a different host."""
|
||||
"""Another host's router keeps its check on its own host; this host's list is left empty."""
|
||||
collect_other_routers()
|
||||
from tests.unit.test_router.core.scripted_host import python_core
|
||||
|
||||
other_host: Final = CallbackHost()
|
||||
other: Final = python_core(
|
||||
CallbackHost(), model_list=[deployment("g", "o")], optional_pre_call_checks=["prompt_caching"]
|
||||
other_host, model_list=[deployment("g", "o")], optional_pre_call_checks=["prompt_caching"]
|
||||
)
|
||||
core: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
|
||||
core.retire()
|
||||
assert host.lists["callbacks"] == other.optional_callbacks
|
||||
assert host.lists["callbacks"] == []
|
||||
assert other_host.lists["callbacks"] == other.optional_callbacks # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
|
||||
other.retire()
|
||||
|
||||
|
||||
def test_retire_hands_a_vacated_check_to_another_live_router_on_the_same_host(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
collect_other_routers()
|
||||
first: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
|
||||
second: Final = _one(make_core, optional_pre_call_checks=["prompt_caching"])
|
||||
first.retire()
|
||||
assert host.lists["callbacks"] == second.optional_callbacks # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
second.retire()
|
||||
|
||||
|
||||
# --- observe -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
|
@ -1131,23 +1153,42 @@ def test_flush_cache_drops_the_response_cache_routing_state_and_prompt_caching_p
|
|||
assert host_callback_names(host) == before
|
||||
|
||||
|
||||
def test_reset_empties_every_success_and_failure_list_and_the_retry_policy(
|
||||
def test_reset_drops_the_routers_own_success_and_failure_callbacks_and_the_retry_policy(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""Note: reset clears the host's success/failure lists outright, foreign callbacks included, but
|
||||
leaves ``callbacks`` and ``input_callback`` alone."""
|
||||
"""reset drops only this router's entries from the host's success/failure lists (other code's
|
||||
callbacks stay) and leaves ``callbacks`` and ``input_callback`` alone."""
|
||||
core: Final = _one(make_core, retry_policy={"RateLimitErrorRetries": 3}, routing_strategy="least-busy")
|
||||
host.lists["success_callback"].append("foreign")
|
||||
host.lists["_async_failure_callback"].append("foreign-failure")
|
||||
core.health().store.set_cache("k", 1)
|
||||
core.reset() # pyright: ignore[reportAttributeAccessIssue]
|
||||
assert {name: callbacks for name, callbacks in host_callback_names(host).items() if callbacks} == {
|
||||
"callbacks": ["LeastBusyLoggingHandler"],
|
||||
"input_callback": ["LeastBusyLoggingHandler"],
|
||||
"success_callback": ["str"],
|
||||
"_async_failure_callback": ["str"],
|
||||
}
|
||||
assert (host.lists["success_callback"], host.lists["_async_failure_callback"]) == (["foreign"], ["foreign-failure"])
|
||||
assert core.retry_policy is None # pyright: ignore[reportAttributeAccessIssue]
|
||||
assert core.health().store.get_cache("k") is None
|
||||
|
||||
|
||||
def test_resetting_one_router_keeps_another_routers_success_and_failure_callbacks(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
collect_other_routers()
|
||||
first: Final = _one(make_core)
|
||||
before_first: Final = {name: list(callbacks) for name, callbacks in host.lists.items()}
|
||||
second: Final = _one(make_core)
|
||||
first.reset() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
for name in ("success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback"):
|
||||
remaining = host.lists.get(name, [])
|
||||
assert all(getattr(callback, "__self__", None) is not first for callback in remaining)
|
||||
theirs = [callback for callback in before_first.get(name, []) if getattr(callback, "__self__", None) is first]
|
||||
assert len(remaining) == len(theirs) and all(callback.__self__ is second for callback in remaining) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
|
||||
|
||||
# --- attribute writes ----------------------------------------------------------------------
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -291,20 +291,29 @@ async def test_an_anthropic_system_block_is_counted_against_the_context_window(m
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_context_window_is_written_into_the_process_cost_map(make_core: MakeCore) -> None:
|
||||
"""Oddity: a deployment's declared ``max_input_tokens`` replaces the process-wide cost-map entry of its
|
||||
model, so it outlives the router and narrows another router's deployment of the same model."""
|
||||
async def test_a_deployment_context_window_stays_out_of_the_models_shared_cost_map_entry(
|
||||
make_core: MakeCore,
|
||||
) -> None:
|
||||
"""A deployment's declared ``max_input_tokens`` is its own: the process-wide cost-map entry of its
|
||||
model keeps the built-in window, so neither a retired nor a live router narrows another router's
|
||||
deployment of the same model."""
|
||||
builtin_window: Final = litellm.model_cost[NO_JSON_MODE_MODEL.removeprefix("openai/")]["max_input_tokens"] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped in the checker's view
|
||||
first: Final = make_core(
|
||||
model_list=[deployment("g", "declares-five", model=NO_JSON_MODE_MODEL, model_info={"max_input_tokens": 5})]
|
||||
model_list=[deployment("g", "declares-five", model=NO_JSON_MODE_MODEL, model_info={"max_input_tokens": 5})],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
first.retire()
|
||||
other: Final = python_core(
|
||||
CallbackHost(),
|
||||
model_list=[deployment("g", "declares-nothing", model=NO_JSON_MODE_MODEL)],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
for key in (NO_JSON_MODE_MODEL, NO_JSON_MODE_MODEL.removeprefix("openai/")):
|
||||
assert litellm.model_cost.get(key, {}).get("max_input_tokens") in (None, builtin_window) # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
|
||||
assert (await other.select("g", messages=LONG))["model_info"]["id"] == "declares-nothing"
|
||||
with pytest.raises(litellm.ContextWindowExceededError, match="Max Input Tokens=5"):
|
||||
await other.select("g", messages=SHORT)
|
||||
await first.select("g", messages=LONG) # pyright: ignore[reportCallIssue] # select takes messages; the checker reads an overload
|
||||
first.retire()
|
||||
assert (await other.select("g", messages=LONG))["model_info"]["id"] == "declares-nothing"
|
||||
other.retire()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -663,3 +663,14 @@ def test_cooldown_time_reads_back_as_given(make_core: MakeCore) -> None:
|
|||
assert updated == 3 and type(updated) is int
|
||||
default: Final = as_router(make_core(model_list=[deployment("gpt", "a")])).settings().cooldown_time
|
||||
assert default == 5 and type(default) is int
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_old_healthy_listing_takes_the_retry_skip_list_off_the_callers_request(make_core: MakeCore) -> None:
|
||||
router: Final = as_router(make_core(model_list=[deployment("gpt", "a"), deployment("gpt", "b")]))
|
||||
request: Final[dict[str, object]] = {"_retry_skipped_deployment_ids": ("a",)}
|
||||
with pytest.warns(DeprecationWarning, match=r"Router\.async_get_healthy_deployments is deprecated"):
|
||||
healthy: Final = await router.async_get_healthy_deployments("gpt", request) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the old untyped dicts
|
||||
assert isinstance(healthy, list)
|
||||
assert [picked_id(entry) for entry in healthy] == ["b"] # pyright: ignore[reportUnknownArgumentType, reportUnknownVariableType] # see above
|
||||
assert request == {}
|
||||
|
|
|
|||
|
|
@ -392,6 +392,17 @@ async def test_unknown_model_group_raises_without_calling_the_host(
|
|||
assert host.calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["acompletion", "completion"])
|
||||
async def test_an_entry_assigned_without_an_id_is_never_called(make_core: MakeCore, host: ScriptedHost, endpoint: str):
|
||||
"""``model_list = [...]`` keeps an entry without ``model_info.id`` as given, but never routes to it."""
|
||||
core: Final = _one_group(make_core)
|
||||
with pytest.warns(DeprecationWarning, match=r"Router\.model_list is deprecated"): # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
|
||||
core.model_list = [{"model_name": "b", "litellm_params": {"model": "openai/b"}}] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable
|
||||
with pytest.raises(litellm.BadRequestError, match="model=b"):
|
||||
await core.call(endpoint, {"model": "b", "messages": MESSAGES})
|
||||
assert host.calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("op", ["no_such_endpoint", "acompletions", ""])
|
||||
async def test_call_with_an_unknown_op_raises_attribute_error_without_calling_the_host(
|
||||
make_core: MakeCore, host: ScriptedHost, op: str
|
||||
|
|
|
|||
|
|
@ -370,16 +370,18 @@ async def test_order_tiers_run_before_a_plain_list_or_the_default_chain(
|
|||
# --- default fallbacks ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_default_fallbacks_are_appended_to_the_process_fallbacks_list_in_place(
|
||||
def test_default_fallbacks_extend_the_process_fallbacks_without_changing_the_process_list(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
process_fallbacks: list[object] = [{"a": ["b"]}]
|
||||
host.set_dynamic(fallbacks=process_fallbacks)
|
||||
|
||||
core = build(make_core, default_fallbacks=["c"])
|
||||
other = build(make_core, default_fallbacks=["d"])
|
||||
|
||||
assert core.settings().fallbacks == [{"a": ["b"]}, {"*": ["c"]}]
|
||||
assert process_fallbacks == [{"a": ["b"]}, {"*": ["c"]}]
|
||||
assert other.settings().fallbacks == [{"a": ["b"]}, {"*": ["d"]}] # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
|
||||
assert process_fallbacks == [{"a": ["b"]}]
|
||||
|
||||
|
||||
def test_default_fallbacks_leave_the_constructor_fallbacks_list_unchanged(make_core: MakeCore) -> None:
|
||||
|
|
|
|||
|
|
@ -300,7 +300,7 @@ def test_a_failed_edit_rolls_back_to_the_serving_router_even_above_a_tightened_l
|
|||
assert restored.litellm_params.tags is None
|
||||
|
||||
|
||||
def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_leaves_the_router_removed(
|
||||
def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_restores_the_router(
|
||||
make_core: MakeCore,
|
||||
) -> None:
|
||||
limit: Final = {"value": 2}
|
||||
|
|
@ -319,8 +319,8 @@ def test_a_failed_edit_without_ignore_invalid_deployments_raises_and_leaves_the_
|
|||
ChangeSet(upsert=[Deployment(**complexity_marker("two", "m2", config=HEURISTIC_V2, tags=["edited"]))]) # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
assert str(raised.value) == HEURISTIC_V2_LIMIT_MESSAGE
|
||||
assert model_ids(core) == ["s", "b", "m1"]
|
||||
assert registered_tags(core, "complexity") == {"one": [()]}
|
||||
assert model_ids(core) == ["s", "b", "m1", "m2"]
|
||||
assert registered_tags(core, "complexity") == {"one": [()], "two": [()]}
|
||||
|
||||
|
||||
# --- the adaptive router -------------------------------------------------------------------------------
|
||||
|
|
@ -373,10 +373,19 @@ def test_an_adaptive_router_registers_one_post_call_hook_and_a_header_hook(
|
|||
|
||||
post_call, header = _hook_counts(host)
|
||||
assert post_call == 1
|
||||
assert header >= 1
|
||||
assert header == 1
|
||||
|
||||
|
||||
def test_deleting_an_adaptive_router_releases_it_and_its_post_call_hook(
|
||||
def test_resyncing_an_adaptive_router_keeps_a_single_header_hook(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
core: Final = make_core(model_list=_adaptive_list())
|
||||
|
||||
core.apply(ChangeSet(upsert=[Deployment(**adaptive_marker("ar", "ar1", ["y"]))])) # pyright: ignore[reportArgumentType]
|
||||
core.apply(ChangeSet(upsert=[Deployment(**adaptive_marker("ar", "ar1", ["x"]))])) # pyright: ignore[reportArgumentType]
|
||||
|
||||
assert _hook_counts(host) == (1, 1)
|
||||
|
||||
|
||||
def test_deleting_an_adaptive_router_releases_it_its_post_call_hook_and_the_header_hook(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=_adaptive_list())
|
||||
|
|
@ -384,7 +393,30 @@ def test_deleting_an_adaptive_router_releases_it_and_its_post_call_hook(
|
|||
core.apply(ChangeSet(delete=["ar1"]))
|
||||
|
||||
assert registered_tags(core, "adaptive") == {}
|
||||
assert _hook_counts(host)[0] == 0
|
||||
assert _hook_counts(host) == (0, 0)
|
||||
|
||||
|
||||
def test_retiring_the_last_adaptive_router_on_the_host_removes_the_header_hook(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=_adaptive_list())
|
||||
|
||||
core.retire()
|
||||
|
||||
assert _hook_counts(host) == (0, 0)
|
||||
|
||||
|
||||
def test_retiring_one_adaptive_router_keeps_the_header_hook_for_another_on_the_host(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
first: Final = make_core(model_list=_adaptive_list())
|
||||
second: Final = make_core(model_list=_adaptive_list())
|
||||
|
||||
first.retire()
|
||||
|
||||
assert _hook_counts(host)[1] == 1
|
||||
second.retire()
|
||||
assert _hook_counts(host) == (0, 0)
|
||||
|
||||
|
||||
def test_a_complexity_router_with_adaptive_enabled_also_registers_an_adaptive_router(make_core: MakeCore) -> None:
|
||||
|
|
|
|||
|
|
@ -315,6 +315,8 @@ async def test_an_access_group_only_caller_reaches_any_deployment_by_its_litellm
|
|||
|
||||
|
||||
# --- tier litellm_params -------------------------------------------------------------------------
|
||||
# ``select`` works on a copy of its caller's request; these read what selection pins and stamps on the
|
||||
# request the router routes (and re-routes on retries), through the cores' in-place selection.
|
||||
|
||||
|
||||
def _pinned(**tier_params: object) -> dict[str, object]: # mutable-ok: tiers # kwargs-ok: tier litellm_params
|
||||
|
|
@ -331,7 +333,7 @@ async def test_a_tier_param_no_deployment_supports_is_dropped(make_core: MakeCor
|
|||
]
|
||||
)
|
||||
request: Final[dict[str, object]] = {}
|
||||
chosen: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
chosen: Final = await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
|
||||
assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue]
|
||||
assert request["temperature"] == 0.2
|
||||
assert "logit_bias" not in request
|
||||
|
|
@ -347,7 +349,7 @@ async def test_a_tier_param_the_request_allowlists_is_kept(make_core: MakeCore)
|
|||
]
|
||||
)
|
||||
request: Final[dict[str, object]] = {"allowed_openai_params": ["logit_bias"]}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
assert request["logit_bias"] == {"1": 2}
|
||||
|
||||
|
||||
|
|
@ -360,7 +362,7 @@ async def test_a_tier_reasoning_effort_removes_the_callers_other_effort_carriers
|
|||
"reasoning": {"effort": "high"},
|
||||
"max_completion_tokens": 100,
|
||||
}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
assert request["reasoning_effort"] == "low"
|
||||
assert "thinking" not in request
|
||||
assert request["output_config"] == {"format": "json"}
|
||||
|
|
@ -372,7 +374,7 @@ async def test_a_tier_reasoning_effort_removes_the_callers_other_effort_carriers
|
|||
async def test_a_tier_without_an_output_ceiling_leaves_the_callers_ceiling(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(model_list=_tiered(tiers=_pinned(temperature=0.1)))
|
||||
request: Final[dict[str, object]] = {"max_completion_tokens": 100}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
assert request["temperature"] == 0.1
|
||||
assert request["max_completion_tokens"] == 100
|
||||
assert "max_tokens" not in request
|
||||
|
|
@ -384,12 +386,12 @@ async def test_a_tier_without_an_output_ceiling_leaves_the_callers_ceiling(make_
|
|||
async def test_a_later_pick_restores_the_callers_ceiling_a_tier_pin_replaced(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(model_list=_tiered(tiers=_pinned(max_tokens=7)))
|
||||
request: Final[dict[str, object]] = {"max_completion_tokens": 100}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
assert request["max_tokens"] == 7
|
||||
assert "max_completion_tokens" not in request
|
||||
assert _metadata(request)["_client_output_ceiling"] == {"max_completion_tokens": 100}
|
||||
|
||||
chosen: Final = await core.select("smart", request, messages=HARD_MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
chosen: Final = await core._select_in_place("smart", request, messages=HARD_MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
|
||||
assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue]
|
||||
assert request["max_completion_tokens"] == 100
|
||||
assert "max_tokens" not in request
|
||||
|
|
@ -401,13 +403,13 @@ async def test_a_second_pick_leaves_the_first_picks_forwarded_marker_params_untr
|
|||
# next attempt, so the tracking key is dropped while the params themselves stay behind.
|
||||
core: Final = make_core(model_list=_tiered(temperature=0.3))
|
||||
request: Final[dict[str, object]] = {}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
forwarded: Final = request["_alias_marker_forwarded_params"]
|
||||
assert isinstance(forwarded, tuple)
|
||||
assert "temperature" in forwarded
|
||||
assert request["temperature"] == 0.3
|
||||
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
assert "_alias_marker_forwarded_params" not in request
|
||||
assert request["temperature"] == 0.3
|
||||
|
||||
|
|
@ -419,7 +421,7 @@ async def test_a_second_pick_leaves_the_first_picks_forwarded_marker_params_untr
|
|||
async def test_the_routing_decision_keeps_its_prompt_signals_by_default(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(model_list=_tiered())
|
||||
request: Final[dict[str, object]] = {}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
decision: Final = _metadata(request)["routing_decision"]
|
||||
assert isinstance(decision, Mapping)
|
||||
assert "signals" in decision
|
||||
|
|
@ -429,7 +431,7 @@ async def test_the_routing_decision_keeps_its_prompt_signals_by_default(make_cor
|
|||
async def test_redacted_message_logging_drops_the_prompt_quoting_routing_decision_fields(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(model_list=_tiered())
|
||||
request: Final[dict[str, object]] = {"standard_callback_dynamic_params": {"turn_off_message_logging": True}}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
decision: Final = _metadata(request)["routing_decision"]
|
||||
assert isinstance(decision, Mapping)
|
||||
assert "signals" not in decision
|
||||
|
|
@ -440,7 +442,7 @@ async def test_redacted_message_logging_drops_the_prompt_quoting_routing_decisio
|
|||
async def test_routing_metadata_goes_to_litellm_metadata_when_the_request_has_it(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(model_list=_tiered())
|
||||
request: Final[dict[str, object]] = {"litellm_metadata": {}}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
await core._select_in_place("smart", request, messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
assert "routing_decision" in _metadata(request, "litellm_metadata")
|
||||
assert "metadata" not in request
|
||||
|
||||
|
|
|
|||
|
|
@ -131,8 +131,27 @@ def test_a_list_of_organizations_fans_out_into_one_deployment_per_organization(m
|
|||
|
||||
entries = core.registry().all()
|
||||
assert [entry["litellm_params"]["organization"] for entry in entries] == ["org-1", "org-2"]
|
||||
assert len(set(ids(entries))) == 1
|
||||
assert len(set(ids(entries))) == 2
|
||||
assert core.registry().index("model_name") == {"gpt": [0, 1]}
|
||||
for entry in entries: # pyright: ignore[reportAny] # untyped in the checker's view
|
||||
found = core.registry().deployment(entry["model_info"]["id"]) # pyright: ignore[reportAny] # untyped in the checker's view
|
||||
assert found is not None
|
||||
assert found.litellm_params.organization == entry["litellm_params"]["organization"]
|
||||
|
||||
|
||||
def test_organization_deployments_get_the_same_ids_on_every_build(make_core: MakeCore) -> None:
|
||||
first = make_core(model_list=[_bare("gpt", "openai/x", organization=["org-1", "org-2"])])
|
||||
second = make_core(model_list=[_bare("gpt", "openai/x", organization=["org-1", "org-2"])])
|
||||
|
||||
assert ids(first.registry().all()) == ids(second.registry().all())
|
||||
|
||||
|
||||
def test_a_declared_id_stays_on_the_first_organization_and_is_suffixed_on_the_rest(make_core: MakeCore) -> None:
|
||||
entry = _bare("gpt", "openai/x", organization=["org-1", "org-2"])
|
||||
entry["model_info"] = {"id": "declared"}
|
||||
core = make_core(model_list=[entry])
|
||||
|
||||
assert ids(core.registry().all()) == ["declared", "declared-org-2"]
|
||||
|
||||
|
||||
def test_deployment_names_list_each_deployments_litellm_model(make_core: MakeCore) -> None:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ access groups; blocked and unhealthy groups; credentials lookups.
|
|||
"""
|
||||
|
||||
import time
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -668,15 +669,31 @@ def test_the_deprecated_generate_model_id_is_the_registry_id_hash(make_core: Mak
|
|||
assert generated == deployment_id("g", {"model": "x"})
|
||||
|
||||
|
||||
def test_assigning_model_list_swaps_the_list_without_reindexing(make_core: MakeCore) -> None:
|
||||
"""Odd but current: the id index still maps ``a`` to position 0, which now holds the new entry."""
|
||||
def test_assigning_model_list_swaps_and_reindexes_the_list(make_core: MakeCore) -> None:
|
||||
"""The replaced deployment's id finds nothing; the new entry is found by its own id and name."""
|
||||
core = make_core(model_list=[deployment("gpt", "a")])
|
||||
version = core.registry().version
|
||||
|
||||
core.model_list = [deployment("n", "n")] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable
|
||||
with pytest.warns(DeprecationWarning, match=r"Router\.model_list is deprecated"): # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
|
||||
core.model_list = [deployment("n", "n")] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable
|
||||
|
||||
assert core.registry().version == version + 1
|
||||
assert core.registry().names(kind="ids") == ["a"]
|
||||
found = core.registry().deployment("a")
|
||||
assert core.registry().names(kind="ids") == ["n"]
|
||||
assert core.registry().deployment("a") is None
|
||||
found = core.registry().deployment("n")
|
||||
assert found is not None
|
||||
assert found.model_name == "n"
|
||||
assert ids(core.registry().deployments("n")) == ["n"]
|
||||
assert core.registry().deployments("gpt") == []
|
||||
|
||||
|
||||
def test_an_entry_assigned_without_an_id_is_not_served(make_core: MakeCore) -> None:
|
||||
"""``model_list = [...]`` keeps entries as given; one without ``model_info.id`` is never routed to."""
|
||||
core = make_core(model_list=[deployment("gpt", "a")])
|
||||
entry: Final = {"model_name": "b", "litellm_params": {"model": "openai/gpt-4o-mini"}}
|
||||
|
||||
with pytest.warns(DeprecationWarning, match=r"Router\.model_list is deprecated"): # pyright: ignore[reportUnknownMemberType] # untyped in the checker's view
|
||||
core.model_list = [entry] # pyright: ignore[reportAttributeAccessIssue] # deprecated attribute, still assignable
|
||||
|
||||
assert core.registry().deployments("b") == []
|
||||
assert core.registry().names(kind="ids") == []
|
||||
|
|
|
|||
|
|
@ -154,14 +154,20 @@ def test_an_upsert_that_empties_a_group_drops_the_group_name(make_core: MakeCore
|
|||
assert core.registry().deployments("emb") == []
|
||||
|
||||
|
||||
def test_a_failing_upsert_of_an_existing_deployment_raises_and_leaves_it_removed(make_core: MakeCore) -> None:
|
||||
"""Odd but current: without ignore_invalid_deployments the old deployment is popped and not restored."""
|
||||
def test_a_failing_upsert_of_an_existing_deployment_raises_and_restores_the_previous_deployment(
|
||||
make_core: MakeCore,
|
||||
) -> None:
|
||||
"""Without ignore_invalid_deployments the failed edit raises, and the deployment it would have
|
||||
replaced keeps serving its previous configuration."""
|
||||
core = _core(make_core)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
core.apply(ChangeSet(upsert=[typed("gpt", "a", model="notaprovider/x")]))
|
||||
|
||||
assert core.registry().names(kind="ids") == ["b", "e", "w", "t1"]
|
||||
restored = core.registry().deployment("a")
|
||||
assert restored is not None
|
||||
assert restored.litellm_params.model == "openai/a"
|
||||
assert core.registry().names(kind="ids") == ["b", "e", "w", "t1", "a"]
|
||||
|
||||
|
||||
def test_a_failing_upsert_under_ignore_invalid_deployments_restores_the_previous_deployment(
|
||||
|
|
@ -241,12 +247,19 @@ def test_deleting_a_wildcard_deployment_stops_it_serving_matching_names(make_cor
|
|||
assert core.registry().deployments("openai/gpt-x") == []
|
||||
|
||||
|
||||
def test_deleting_a_wildcard_deployment_keeps_it_among_provider_default_deployment_ids(make_core: MakeCore) -> None:
|
||||
"""Odd but current: the provider default id list is append-only."""
|
||||
def test_deleting_a_wildcard_deployment_drops_it_from_the_provider_default_deployment_ids(make_core: MakeCore) -> None:
|
||||
core = _core(make_core)
|
||||
|
||||
core.apply(ChangeSet(delete=["w"]))
|
||||
|
||||
assert core.settings().provider_default_deployment_ids == []
|
||||
|
||||
|
||||
def test_deleting_a_plain_deployment_keeps_the_wildcards_provider_default_deployment_id(make_core: MakeCore) -> None:
|
||||
core = _core(make_core)
|
||||
|
||||
core.apply(ChangeSet(delete=["a"]))
|
||||
|
||||
assert core.settings().provider_default_deployment_ids == ["w"]
|
||||
|
||||
|
||||
|
|
@ -300,23 +313,31 @@ def test_set_model_list_leaves_the_given_list_unchanged(make_core: MakeCore) ->
|
|||
assert given == [{"model_name": "new", "litellm_params": {"model": "openai/n", "api_key": "k"}}]
|
||||
|
||||
|
||||
def test_set_model_list_keeps_the_old_wildcard_routes(make_core: MakeCore) -> None:
|
||||
"""Odd but current: the wildcard pattern router is not reset, so a replaced wildcard still matches."""
|
||||
def test_set_model_list_drops_the_old_wildcard_routes(make_core: MakeCore) -> None:
|
||||
core = _core(make_core)
|
||||
|
||||
core.set_model_list([deployment("new", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core
|
||||
|
||||
assert names_and_ids(core.registry().deployments("openai/gpt-x")) == [("openai/gpt-x", "w")]
|
||||
assert core.settings().provider_default_deployment_ids == ["w"]
|
||||
assert core.registry().deployments("openai/gpt-x") == []
|
||||
assert core.settings().provider_default_deployment_ids == []
|
||||
|
||||
|
||||
def test_set_model_list_appends_to_the_deployment_names(make_core: MakeCore) -> None:
|
||||
"""Odd but current: deployment_names keep the replaced deployments' models."""
|
||||
def test_set_model_list_serves_the_new_lists_wildcard_routes(make_core: MakeCore) -> None:
|
||||
core = _core(make_core)
|
||||
|
||||
core.set_model_list([deployment("anthropic/*", "w2", model="anthropic/*")]) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # public on the core
|
||||
|
||||
assert core.registry().deployments("openai/gpt-x") == []
|
||||
assert names_and_ids(core.registry().deployments("anthropic/claude-x")) == [("anthropic/claude-x", "w2")]
|
||||
assert core.settings().provider_default_deployment_ids == ["w2"]
|
||||
|
||||
|
||||
def test_set_model_list_replaces_the_deployment_names(make_core: MakeCore) -> None:
|
||||
core = make_core(model_list=[deployment("gpt", "a")])
|
||||
|
||||
core.set_model_list([deployment("new", "n")]) # pyright: ignore[reportAttributeAccessIssue] # public on the core
|
||||
|
||||
assert core.registry().names(kind="deployment_names") == ["openai/a", "openai/n"]
|
||||
assert core.registry().names(kind="deployment_names") == ["openai/n"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -676,7 +676,7 @@ async def test_retry_skipped_deployment_ids_given_as_a_list_are_ignored(make_cor
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_select_pops_the_narrowing_keys_from_the_request(make_core: MakeCore) -> None:
|
||||
async def test_select_honors_the_narrowing_keys_but_leaves_them_on_the_callers_request(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(model_list=list(THREE))
|
||||
request: Final[dict[str, object]] = {
|
||||
"_target_order": None,
|
||||
|
|
@ -684,6 +684,22 @@ async def test_select_pops_the_narrowing_keys_from_the_request(make_core: MakeCo
|
|||
"_retry_skipped_deployment_ids": ("b",),
|
||||
}
|
||||
assert ids_of(await core.select("g", request, kind="healthy")) == ["c"]
|
||||
assert request == {
|
||||
"_target_order": None,
|
||||
"_excluded_deployment_ids": ["a"],
|
||||
"_retry_skipped_deployment_ids": ("b",),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # untyped in the checker's view
|
||||
async def test_in_place_selection_pops_the_narrowing_keys_from_the_routed_request(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(model_list=list(THREE))
|
||||
request: Final[dict[str, object]] = {
|
||||
"_target_order": None,
|
||||
"_excluded_deployment_ids": ["a"],
|
||||
"_retry_skipped_deployment_ids": ("b",),
|
||||
}
|
||||
assert ids_of(await core._select_in_place("g", request, kind="healthy")) == ["c"] # pyright: ignore[reportAttributeAccessIssue, reportUnknownArgumentType, reportUnknownMemberType]
|
||||
assert request == {}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -78,6 +78,19 @@ def _metadata(request: Mapping[str, object]) -> Mapping[str, object]:
|
|||
return metadata
|
||||
|
||||
|
||||
async def _routed_metadata(
|
||||
core: RouterCore,
|
||||
host: ScriptedHost,
|
||||
model: str,
|
||||
messages: list[dict[str, str]],
|
||||
**request: object, # kwargs-ok: the request's own keyword payload
|
||||
) -> Mapping[str, object]:
|
||||
"""The metadata the routed call carried: ``select`` leaves its caller's request alone, so the
|
||||
routing stamps are read where they land, on the request the router sends."""
|
||||
await core.acompletion(model=model, messages=messages, **request) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
return _metadata(host.calls[-1].kwargs)
|
||||
|
||||
|
||||
# --- the complexity router ------------------------------------------------------------------------
|
||||
|
||||
|
||||
|
|
@ -92,11 +105,11 @@ async def test_a_complexity_router_picks_the_tier_deployment_for_the_prompt(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_complexity_router_pick_stamps_the_routing_decision_on_the_request(make_core: MakeCore) -> None:
|
||||
async def test_a_complexity_router_pick_stamps_the_routing_decision_on_the_routed_request(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=_tiered())
|
||||
request: Final[dict[str, object]] = {}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
metadata: Final = _metadata(request)
|
||||
metadata: Final = await _routed_metadata(core, host, "smart", MESSAGES)
|
||||
decision: Final = metadata["routing_decision"]
|
||||
assert isinstance(decision, Mapping)
|
||||
assert decision["router_model_name"] == "smart"
|
||||
|
|
@ -121,11 +134,11 @@ async def test_completion_through_a_complexity_router_reports_the_router_name_as
|
|||
@pytest.mark.asyncio
|
||||
async def test_an_alias_to_a_complexity_router_routes_through_it(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"})
|
||||
request: Final[dict[str, object]] = {}
|
||||
chosen: Final = await core.select("al", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
chosen: Final = await core.select("al", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType]
|
||||
assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue]
|
||||
assert _metadata(request)["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue]
|
||||
await core.acompletion(model="al", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
|
||||
metadata: Final = await _routed_metadata(core, host, "al", MESSAGES)
|
||||
assert metadata["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue]
|
||||
assert host.calls[-1].deployment_id == "s"
|
||||
assert host.calls[-1].model_group == "al"
|
||||
|
||||
|
||||
|
|
@ -137,15 +150,14 @@ async def test_a_complexity_router_takes_precedence_over_plain_deployments_of_it
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_complexity_router_tier_naming_an_unknown_model_falls_back_to_the_default_model(
|
||||
make_core: MakeCore,
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core: Final = make_core(
|
||||
model_list=[deployment("big", "b"), _complexity_router(tiers={**TIERS, "SIMPLE": "missing"})]
|
||||
)
|
||||
request: Final[dict[str, object]] = {}
|
||||
chosen: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
chosen: Final = await core.select("smart", {}, messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType]
|
||||
assert chosen["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue]
|
||||
decision: Final = _metadata(request)["routing_decision"]
|
||||
decision: Final = (await _routed_metadata(core, host, "smart", MESSAGES))["routing_decision"]
|
||||
assert isinstance(decision, Mapping)
|
||||
assert decision["cause"] == "health_default_fallback"
|
||||
assert decision["routed_model"] == "big"
|
||||
|
|
@ -282,13 +294,25 @@ async def test_tier_litellm_params_override_the_callers_params_on_the_call(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tier_litellm_params_are_pinned_onto_the_request_by_select(make_core: MakeCore) -> None:
|
||||
async def test_select_leaves_the_callers_request_without_the_tier_pins(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(model_list=_tiered(tiers=PINNED_TIERS))
|
||||
request: Final[dict[str, object]] = {"temperature": 0.9, "max_completion_tokens": 100}
|
||||
await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
assert request["temperature"] == 0.1
|
||||
assert request["max_tokens"] == 7
|
||||
assert "max_completion_tokens" not in request
|
||||
picked: Final = await core.select("smart", request, messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType]
|
||||
assert picked["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue] # a deployment dict
|
||||
assert request == {"temperature": 0.9, "max_completion_tokens": 100}
|
||||
|
||||
|
||||
@pytest.mark.asyncio # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # untyped in the checker's view
|
||||
async def test_select_leaves_the_callers_request_and_metadata_without_marker_params_or_stamps(
|
||||
make_core: MakeCore,
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=_tiered(temperature=0.3))
|
||||
metadata: Final[dict[str, object]] = {"tags": ["t"]}
|
||||
request: Final[dict[str, object]] = {"metadata": metadata, "specific_deployment": False}
|
||||
await core.select("smart", request, messages=HARD_MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
assert request == {"metadata": {"tags": ["t"]}, "specific_deployment": False}
|
||||
assert request["metadata"] is metadata
|
||||
assert metadata == {"tags": ["t"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -351,13 +375,16 @@ def _session_request(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_subagent_request_follows_the_router_its_session_started_on(make_core: MakeCore) -> None:
|
||||
async def test_a_subagent_request_follows_the_router_its_session_started_on(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=_tiered())
|
||||
await core.select("smart", _session_request(), messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
subagent: Final = _session_request(agent_id="agent-1")
|
||||
chosen: Final = await core.select("big", subagent, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
chosen: Final = await core.select("big", _session_request(agent_id="agent-1"), messages=MESSAGES) # pyright: ignore[reportCallIssue, reportUnknownVariableType]
|
||||
assert chosen["model_info"]["id"] == "s" # pyright: ignore[reportIndexIssue]
|
||||
assert _metadata(subagent)["model_group"] == "smart"
|
||||
metadata: Final = await _routed_metadata(core, host, "big", MESSAGES, **_session_request(agent_id="agent-1"))
|
||||
assert host.calls[-1].deployment_id == "s"
|
||||
assert metadata["model_group"] == "smart"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -428,24 +455,31 @@ async def test_a_routing_plugin_narrows_the_candidates_to_the_models_it_keeps(ma
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routing_plugin_signals_and_narrowed_candidates_are_stamped_on_the_request(make_core: MakeCore) -> None:
|
||||
async def test_routing_plugin_signals_and_narrowed_candidates_are_stamped_on_the_routed_request(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/b"}))])
|
||||
request: Final[dict[str, object]] = {}
|
||||
await core.select("g", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
assert request == {
|
||||
"metadata": {
|
||||
"routing_plugin_signals": {"narrowing": {"kept": ["openai/b"]}},
|
||||
"_routing_plugin_candidate_models": ["openai/b"],
|
||||
}
|
||||
}
|
||||
metadata: Final = await _routed_metadata(core, host, "g", MESSAGES)
|
||||
assert metadata["routing_plugin_signals"] == {"narrowing": {"kept": ["openai/b"]}}
|
||||
assert metadata["_routing_plugin_candidate_models"] == ["openai/b"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_routing_plugin_that_keeps_every_candidate_stamps_only_its_signals(make_core: MakeCore) -> None:
|
||||
async def test_a_routing_plugin_that_keeps_every_candidate_stamps_only_its_signals(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/a", "openai/b"}))])
|
||||
metadata: Final = await _routed_metadata(core, host, "g", MESSAGES)
|
||||
assert metadata["routing_plugin_signals"] == {"narrowing": {"kept": ["openai/a", "openai/b"]}}
|
||||
assert "_routing_plugin_candidate_models" not in metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # untyped in the checker's view
|
||||
async def test_select_leaves_the_callers_request_without_routing_plugin_stamps(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(model_list=list(TWO), plugins=[Narrowing(frozenset({"openai/b"}))])
|
||||
request: Final[dict[str, object]] = {}
|
||||
await core.select("g", request, messages=MESSAGES) # pyright: ignore[reportCallIssue]
|
||||
assert request == {"metadata": {"routing_plugin_signals": {"narrowing": {"kept": ["openai/a", "openai/b"]}}}}
|
||||
assert request == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -2081,7 +2081,8 @@ def test_add_deployment_does_not_leak_custom_metadata_to_shared_backend_key():
|
|||
|
||||
def test_shared_backend_model_info_keeps_schema_fields_and_drops_the_rest():
|
||||
"""Unit test of the whitelist helper: cost-map schema fields survive,
|
||||
custom pricing overrides and per-deployment metadata do not.
|
||||
custom pricing overrides, per-deployment metadata and per-deployment token
|
||||
limits do not.
|
||||
"""
|
||||
from litellm.types.utils import shared_backend_model_info
|
||||
|
||||
|
|
@ -2106,7 +2107,6 @@ def test_shared_backend_model_info_keeps_schema_fields_and_drops_the_rest():
|
|||
assert filtered == {
|
||||
"mode": "chat",
|
||||
"litellm_provider": "openai",
|
||||
"max_tokens": 128000,
|
||||
"supports_vision": True,
|
||||
"supported_endpoints": ["/v1/responses"],
|
||||
"use_openai_responses_path": True,
|
||||
|
|
@ -3052,7 +3052,7 @@ def test_register_deployment_in_model_cost_writes_both_key_families():
|
|||
"""
|
||||
A deployment contributes its full model_info under its unique id and the
|
||||
cost-map subset under the shared backend key, and the shared key must not
|
||||
pick up the deployment's private metadata.
|
||||
pick up the deployment's private metadata or its declared token limits.
|
||||
"""
|
||||
model_keys = {
|
||||
"both-families-id": copy.deepcopy(litellm.model_cost.get("both-families-id")),
|
||||
|
|
@ -3068,7 +3068,8 @@ def test_register_deployment_in_model_cost_writes_both_key_families():
|
|||
|
||||
assert litellm.model_cost["both-families-id"]["max_input_tokens"] == 999
|
||||
shared = litellm.model_cost["hosted_vllm/both-families-backend"]
|
||||
assert shared["max_input_tokens"] == 999
|
||||
assert shared["litellm_provider"] == "hosted_vllm"
|
||||
assert "max_input_tokens" not in shared
|
||||
assert "id" not in shared
|
||||
finally:
|
||||
_restore_model_cost_entries(model_keys)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue