diff --git a/litellm/router.py b/litellm/router.py index 7d566bd0744..df57cd5d445 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -34,7 +34,6 @@ import asyncio import contextlib import copy import enum -import hashlib import inspect import json import logging @@ -185,6 +184,20 @@ from litellm.llms.openai_like.model_info import ( ) from litellm.responses.bridge_check import keep_registered_mode from litellm.responses.retrieve_stream_query import requests_streamed_retrieval, without_retrieve_stream_defaults +from litellm.router_api import ( + SETTING_FIELDS, + ApplyResult, + ChangeSet, + RouterHealth, + RouterRegistry, + RouterSettings, + RoutingObservers, + SelectKind, + aapply_changes, + apply_changes, +) +from litellm.router_api import deployment_id as _deployment_id +from litellm.router_deprecated import DeprecatedRouterMembers from litellm.router_strategy.auto_router.auto_router_model_naming import ( AUTO_ROUTER_MODEL_PREFIX, STRATEGY_ROUTER_PARAM_FIELDS, @@ -511,6 +524,7 @@ if TYPE_CHECKING: from litellm.responses.streaming_iterator import ( BaseResponsesAPIStreamingIterator, ) + from litellm.router_api import UntypedDict from litellm.router_strategy.adaptive_router.adaptive_router import ( AdaptiveRouter, ) @@ -1264,17 +1278,162 @@ def _base_model_widened_window( return MappingProxyType({"max_input_tokens": base_max_input_tokens}) -class Router: - @property - def _routing_groups(self) -> dict[str, RoutingGroup]: - return self.routing_groups +_SETTING_ATTRIBUTES: Final = frozenset("_core_" + name for name in SETTING_FIELDS) - @_routing_groups.setter - def _routing_groups(self, value: dict[str, RoutingGroup]) -> None: - self.routing_groups = value - model_names: set[str] = set() - cache_responses: bool | None = False +class Router(DeprecatedRouterMembers): + _core_model_names: set[str] = set() + _core_cache_responses: bool | None = False + _registry_version: int = 0 + + # -- entry points: everything outside code needs besides the endpoint methods --------------- + # See litellm/router_api.py. The members outside code used before these live in + # DeprecatedRouterMembers (litellm/router_deprecated.py). + + def registry(self) -> RouterRegistry: + """Deployment lookups: by id, group, alias, wildcard and team, plus configured plugins.""" + cached: Final = self.__dict__.get("_registry_view") + if isinstance(cached, RouterRegistry): + return cached + view: Final = RouterRegistry(self) + self.__dict__["_registry_view"] = view + return view + + def settings(self) -> RouterSettings: + """ + A frozen snapshot of the router's configuration. It is built once and reused until a + setting attribute is rebound or deleted (``__setattr__``/``__delattr__`` drop it), so the + request path can read it cheaply. + """ + cached: Final = self.__dict__.get("_settings_snapshot") + if isinstance(cached, RouterSettings): + return cached + snapshot: Final = RouterSettings.of(self) + self.__dict__["_settings_snapshot"] = snapshot + return snapshot + + def __setattr__(self, name: str, value: object) -> None: + if name in _SETTING_ATTRIBUTES: + self.__dict__.pop("_settings_snapshot", None) + elif name == "_core_model_list" and "_core_model_list" in self.__dict__: + # A whole new deployment list (``router.model_list = ...``) is a deployment change too. + self.__dict__["_registry_version"] = self._registry_version + 1 + super().__setattr__(name, value) + + def __delattr__(self, name: str) -> None: + if name in _SETTING_ATTRIBUTES: + self.__dict__.pop("_settings_snapshot", None) + super().__delattr__(name) + + def health(self) -> RouterHealth: + """Cooldowns, blocked deployments, health checks and usage.""" + cached: Final = self.__dict__.get("_health_view") + if isinstance(cached, RouterHealth): + return cached + view: Final = RouterHealth(self) + self.__dict__["_health_view"] = view + return view + + @overload + async def select( + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the selection cores take a dict + *, + kind: Literal["pick", "pass_through"] = "pick", + messages: list[dict[str, str]] | None = None, # mutable-ok: the selection cores take a list + input: str | list[object] | None = None, # mutable-ok: the selection cores take a list + specific_deployment: bool | None = False, + parent_otel_span: Span | None = None, + health_check_probe: bool = False, + ) -> "UntypedDict": ... # a mutable dict, as before: the cores' untyped deployment dict, unchanged + + @overload + async def select( + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the selection cores take a dict + *, + kind: Literal["healthy"], + messages: list[dict[str, str]] | None = None, # mutable-ok: the selection cores take a list + input: str | list[object] | None = None, # mutable-ok: the selection cores take a list + specific_deployment: bool | None = False, + parent_otel_span: Span | None = None, + health_check_probe: bool = False, + ) -> "list[UntypedDict] | UntypedDict": ... # mutable-ok: the core's type (list[dict] | dict), unchanged + + async def select( # pyright: ignore[reportUnknownParameterType] # the selection cores return untyped deployment dicts + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the selection cores take a dict + *, + kind: SelectKind = "pick", + messages: list[dict[str, str]] | None = None, # mutable-ok: the selection cores take a list + input: str | list[object] | None = None, # mutable-ok: the selection cores take a list + specific_deployment: bool | None = False, + parent_otel_span: Span | None = None, + health_check_probe: bool = False, + ): + """ + Pick a deployment for ``model`` without calling it (``pick``), list the deployments that + are healthy for the request (``healthy``), or pick one for a pass-through route + (``pass_through``). + """ + if kind == "pick": + return await self._core_async_get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts + model=model, + request_kwargs=request_kwargs if request_kwargs is not None else {}, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) + if kind == "healthy": + return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts + model=model, + request_kwargs=request_kwargs if request_kwargs is not None else {}, + messages=messages, + input=input, + specific_deployment=specific_deployment, + parent_otel_span=parent_otel_span, + health_check_probe=health_check_probe, + ) + if kind == "pass_through": + return self._core_get_available_deployment_for_pass_through( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts + model=model, + messages=messages, + input=input, + specific_deployment=specific_deployment, + request_kwargs=request_kwargs, + ) + raise ValueError(f"unknown select kind {kind!r}") + + def apply(self, changes: ChangeSet) -> ApplyResult: + """Apply runtime changes: settings, deployments, the routing state's Redis cache.""" + return apply_changes(self, changes) + + async def aapply(self, changes: ChangeSet) -> ApplyResult: + """``apply``, plus the changes that need I/O (``refresh_model_info``).""" + return await aapply_changes(self, changes) + + def observe(self, logging_obj: "LiteLLMLogging | None" = None, *, request_scoped: bool = False) -> RoutingObservers: + """ + The router's routing observers. ``request_scoped=True`` stops registering them globally, + so each request binds them instead; passing a request's ``logging_obj`` binds them to it. + """ + if request_scoped: + self._core_serve_through_request_logging() + if logging_obj is not None: + self._core_bind_to_request_logging(logging_obj) + return RoutingObservers(slack_alerting_logger=getattr(self, "_core_slack_alerting_logger", None)) + + def register_plugin(self, plugin: CustomRoutingStrategyBase) -> None: + """Route with a custom strategy instead of the configured one.""" + self._core_set_custom_routing_strategy(plugin) + + def retire(self) -> None: + """Release the router's global callback registrations and routing state.""" + self._core_discard() + default_cache_time_seconds: int = 1 * 60 * 60 # 1 hour tenacity = None leastbusy_logger: LeastBusyLoggingHandler | None = None @@ -1433,13 +1592,13 @@ class Router: self.set_verbose = set_verbose self.ignore_invalid_deployments = ignore_invalid_deployments self.auto_router_capability_limit = auto_router_capability_limit - self.fallback_access_check: Final = fallback_access_check - self.fallback_budget_check: Final = fallback_budget_check + self._core_fallback_access_check: Final = fallback_access_check + self._core_fallback_budget_check: Final = fallback_budget_check self.debug_level = debug_level - self.enable_pre_call_checks = enable_pre_call_checks - self.enable_tag_filtering = enable_tag_filtering - self.tag_filtering_match_any = tag_filtering_match_any - self.tag_routing_prefix = tag_routing_prefix + self._core_enable_pre_call_checks = enable_pre_call_checks + self._core_enable_tag_filtering = enable_tag_filtering + self._core_tag_filtering_match_any = tag_filtering_match_any + self._core_tag_routing_prefix = tag_routing_prefix from litellm._service_logger import ServiceLogging self.service_logger_obj: ServiceLogging = ServiceLogging() @@ -1449,20 +1608,20 @@ class Router: verbose_router_logger.setLevel(logging.INFO) elif debug_level == "DEBUG": verbose_router_logger.setLevel(logging.DEBUG) - self.router_general_settings: RouterGeneralSettings = router_general_settings or RouterGeneralSettings() + self._core_router_general_settings: RouterGeneralSettings = router_general_settings or RouterGeneralSettings() self.assistants_config = assistants_config - self.search_tools = search_tools or [] - self.guardrail_list = guardrail_list or [] - self.deployment_names: list[str] = [] # names of models under litellm_params. ex. azure/chatgpt-v-2 - self.deployment_latency_map = {} + self._core_search_tools = search_tools or [] + self._core_guardrail_list = guardrail_list or [] + self._core_deployment_names: list[str] = [] # names of models under litellm_params. ex. azure/chatgpt-v-2 + self._core_deployment_latency_map = {} ### CACHING ### cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = "local" # default to an in-memory cache redis_cache = None cache_config: Final[dict[str, Any]] = {} self.client_ttl = client_ttl - self.kubernetes_pod_discovery: Final = KubernetesPodDiscovery() + self._core_kubernetes_pod_discovery: Final = KubernetesPodDiscovery() if redis_url is not None or (redis_host is not None and redis_port is not None): cache_type = "redis" @@ -1492,15 +1651,15 @@ class Router: if litellm.cache is None: # the cache can be initialized on the proxy server. We should not overwrite it litellm.cache = litellm.Cache(type=cache_type, **cache_config) - self.cache_responses = cache_responses - self.cache = DualCache( + self._core_cache_responses = cache_responses + self._core_cache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache() ) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc. self._claude_code_session_router_cache: DualCache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache(), ) - self.prompt_caching_pin_cache: DualCache = DualCache( + self._core_prompt_caching_pin_cache: DualCache = DualCache( redis_cache=redis_cache, in_memory_cache=InMemoryCache(max_size_in_memory=PROMPT_CACHE_PIN_CACHE_MAX_SIZE), ) @@ -1516,21 +1675,21 @@ class Router: self.default_priority = ( None if isinstance(parsed_default_priority, InvalidPriority) else parsed_default_priority ) - self.default_deployment = ( + self._core_default_deployment = ( None # use this to track the users default deployment, when they want to use model = * ) - self.default_max_parallel_requests = default_max_parallel_requests - self.provider_default_deployment_ids: list[str] = [] - self.pattern_router = PatternMatchRouter() - self.team_pattern_routers: dict[str, PatternMatchRouter] = {} # {"TEAM_ID": PatternMatchRouter} - self.auto_routers: dict[str, list[TaggedPreRoutingStrategy[AutoRouter]]] = {} - self.complexity_routers: dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] = {} - self.adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] = {} - self.quality_routers: dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] = {} - self.routing_plugins: list[RoutingPlugin] = list(plugins) if plugins else [] + self._core_default_max_parallel_requests = default_max_parallel_requests + self._core_provider_default_deployment_ids: list[str] = [] + self._core_pattern_router = PatternMatchRouter() + self._core_team_pattern_routers: dict[str, PatternMatchRouter] = {} # {"TEAM_ID": PatternMatchRouter} + self._core_auto_routers: dict[str, list[TaggedPreRoutingStrategy[AutoRouter]]] = {} + self._core_complexity_routers: dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] = {} + self._core_adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] = {} + self._core_quality_routers: dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] = {} + self._core_routing_plugins: list[RoutingPlugin] = list(plugins) if plugins else [] # Initialize model_group_alias early since it's used in set_model_list - self.model_group_alias: dict[str, str | RouterModelGroupAliasItem] = ( + self._core_model_group_alias: dict[str, str | RouterModelGroupAliasItem] = ( model_group_alias or {} ) # dict to store aliases for router, ex. {"gpt-4": "gpt-3.5-turbo"}, all requests with gpt-4 -> get routed to gpt-3.5-turbo group @@ -1538,10 +1697,10 @@ class Router: self.model_id_to_deployment_index_map: dict[str, int] = {} # Initialize model name to deployment indices mapping for O(1) lookups # Maps model_name -> list of indices in model_list - self.model_name_to_deployment_indices: dict[str, list[int]] = {} + self._core_model_name_to_deployment_indices: dict[str, list[int]] = {} # Maps (team_id, team_public_model_name) -> list of indices in model_list - self.team_model_to_deployment_indices: dict[tuple[str, str], list[int]] = {} - self.team_public_model_names: frozenset[str] = frozenset() + self._core_team_model_to_deployment_indices: dict[tuple[str, str], list[int]] = {} + self._core_team_public_model_names: frozenset[str] = frozenset() # Guards model_list and the three index maps above: writers change them, and readers turn an # index into a deployment, under it, so a lookup never lands between a removal and its re-index self._deployment_index_lock = threading.RLock() @@ -1558,8 +1717,8 @@ class Router: # ``id()``-reuse risk after GC). See # ``litellm.proxy.auth.auth_checks._is_model_cost_zero``. self._zero_cost_cache: dict[str, bool] = {} - self.cached_deployment_model_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( - self.get_deployment_model_info + self._core_cached_deployment_model_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( + self._core_get_deployment_model_info ) self._ptu_share_teams = lru_cache(maxsize=1)(self._collect_ptu_share_teams) self._cached_team_ptu_ceiling = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)(self._resolve_team_ptu_ceiling) @@ -1583,31 +1742,33 @@ class Router: if model_list is not None: # set_model_list will build indices automatically self.set_model_list(model_list) - self.healthy_deployments: list = self.model_list + self.healthy_deployments: list = self._core_model_list for m in model_list: if "model" in m["litellm_params"]: - self.deployment_latency_map[m["litellm_params"]["model"]] = 0 + self._core_deployment_latency_map[m["litellm_params"]["model"]] = 0 else: - self.model_list: list = [] # initialize an empty list - to allow _add_deployment and delete_deployment to work + self._core_model_list: list = [] # initialize an empty list - to allow _add_deployment and delete_deployment to work # Track this router so a price data reload can rebuild its deployments' # cost-map entries from the list it is serving at that moment, whether # those deployments came from model_list or were added later from the DB. _live_routers.add(self) if allowed_fails is not None: - self.allowed_fails = allowed_fails + self._core_allowed_fails = allowed_fails else: - self.allowed_fails = litellm.allowed_fails + self._core_allowed_fails = litellm.allowed_fails self.explicit_allowed_fails = allowed_fails - self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS - self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time) - self.disable_cooldowns = disable_cooldowns + self._core_cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS + self._core_cooldown_cache = CooldownCache( + cache=self._core_cache, default_cooldown_time=self._core_cooldown_time + ) + self._core_disable_cooldowns = disable_cooldowns self.fallback_on_unsupported_params = fallback_on_unsupported_params - self.enable_health_check_routing = enable_health_check_routing + self._core_enable_health_check_routing = enable_health_check_routing self.enable_weighted_failover = enable_weighted_failover self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enable_mid_stream_fallback_continuation) - self.health_check_ignore_transient_errors = health_check_ignore_transient_errors - self.background_health_check_model_groups: frozenset[str] | None = ( + self._core_health_check_ignore_transient_errors = health_check_ignore_transient_errors + self._core_background_health_check_model_groups: frozenset[str] | None = ( frozenset(background_health_check_model_groups) if background_health_check_model_groups is not None else None @@ -1615,7 +1776,9 @@ class Router: _staleness: Final = health_check_staleness_threshold or ( DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER ) - self.health_state_cache = DeploymentHealthCache(cache=self.cache, staleness_threshold=float(_staleness)) + self._core_health_state_cache = DeploymentHealthCache( + cache=self._core_cache, staleness_threshold=float(_staleness) + ) if num_retries is not None: self.num_retries = num_retries @@ -1640,7 +1803,7 @@ class Router: self.stream_timeout = stream_timeout self.retry_after = retry_after - self.routing_strategy = self._normalize_strategy(routing_strategy) + self._core_routing_strategy = self._normalize_strategy(routing_strategy) self._routing_groups_input: list[RoutingGroup | dict] | None = routing_groups ## SETTING FALLBACKS ## @@ -1649,7 +1812,7 @@ class Router: self.validate_fallbacks(fallback_param=_fallbacks) ### set fallbacks - self.fallbacks = _fallbacks + self._core_fallbacks = _fallbacks if default_fallbacks is not None or litellm.default_fallbacks is not None: _fallbacks = default_fallbacks or litellm.default_fallbacks @@ -1657,16 +1820,16 @@ class Router: raise TypeError( f"default_fallbacks must be a list of model names, got '{_fallbacks}'. Use ['{_fallbacks}']" ) - if self.fallbacks is not None: - self.fallbacks.append({"*": _fallbacks}) + if self._core_fallbacks is not None: + self._core_fallbacks.append({"*": _fallbacks}) else: - self.fallbacks = [{"*": _fallbacks}] + self._core_fallbacks = [{"*": _fallbacks}] - self.context_window_fallbacks = context_window_fallbacks or litellm.context_window_fallbacks + self._core_context_window_fallbacks = context_window_fallbacks or litellm.context_window_fallbacks _content_policy_fallbacks: Final = content_policy_fallbacks or litellm.content_policy_fallbacks self.validate_fallbacks(fallback_param=_content_policy_fallbacks) - self.content_policy_fallbacks = _content_policy_fallbacks + self._core_content_policy_fallbacks = _content_policy_fallbacks self.total_calls: defaultdict = defaultdict(int) # dict to store total calls made to each model self.fail_calls: defaultdict = defaultdict(int) # dict to store fail_calls made to each model self.success_calls: defaultdict = defaultdict(int) # dict to store success_calls made to each model @@ -1676,10 +1839,10 @@ class Router: self.chat = litellm.Chat(params=default_litellm_params, router_obj=self) # default litellm args - self.default_litellm_params = default_litellm_params - self.default_litellm_params.setdefault("timeout", timeout) - self.default_litellm_params.setdefault("max_retries", 0) - self.default_litellm_params.setdefault("metadata", {}).update({"caching_groups": caching_groups}) + self._core_default_litellm_params = default_litellm_params + self._core_default_litellm_params.setdefault("timeout", timeout) + self._core_default_litellm_params.setdefault("max_retries", 0) + self._core_default_litellm_params.setdefault("metadata", {}).update({"caching_groups": caching_groups}) self.deployment_stats: dict = {} # used for debugging load balancing """ @@ -1727,10 +1890,10 @@ class Router: else: litellm.failure_callback = [self.deployment_callback_on_failure] self.routing_strategy_args = routing_strategy_args - self.provider_budget_config = provider_budget_config + self._core_provider_budget_config = provider_budget_config self.router_budget_logger: RouterBudgetLimiting | None = None if RouterBudgetLimiting.should_init_router_budget_limiter( - model_list=model_list, provider_budget_config=self.provider_budget_config + model_list=model_list, provider_budget_config=self._core_provider_budget_config ): if optional_pre_call_checks is not None: optional_pre_call_checks.append("router_budget_limiting") @@ -1750,17 +1913,17 @@ class Router: self.model_group_retry_policy: dict[str, RetryPolicy] | None = model_group_retry_policy - self.allowed_fails_policy: AllowedFailsPolicy | None = None + self._core_allowed_fails_policy: AllowedFailsPolicy | None = None if allowed_fails_policy is not None: if isinstance(allowed_fails_policy, dict): - self.allowed_fails_policy = AllowedFailsPolicy(**allowed_fails_policy) + self._core_allowed_fails_policy = AllowedFailsPolicy(**allowed_fails_policy) elif isinstance(allowed_fails_policy, AllowedFailsPolicy): - self.allowed_fails_policy = allowed_fails_policy + self._core_allowed_fails_policy = allowed_fails_policy - if self.allowed_fails_policy is not None: + if self._core_allowed_fails_policy is not None: verbose_router_logger.info( "\x1b[32mRouter Custom Allowed Fails Policy Set:\n%s\x1b[0m", - self.allowed_fails_policy.model_dump(exclude_none=True), + self._core_allowed_fails_policy.model_dump(exclude_none=True), ) self.alerting_config: AlertingConfig | None = alerting_config @@ -1805,10 +1968,10 @@ class Router: @staticmethod def _register_continuation_prefill_check() -> None: - # Process-global on purpose: Router.discard() must not drop a filter another router still needs. + # Process-global on purpose: Router._core_discard() must not drop a filter another router still needs. litellm.logging_callback_manager.add_litellm_callback(ContinuationPrefillDeploymentCheck()) - def discard(self): + def _core_discard(self) -> None: """ Pseudo-destructor to be invoked to clean up global data structures when router is no longer used. For now, unhook router's callbacks from all lists @@ -1842,16 +2005,16 @@ class Router: adaptive_keys: Final = frozenset( map(litellm.logging_callback_manager.get_custom_logger_key, adaptive_router_hooks) ) - adaptive_survivor: Final = next((router for router in survivors if router.adaptive_routers), None) + adaptive_survivor: Final = next((router for router in survivors if router._core_adaptive_routers), None) if adaptive_survivor is not None and vacated_keys & adaptive_keys: adaptive_survivor._sync_adaptive_router_hooks() - def serve_through_request_logging(self) -> None: + def _core_serve_through_request_logging(self) -> None: self._request_scoped_selectors = (*self._callback_selectors(), *self._request_adaptive_router_hooks()) self._request_scoped_policies = (*self._optional_loggers(), *self._slack_alerting_loggers()) self._release_global_callbacks() - def bind_to_request_logging(self, logging_obj: LiteLLMLogging) -> None: + def _core_bind_to_request_logging(self, logging_obj: LiteLLMLogging) -> None: logging_obj.add_dynamic_function_callbacks( async_success=self.deployment_callback_on_success, failure=self.deployment_callback_on_failure, @@ -1882,7 +2045,7 @@ class Router: return tuple( AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy) - for tagged in chain.from_iterable(self.adaptive_routers.values()) + for tagged in chain.from_iterable(self._core_adaptive_routers.values()) ) def _routing_callbacks(self) -> Iterable[object]: @@ -1903,7 +2066,7 @@ class Router: strategy.retire() def _slack_alerting_loggers(self) -> tuple[CustomLogger, ...]: - slack_alerting_logger: Final = getattr(self, "slack_alerting_logger", None) + slack_alerting_logger: Final = getattr(self, "_core_slack_alerting_logger", None) return (slack_alerting_logger,) if isinstance(slack_alerting_logger, CustomLogger) else () def _adaptive_router_hooks(self) -> tuple[CustomLogger, ...]: @@ -1912,7 +2075,7 @@ class Router: ) own_strategy_ids: Final = frozenset( - id(tagged.strategy) for tagged in chain.from_iterable(self.adaptive_routers.values()) + id(tagged.strategy) for tagged in chain.from_iterable(self._core_adaptive_routers.values()) ) hooks: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook) return tuple( @@ -1945,7 +2108,7 @@ class Router: else: return RedisCache(**cache_config) - def update_redis_cache(self, cache: RedisCache) -> None: + def _core_update_redis_cache(self, cache: RedisCache) -> None: """ Update the redis cache for the router, if none set. @@ -1956,11 +2119,9 @@ class Router: ``` and caching to just work. """ - self.cache.attach_redis_cache(cache) + self._core_cache.attach_redis_cache(cache) self._claude_code_session_router_cache.attach_redis_cache(cache) - self.prompt_caching_pin_cache.attach_redis_cache(cache) - - _update_redis_cache = update_redis_cache + self._core_prompt_caching_pin_cache.attach_redis_cache(cache) # Maps a routing strategy string to the attribute on `self` that holds # the default group's strategy selector for that strategy. (The selectors @@ -2001,25 +2162,25 @@ class Router: selector: RouterStrategySelector | None = None match self._normalize_strategy(strategy): case RoutingStrategy.LEAST_BUSY.value: - selector = LeastBusyLoggingHandler(router_cache=self.cache) + selector = LeastBusyLoggingHandler(router_cache=self._core_cache) case RoutingStrategy.USAGE_BASED_ROUTING.value: selector = LowestTPMLoggingHandler( - router_cache=self.cache, + router_cache=self._core_cache, routing_args=routing_strategy_args, ) case RoutingStrategy.USAGE_BASED_ROUTING_V2.value: selector = LowestTPMLoggingHandler_v2( - router_cache=self.cache, + router_cache=self._core_cache, routing_args=routing_strategy_args, ) case RoutingStrategy.LATENCY_BASED.value: selector = LowestLatencyLoggingHandler( - router_cache=self.cache, + router_cache=self._core_cache, routing_args=routing_strategy_args, ) case RoutingStrategy.COST_BASED.value: selector = LowestCostLoggingHandler( - router_cache=self.cache, + router_cache=self._core_cache, routing_args={}, ) case _: @@ -2072,7 +2233,7 @@ class Router: Latency/usage state survives the rebuild: it lives in the shared router cache, not on the selector. """ - strategy: Final = self._normalize_strategy(self.routing_strategy) + strategy: Final = self._normalize_strategy(self._core_routing_strategy) if strategy == "lar1": from litellm.router_strategy.lar1_routing import apply_lar1_routing_strategy @@ -2146,10 +2307,12 @@ class Router: self._replace_routing_groups(()) return - known_model_names: Final = frozenset(m["model_name"] for m in (self.model_list or ()) if m.get("model_name")) + known_model_names: Final = frozenset( + m["model_name"] for m in (self._core_model_list or ()) if m.get("model_name") + ) groups: Final = parse_routing_groups(groups_input, known_model_names=known_model_names) - alias_names: Final = frozenset(self.model_group_alias or ()) + alias_names: Final = frozenset(self._core_model_group_alias or ()) for group in groups: if group.group_name in known_model_names or group.group_name in alias_names: if group.routing_strategy == "priority": @@ -2187,7 +2350,7 @@ class Router: if selector is not None: self._register_router_selector(selector) - self.routing_groups: dict[str, RoutingGroup] = {group.group_name: group for group, _ in built} + self._core_routing_groups: dict[str, RoutingGroup] = {group.group_name: group for group, _ in built} self._model_to_group: dict[str, str] = { model_name: group.group_name for group, _ in built @@ -2211,13 +2374,13 @@ class Router: models win over indirection); config-time collisions are rejected by `_init_routing_groups`. """ - if not self.routing_groups: + if not self._core_routing_groups: return None - group: Final = self.routing_groups.get(model_name) + group: Final = self._core_routing_groups.get(model_name) if ( group is None - or model_name in self.model_name_to_deployment_indices - or model_name in (self.model_group_alias or {}) + or model_name in self._core_model_name_to_deployment_indices + or model_name in (self._core_model_group_alias or {}) ): return None return group @@ -2236,7 +2399,7 @@ class Router: specific deployment > model id > model_group_alias > routing group > model_name > team/pattern/default fallbacks. """ - if not self.routing_groups: + if not self._core_routing_groups: return None routing_group: Final = self.get_routing_group(model) if routing_group is None: @@ -2244,15 +2407,15 @@ class Router: return [ apply_routing_group_priority(routing_group, member, deployment) for member in routing_group.models - for deployment in self.get_all_deployments(model_name=member, team_id=team_id) + for deployment in self._core_get_all_deployments(model_name=member, team_id=team_id) ] def _is_priority_routing_group(self, model: str) -> bool: - resolved: Final = self.get_model_from_alias(model=model) or model + resolved: Final = self._core_get_model_from_alias(model=model) or model group: Final = self.get_routing_group(resolved) return group is not None and group.routing_strategy == "priority" - def is_recognized_model(self, model: str) -> bool: + def _core_is_recognized_model(self, model: str) -> bool: """ Whether `model` names something this router serves directly: a deployment model_name, a deployment id, a `model_group_alias`, or a @@ -2261,13 +2424,13 @@ class Router: default-deployment, and deployment-name fallbacks stay caller policy. """ return ( - model in self.model_names - or self.has_model_id(model) - or (self.model_group_alias is not None and model in self.model_group_alias) + model in self._core_model_names + or self._core_has_model_id(model) + or (self._core_model_group_alias is not None and model in self._core_model_group_alias) or self.get_routing_group(model) is not None ) - def routing_group_has_alternatives(self, model_group: str | None) -> bool: + def _core_routing_group_has_alternatives(self, model_group: str | None) -> bool: """ True when `model_group` names a callable routing group whose member union spans more than one deployment. Cooldown handling passes the @@ -2278,14 +2441,14 @@ class Router: """ if model_group is None: return False - resolved: Final = self.get_model_from_alias(model=model_group) or model_group + resolved: Final = self._core_get_model_from_alias(model=model_group) or model_group group: Final = self.get_routing_group(resolved) if group is None: return False - return sum(len(self.model_name_to_deployment_indices.get(member) or ()) for member in group.models) > 1 + return sum(len(self._core_model_name_to_deployment_indices.get(member) or ()) for member in group.models) > 1 - def team_model_has_alternatives(self, deployment_id: str) -> bool: - deployment: Final = self.get_deployment(model_id=deployment_id) + def _core_team_model_has_alternatives(self, deployment_id: str) -> bool: + deployment: Final = self._core_get_deployment(model_id=deployment_id) if deployment is None: return False team_id: Final = deployment.model_info.team_id @@ -2333,7 +2496,7 @@ class Router: caches it for the router's lifetime so its usage/latency state persists across requests. """ - if strategy == self._normalize_strategy(self.routing_strategy): + if strategy == self._normalize_strategy(self._core_routing_strategy): attr: Final = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy) return getattr(self, attr, None) if attr is not None else None with self._override_selectors_lock: @@ -2379,14 +2542,14 @@ class Router: def _globally_registered_strategies(self) -> frozenset[str]: configured: Final = ( - self.routing_strategy, - *(group.routing_strategy for group in self.routing_groups.values()), + self._core_routing_strategy, + *(group.routing_strategy for group in self._core_routing_groups.values()), ) return frozenset( normalized for normalized in map(self._normalize_strategy, configured) if normalized is not None ) - def arm_routing_read_prefetch(self, model: str, request_kwargs: dict[str, object] | None = None) -> None: + def _core_arm_routing_read_prefetch(self, model: str, request_kwargs: dict[str, object] | None = None) -> None: """Declare the cooldown read (and, for usage-based routing, the usage read) that `async_get_available_deployment` will make for `model` on the request's Redis batch, so admission's flush carries it. A miss (alias, no batch) costs nothing: routing then reads as it always has.""" @@ -2397,7 +2560,7 @@ class Router: if strategy == "usage-based-routing-v2" and isinstance(selector, LowestTPMLoggingHandler_v2) else None ) - deployments: Final = self.get_model_list(model_name=model) + deployments: Final = self._core_get_model_list(model_name=model) if deployments: RoutingPrefetch.arm(self, usage_selector, deployments) except Exception as e: # noqa: BLE001 # a prefetch is an optimisation, never a reason to fail the request @@ -2422,7 +2585,7 @@ class Router: driven by the router's top-level `routing_strategy` / `routing_strategy_args`. - `self.routing_strategy` may be either a string or a `RoutingStrategy` + `self._core_routing_strategy` may be either a string or a `RoutingStrategy` enum member (the constructor accepts both), so it is normalized to a string here. Downstream call sites and `_select_deployment_*` arms compare against string literals. @@ -2434,20 +2597,20 @@ class Router: self._bind_override_selector_to_request(override, override_selector, request_kwargs) return override, override_selector - resolved_model: Final = self.get_model_from_alias(model=model) or model + resolved_model: Final = self._core_get_model_from_alias(model=model) or model group_name: Final = ( resolved_model if self.get_routing_group(resolved_model) is not None else self._model_to_group.get(resolved_model) ) if group_name is None: - strategy = self._normalize_strategy(self.routing_strategy) + strategy = self._normalize_strategy(self._core_routing_strategy) attr: Final = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy or "") selector = getattr(self, attr, None) if attr is not None else None verbose_router_logger.debug("routing_group=default model=%s strategy=%s", model, strategy) return strategy, selector - group: Final = self.routing_groups[group_name] + group: Final = self._core_routing_groups[group_name] if group.routing_strategy == "priority": return "simple-shuffle", None strategy = self._normalize_strategy(group.routing_strategy) @@ -2978,7 +3141,7 @@ class Router: if self.optional_callbacks is None: self.optional_callbacks = [] affinity_callback: Final = DeploymentAffinityCheck( - cache=self.cache, + cache=self._core_cache, ttl_seconds=self.deployment_affinity_ttl_seconds, enable_user_key_affinity=False, enable_responses_api_affinity=False, @@ -3025,7 +3188,7 @@ class Router: existing_affinity_callback.model_group_affinity_config = self.model_group_affinity_config else: affinity_callback: Final = DeploymentAffinityCheck( - cache=self.cache, + cache=self._core_cache, ttl_seconds=self.deployment_affinity_ttl_seconds, enable_user_key_affinity=enable_user_key_affinity, enable_responses_api_affinity=enable_responses_api_affinity, @@ -3058,19 +3221,19 @@ class Router: continue if pre_call_check == "prompt_caching": _callback = PromptCachingDeploymentCheck( - cache=self.prompt_caching_pin_cache, is_priority_group=self._is_priority_routing_group + cache=self._core_prompt_caching_pin_cache, is_priority_group=self._is_priority_routing_group ) elif pre_call_check == "router_budget_limiting": - if self.get_router_deployment_budget_limiter() is not None: + if self._core_get_router_deployment_budget_limiter() is not None: continue _callback = RouterBudgetLimiting( - dual_cache=self.cache, - provider_budget_config=self.provider_budget_config, - model_list=self.model_list, + dual_cache=self._core_cache, + provider_budget_config=self._core_provider_budget_config, + model_list=self._core_model_list, ) self.router_budget_logger = _callback elif pre_call_check == "enforce_model_rate_limits": - _callback = ModelRateLimitingCheck(dual_cache=self.cache) + _callback = ModelRateLimitingCheck(dual_cache=self._core_cache) if _callback is None: continue @@ -3278,14 +3441,14 @@ class Router: ### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit) ## only run if model group given, not model id - if model in self.model_names or not self.has_model_id(model): + if model in self._core_model_names or not self._core_has_model_id(model): self.routing_strategy_pre_call_checks(deployment=deployment) input_kwargs: Final = without_router_only_kwargs( { **litellm_params, "messages": messages, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -3472,7 +3635,7 @@ class Router: if not is_prompt_management_model: controls: Final = per_request_fallback_controls(kwargs) kwargs[MID_STREAM_FALLBACK_CONTROLS_KEY] = controls # rebind-ok: forwarded to every hop - request_priority, request_kwargs = self.split_request_priority(model=model, kwargs=kwargs) + request_priority, request_kwargs = self._core_split_request_priority(model=model, kwargs=kwargs) original_function: Final[Callable[..., Awaitable[object]]] = ( self._prompt_management_acompletion(model=model, messages=messages) if is_prompt_management_model @@ -3705,12 +3868,12 @@ class Router: try: # Use the router's fallback system model_group: Final = cast(str, initial_kwargs.get("model")) - fallbacks: Final[list | None] = initial_kwargs.get("fallbacks", self.fallbacks) + fallbacks: Final[list | None] = initial_kwargs.get("fallbacks", self._core_fallbacks) context_window_fallbacks: Final[list | None] = initial_kwargs.get( - "context_window_fallbacks", self.context_window_fallbacks + "context_window_fallbacks", self._core_context_window_fallbacks ) content_policy_fallbacks: Final[list | None] = initial_kwargs.get( - "content_policy_fallbacks", self.content_policy_fallbacks + "content_policy_fallbacks", self._core_content_policy_fallbacks ) initial_kwargs["original_function"] = self._acompletion continuation_messages: Final = ( @@ -3958,7 +4121,7 @@ class Router: ) -> Mapping[str, object]: """The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies.""" deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} - default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self.default_litellm_params) + default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params) defaults: Final = {key: value for key, value in default_params.items() if value is not None} deployment_tool_choice: Final = ( {"tool_choice": deployment_params["tool_choice"]} @@ -3969,9 +4132,9 @@ class Router: def _names_deployment_that_cannot_continue(self, target: object) -> bool: name: Final = fallback_target_name(target) - if name is None or name in self.model_names or not self.has_model_id(name): + if name is None or name in self._core_model_names or not self._core_has_model_id(name): return False - deployment: Final = self.get_deployment(model_id=name) + deployment: Final = self._core_get_deployment(model_id=name) return deployment is not None and not deployment_can_continue(deployment.model_dump(exclude_none=True)) def _mid_stream_continuation_eligible( @@ -3988,7 +4151,7 @@ class Router: if isinstance(e.original_exception, litellm.ContentPolicyViolationError): return False cut_deployment: Final = ( - self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None + self._core_get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None ) if isinstance(cut_deployment_id, str) and cut_deployment is None: return False @@ -3998,7 +4161,9 @@ class Router: lookup_groups: Final = fallback_lookup_groups( request_kwargs, requested_group if isinstance(requested_group, str) else None ) - targets: Final = continuation_fallback_targets(request_kwargs.get("fallbacks", self.fallbacks), lookup_groups) + targets: Final = continuation_fallback_targets( + request_kwargs.get("fallbacks", self._core_fallbacks), lookup_groups + ) if any(fallback_target_overrides_request_params(target) for target in targets): return False if any(self._names_deployment_that_cannot_continue(target) for target in targets): @@ -4297,12 +4462,14 @@ class Router: _request_fallback_list( initial_kwargs, "content_policy_fallbacks", - cast("list[object] | None", self.content_policy_fallbacks), # cast-ok: untyped router attribute + cast( # cast-ok: untyped router attribute + "list[object] | None", self._core_content_policy_fallbacks + ), ), _request_fallback_list( initial_kwargs, "fallbacks", - cast("list[object] | None", self.fallbacks), # cast-ok: untyped router attribute + cast("list[object] | None", self._core_fallbacks), # cast-ok: untyped router attribute ), isinstance(hop_depth, int) and hop_depth > 0, ) @@ -4323,7 +4490,7 @@ class Router: Router._raise_original_mid_stream_error(e) cut_deployment_id: Final = (get_hidden_params(source_iterator) or {}).get("model_id") cut_deployment: Final = ( - self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None + self._core_get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None ) if constrains_responses_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)): Router._raise_original_mid_stream_error(e) @@ -4373,13 +4540,13 @@ class Router: try: model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: model group fallbacks: Final[list | None] = initial_kwargs.get( # mutable-ok: matches the common_utils list|None param - "fallbacks", self.fallbacks + "fallbacks", self._core_fallbacks ) context_window_fallbacks: Final[list | None] = initial_kwargs.get( # mutable-ok: matches the param below - "context_window_fallbacks", self.context_window_fallbacks + "context_window_fallbacks", self._core_context_window_fallbacks ) content_policy_fallbacks: Final[list | None] = initial_kwargs.get( # mutable-ok: matches the param below - "content_policy_fallbacks", self.content_policy_fallbacks + "content_policy_fallbacks", self._core_content_policy_fallbacks ) initial_kwargs["original_function"] = ( # rebind-ok: the fallback chain re-enters on the same kwargs self._ageneric_api_call_with_fallbacks_responses_attempt @@ -4498,14 +4665,14 @@ class Router: try: model_group: Final = cast(str, initial_kwargs.get("model")) - fallbacks: Final[list | None] = initial_kwargs.get("fallbacks", router_self.fallbacks) + fallbacks: Final[list | None] = initial_kwargs.get("fallbacks", router_self._core_fallbacks) context_window_fallbacks: Final[list | None] = initial_kwargs.get( "context_window_fallbacks", - router_self.context_window_fallbacks, + router_self._core_context_window_fallbacks, ) content_policy_fallbacks: Final[list | None] = initial_kwargs.get( "content_policy_fallbacks", - router_self.content_policy_fallbacks, + router_self._core_content_policy_fallbacks, ) initial_kwargs["original_function"] = router_self._completion initial_kwargs["messages"] = messages @@ -4619,7 +4786,7 @@ class Router: parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) start_time: Final = time.time() - deployment = await self.async_get_available_deployment( + deployment = await self._core_async_get_available_deployment( model=model, messages=messages, specific_deployment=kwargs.pop("specific_deployment", None), @@ -4683,7 +4850,7 @@ class Router: { **litellm_params, "messages": messages, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -4791,7 +4958,7 @@ class Router: """ kwargs.setdefault("litellm_trace_id", str(uuid.uuid4())) model_group_alias: str | None = None - if self.get_model_from_alias(model=model): + if self._core_get_model_from_alias(model=model): model_group_alias = model kwargs.setdefault(metadata_variable_name, {}).update( {"model_group": model, "model_group_alias": model_group_alias} @@ -4867,7 +5034,7 @@ class Router: Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name """ # 1) copy your defaults and pull out metadata - defaults: Final = dict(without_retrieve_stream_defaults(self.default_litellm_params, kwargs)) + defaults: Final = dict(without_retrieve_stream_defaults(self._core_default_litellm_params, kwargs)) metadata_defaults: Final = defaults.pop("metadata", {}) or {} # 2) add any non-metadata defaults that aren't already in kwargs @@ -4888,7 +5055,7 @@ class Router: This deployment is deliberately never registered with the router (no upsert_deployment/add_deployment call): doing so used to add it to - self.model_list under the shared model_name, which made a request-scoped, + self._core_model_list under the shared model_name, which made a request-scoped, caller-supplied provider credential a permanent, load-balanced deployment that every other caller of that model group could be routed onto. Its pricing is still registered directly, so a custom price configured on the @@ -4902,7 +5069,7 @@ class Router: function_name=function_name, ) model_group: Final = kwargs.get(metadata_variable_name, {}).get("model_group") - _model_id: Final = self.generate_model_id(model_group=model_group, litellm_params=dynamic_litellm_params) + _model_id: Final = self._core_generate_model_id(model_group=model_group, litellm_params=dynamic_litellm_params) original_model_id: Final = model_info.get("id") model_info["id"] = _model_id model_info["original_model_id"] = original_model_id @@ -4957,7 +5124,7 @@ class Router: ): kwargs.pop(key, None) kwargs.pop("daily_reset_time", None) # pyright: ignore[reportUnknownMemberType] # kwargs is an untyped dict - defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self.default_litellm_params) + defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self._core_default_litellm_params) self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=defaults) model_info = deployment.get("model_info", {}).copy() @@ -5031,7 +5198,7 @@ class Router: # Refund and clear any reservation the previous deployment attempt left # here before it's wiped below, instead of relying on that attempt's # (possibly still-pending) failure event to do it. - refund_stale_reservation_before_retry(self.cache, kwargs) + refund_stale_reservation_before_retry(self._core_cache, kwargs) set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment)) kwargs[metadata_variable_name].setdefault( @@ -5100,7 +5267,7 @@ class Router: if self.stream_timeout is not None else self.request_timeout if self.request_timeout is not None - else self.default_litellm_params.get("stream_timeout") + else self._core_default_litellm_params.get("stream_timeout") ) passthrough_timeout: Final = partial( resolve_llm_passthrough_timeout, @@ -5196,7 +5363,7 @@ class Router: or data.get("stream_timeout", None) # timeout set on litellm_params for this deployment or self.stream_timeout # timeout set on router or self.request_timeout # litellm_settings.request_timeout (per-attempt) - or self.default_litellm_params.get("stream_timeout", None) + or self._core_default_litellm_params.get("stream_timeout", None) ) def _get_non_stream_timeout( @@ -5211,7 +5378,7 @@ class Router: or data.get("request_timeout", None) # timeout set on litellm_params for this deployment or self.request_timeout # litellm_settings.request_timeout (per-attempt) or self.timeout # timeout set on router (router_settings.timeout) - or self.default_litellm_params.get("timeout", None) + or self._core_default_litellm_params.get("timeout", None) ) def _get_timeout(self, kwargs: Mapping[str, object], data: Mapping[str, object] | GenericLiteLLMParams) -> object: @@ -5472,12 +5639,12 @@ class Router: stream=False, **kwargs, ): - queued_priority, _ = self.split_request_priority(model=model, kwargs={**kwargs, "priority": priority}) + queued_priority, _ = self._core_split_request_priority(model=model, kwargs={**kwargs, "priority": priority}) if queued_priority is None: raise priority_error(model=model, message=unqueueable_priority_message(priority)) return await self.acompletion(model=model, messages=messages, stream=stream, priority=queued_priority, **kwargs) - def split_request_priority( + def _core_split_request_priority( self, model: str, kwargs: Mapping[str, object] ) -> tuple[int | None, Mapping[str, object]]: request_priority: Final = resolve_request_priority( @@ -5496,7 +5663,7 @@ class Router: def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool: return request_drops_params( - kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs) + kwargs, self._core_default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs) ) async def _schedule_factory( @@ -5548,7 +5715,7 @@ class Router: ) def _is_prompt_management_model(self, model: str) -> bool: - model_list: Final = self.get_model_list(model_name=model) + model_list: Final = self._core_get_model_list(model_name=model) if model_list is None or len(model_list) != 1: return False @@ -5560,7 +5727,7 @@ class Router: return split_litellm_model in litellm._known_custom_logger_compatible_callbacks def _pinned_deployment(self, model: str) -> Deployment | None: - return self.get_deployment(model_id=model) if model not in self.model_names else None + return self._core_get_deployment(model_id=model) if model not in self._core_model_names else None def _request_deployment_params(self, model: str, kwargs: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: deployment: Final = self._pinned_deployment(model) @@ -5569,7 +5736,7 @@ class Router: candidates: Final = ( self._get_deployment_by_litellm_model(model=model) if kwargs.get("specific_deployment") is True - else self.get_model_list(model_name=model, team_id=get_request_team_id(kwargs)) or () + else self._core_get_model_list(model_name=model, team_id=get_request_team_id(kwargs)) or () ) return tuple(candidate["litellm_params"] for candidate in candidates) @@ -5603,7 +5770,7 @@ class Router: caller_timeout: Final = kwargs.get("timeout") or kwargs.get("request_timeout") caller_stream_timeout: Final = kwargs.get("stream_timeout") specific_deployment: Final = kwargs.pop("specific_deployment", None) - prompt_management_deployment: Final = await self.async_get_available_deployment( + prompt_management_deployment: Final = await self._core_async_get_available_deployment( model=model, messages=cast(list[dict[str, str]], messages), # cast-ok: selection reads messages structurally specific_deployment=specific_deployment, @@ -5669,7 +5836,7 @@ class Router: kwargs["prompt_variables"] = prompt_variables kwargs["prompt_label"] = prompt_label - _model_list: Final = self.get_model_list(model_name=model) + _model_list: Final = self._core_get_model_list(model_name=model) if _model_list is None or len(_model_list) == 0: # if direct call to model kwargs.pop("original_function") return await litellm.acompletion(**kwargs) @@ -5729,7 +5896,7 @@ class Router: **{ **data, "prompt": prompt, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -5769,7 +5936,7 @@ class Router: try: verbose_router_logger.debug("Inside _image_generation()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self.async_get_available_deployment( + deployment: Final = await self._core_async_get_available_deployment( model=model, messages=[{"role": "user", "content": "prompt"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -5791,7 +5958,7 @@ class Router: **{ **data, "prompt": prompt, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -5854,7 +6021,7 @@ class Router: try: verbose_router_logger.debug("Inside _atranscription()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self.async_get_available_deployment( + deployment: Final = await self._core_async_get_available_deployment( model=model, messages=[{"role": "user", "content": "prompt"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -5874,7 +6041,7 @@ class Router: **{ **data, "file": file, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -5947,7 +6114,7 @@ class Router: try: verbose_router_logger.debug("Inside _aspeech()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self.async_get_available_deployment( + deployment: Final = await self._core_async_get_available_deployment( model=model, messages=[{"role": "user", "content": "prompt"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -6008,7 +6175,7 @@ class Router: model_name = None try: verbose_router_logger.debug("Inside _rerank()- model: %s", model) - deployment: Final = await self.async_get_available_deployment( + deployment: Final = await self._core_async_get_available_deployment( model=model, specific_deployment=kwargs.pop("specific_deployment", None), request_kwargs=kwargs, @@ -6026,7 +6193,7 @@ class Router: response: Final = await litellm.arerank( **{ **data, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -6068,14 +6235,16 @@ class Router: ) data: Final = deployment["litellm_params"].copy() - for k, v in self.default_litellm_params.items(): + for k, v in self._core_default_litellm_params.items(): if k not in kwargs: # prioritize model-specific params > default router params kwargs[k] = v elif k == "metadata": kwargs[k].update(v) # call via litellm.completion() - return litellm.text_completion(**{**data, "prompt": prompt, "caching": self.cache_responses, **kwargs}) + return litellm.text_completion( + **{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs} + ) except Exception as e: raise e @@ -6094,7 +6263,7 @@ class Router: kwargs["original_function"] = self._atext_completion self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) - request_priority, request_kwargs = self.split_request_priority(model=model, kwargs=kwargs) + request_priority, request_kwargs = self._core_split_request_priority(model=model, kwargs=kwargs) if request_priority is None: return await self.async_function_with_fallbacks(**request_kwargs) return await self._schedule_factory( @@ -6119,7 +6288,7 @@ class Router: try: verbose_router_logger.debug("Inside _atext_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self.async_get_available_deployment( + deployment: Final = await self._core_async_get_available_deployment( model=model, messages=[{"role": "user", "content": prompt}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -6141,7 +6310,7 @@ class Router: **{ **data, "prompt": prompt, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -6189,7 +6358,7 @@ class Router: try: verbose_router_logger.debug("Inside _aadapter_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self.async_get_available_deployment( + deployment: Final = await self._core_async_get_available_deployment( model=model, messages=[{"role": "user", "content": "default text"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -6211,7 +6380,7 @@ class Router: **{ **data, "adapter_id": adapter_id, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -6301,7 +6470,7 @@ class Router: **kwargs: Additional arguments """ guardrail_name: Final = model - selected_guardrail: Final = self.get_available_guardrail( + selected_guardrail: Final = self._core_get_available_guardrail( guardrail_name=guardrail_name, ) @@ -6314,7 +6483,7 @@ class Router: response: Final = await original_generic_function(**kwargs) return response - def get_available_guardrail( + def _core_get_available_guardrail( self, guardrail_name: str, ) -> "GuardrailTypedDict": @@ -6329,7 +6498,7 @@ class Router: """ from litellm.router_strategy.simple_shuffle import simple_shuffle - healthy_deployments: Final = [g for g in self.guardrail_list if g.get("guardrail_name") == guardrail_name] + healthy_deployments: Final = [g for g in self._core_guardrail_list if g.get("guardrail_name") == guardrail_name] if not healthy_deployments: raise ValueError(f"No guardrail found with name: {guardrail_name}") @@ -6339,7 +6508,7 @@ class Router: # Use simple_shuffle for weighted selection return simple_shuffle( - resolve_model_alias=self.get_model_from_alias, + resolve_model_alias=self._core_get_model_from_alias, healthy_deployments=healthy_deployments, model=guardrail_name, request_kwargs=None, @@ -6418,12 +6587,14 @@ class Router: parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) try: with self._video_call_pin(handler_name, kwargs), self._route_capability_pin(handler_name): - deployment = await self.async_get_available_deployment( # rebind-ok: set on success, see pre-init - model=model, - request_kwargs=kwargs, - messages=kwargs.get("messages", None), - input=kwargs.get("input", None), - specific_deployment=kwargs.pop("specific_deployment", None), + deployment = ( # rebind-ok: set on success, see pre-init + await self._core_async_get_available_deployment( + model=model, + request_kwargs=kwargs, + messages=kwargs.get("messages", None), + input=kwargs.get("input", None), + specific_deployment=kwargs.pop("specific_deployment", None), + ) ) except Exception as e: if passthrough_on_no_deployment: @@ -6450,7 +6621,7 @@ class Router: response_kwargs: Final = { **without_retrieve_stream_defaults(data, kwargs), - "caching": self.cache_responses, + "caching": self._core_cache_responses, **kwargs, "model": model_name, **_with_router_resolved_session_model(kwargs.get("session"), model_name), @@ -6805,7 +6976,7 @@ class Router: if isinstance(request_num_retries, int): return request_num_retries deployment_id: Final = routed_deployment_id(kwargs) - deployment: Final = self.get_deployment(deployment_id) if deployment_id is not None else None + deployment: Final = self._core_get_deployment(deployment_id) if deployment_id is not None else None deployment_num_retries: Final = _deployment_num_retries(deployment) if deployment_num_retries is not None: return deployment_num_retries @@ -6889,17 +7060,21 @@ class Router: context_window_fallbacks=_request_fallback_list( kwargs, "context_window_fallbacks", - cast("list[object] | None", self.context_window_fallbacks), # cast-ok: untyped router attribute + cast( # cast-ok: untyped router attribute + "list[object] | None", self._core_context_window_fallbacks + ), ), content_policy_fallbacks=_request_fallback_list( kwargs, "content_policy_fallbacks", - cast("list[object] | None", self.content_policy_fallbacks), # cast-ok: untyped router attribute + cast( # cast-ok: untyped router attribute + "list[object] | None", self._core_content_policy_fallbacks + ), ), regular_fallbacks=_request_fallback_list( kwargs, "fallbacks", - cast("list[object] | None", self.fallbacks), # cast-ok: untyped router attribute + cast("list[object] | None", self._core_fallbacks), # cast-ok: untyped router attribute ), ) except Exception: # noqa: BLE001 # should_retry_this_error declines by raising the error it was given @@ -6936,7 +7111,7 @@ class Router: ): return _StreamRetriesExhausted(last_error) if get_metadata_variable_name_from_kwargs(retry_kwargs) in retry_kwargs: - self.log_retry(kwargs=retry_kwargs, e=trigger) + self._core_log_retry(kwargs=retry_kwargs, e=trigger) await asyncio.sleep( self._time_to_sleep_before_retry( e=trigger, @@ -7025,13 +7200,13 @@ class Router: try: model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: model group fallbacks: Final[list | None] = initial_kwargs.get( # mutable-ok: matches the common_utils list|None param - "fallbacks", self.fallbacks + "fallbacks", self._core_fallbacks ) context_window_fallbacks: Final[list | None] = initial_kwargs.get( # mutable-ok: matches the param below - "context_window_fallbacks", self.context_window_fallbacks + "context_window_fallbacks", self._core_context_window_fallbacks ) content_policy_fallbacks: Final[list | None] = initial_kwargs.get( # mutable-ok: matches the param below - "content_policy_fallbacks", self.content_policy_fallbacks + "content_policy_fallbacks", self._core_content_policy_fallbacks ) initial_kwargs["original_function"] = self._ageneric_api_call_with_fallbacks_anthropic_messages_attempt self._update_kwargs_before_fallbacks( @@ -7227,7 +7402,7 @@ class Router: **{ **data, "custom_llm_provider": custom_llm_provider, - "caching": self.cache_responses, + "caching": self._core_cache_responses, **kwargs, } ) @@ -7307,7 +7482,7 @@ class Router: **{ **data, "input": input, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -7353,7 +7528,7 @@ class Router: verbose_router_logger.debug("Inside _aembedding()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) with embedding_deployment_selection(): - deployment: Final = await self.async_get_available_deployment( + deployment: Final = await self._core_async_get_available_deployment( model=model, input=input, specific_deployment=kwargs.pop("specific_deployment", None), @@ -7386,7 +7561,7 @@ class Router: **{ **data, "input": input, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -7436,7 +7611,7 @@ class Router: verbose_router_logger.debug("Inside _atext_completion()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - healthy_deployments: Final = await self.async_get_healthy_deployments( + healthy_deployments: Final = await self._core_async_get_healthy_deployments( model=model, messages=[{"role": "user", "content": "files-api-fake-text"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -7508,7 +7683,7 @@ class Router: **{ **data, "custom_llm_provider": custom_llm_provider, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs_copy, } @@ -7576,7 +7751,7 @@ class Router: from litellm.vector_stores import acreate as avector_store_create_sdk parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self.async_get_available_deployment( + deployment: Final = await self._core_async_get_available_deployment( model=model, messages=[{"role": "user", "content": "vector-store-api-fake-text"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -7609,7 +7784,7 @@ class Router: **{ **data, "custom_llm_provider": custom_llm_provider, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -7675,7 +7850,7 @@ class Router: verbose_router_logger.debug("Inside _acreate_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) with batch_create_routing(): - deployment: Final = await self.async_get_available_deployment( + deployment: Final = await self._core_async_get_available_deployment( model=model, messages=[{"role": "user", "content": "files-api-fake-text"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -7705,7 +7880,7 @@ class Router: **{ **data, "custom_llm_provider": custom_llm_provider, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -7741,7 +7916,7 @@ class Router: if model is not None: filtered_model_list: ( list[DeploymentTypedDict] | list[dict] | dict | None - ) = await self.async_get_healthy_deployments( + ) = await self._core_async_get_healthy_deployments( model=model, messages=[{"role": "user", "content": "retrieve-api-fake-text"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -7749,7 +7924,7 @@ class Router: parent_otel_span=parent_otel_span, ) else: - filtered_model_list = self.get_model_list() + filtered_model_list = self._core_get_model_list() if filtered_model_list is None: raise Exception("Router not yet initialized.") @@ -7873,7 +8048,7 @@ class Router: try: verbose_router_logger.debug("Inside _acancel_batch()- model: %s", model) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - deployment: Final = await self.async_get_available_deployment( + deployment: Final = await self._core_async_get_available_deployment( model=model, messages=[{"role": "user", "content": "batch-api-fake-text"}], specific_deployment=kwargs.pop("specific_deployment", None), @@ -7882,7 +8057,7 @@ class Router: selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id") data: Final = deployment["litellm_params"].copy() - resolved_credentials: Final = self.get_deployment_credentials_with_provider( + resolved_credentials: Final = self._core_get_deployment_credentials_with_provider( model_id=selected_deployment_id or model ) if resolved_credentials is not None: @@ -7910,7 +8085,7 @@ class Router: **{ **data, "custom_llm_provider": custom_llm_provider, - "caching": self.cache_responses, + "caching": self._core_cache_responses, "client": model_client, **kwargs, } @@ -7938,7 +8113,7 @@ class Router: Return all the batches across all deployments of a model group. """ - filtered_model_list: Final = self.get_model_list(model_name=model) + filtered_model_list: Final = self._core_get_model_list(model_name=model) if filtered_model_list is None: raise Exception("Router not yet initialized.") @@ -7987,8 +8162,8 @@ class Router: model=kwargs.get("model", ""), kwargs=kwargs, ) - if kwargs.get("model") and self.get_model_list(model_name=kwargs["model"]): - deployment: Final = await self.async_get_available_deployment( + if kwargs.get("model") and self._core_get_model_list(model_name=kwargs["model"]): + deployment: Final = await self._core_async_get_available_deployment( model=kwargs["model"], request_kwargs=kwargs, ) @@ -8750,7 +8925,7 @@ class Router: # Use wildcard-aware lookup so order-based fallback also works for model # groups resolved via pattern routing (e.g. `openai/*` -> `openai/gpt-4.1-mini`). order_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group - all_deployments: Final = self.get_model_list(model_name=order_model_group, team_id=_request_team_id) or () + all_deployments: Final = self._core_get_model_list(model_name=order_model_group, team_id=_request_team_id) or () _order_set: Final[set] = { litellm.utils.get_deployment_order(d) for d in all_deployments @@ -9002,9 +9177,13 @@ class Router: include_fallback_errors: Final = kwargs.get("include_fallback_errors", False) is True disable_fallbacks: Final[bool | None] = kwargs.pop("disable_fallbacks", False) record_disable_fallbacks(kwargs, disable_fallbacks is True) - fallbacks: Final[list | None] = kwargs.get("fallbacks", self.fallbacks) - context_window_fallbacks: list | None = kwargs.get("context_window_fallbacks", self.context_window_fallbacks) - content_policy_fallbacks: list | None = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) + fallbacks: Final[list | None] = kwargs.get("fallbacks", self._core_fallbacks) + context_window_fallbacks: list | None = kwargs.get( + "context_window_fallbacks", self._core_context_window_fallbacks + ) + content_policy_fallbacks: list | None = kwargs.get( + "content_policy_fallbacks", self._core_content_policy_fallbacks + ) mock_timeout: Final = kwargs.pop("mock_timeout", None) @@ -9109,10 +9288,10 @@ class Router: async def async_function_with_retries(self, *args, **kwargs): verbose_router_logger.debug("Inside async function with retries.") original_function: Final = kwargs.pop("original_function") - fallbacks: Final = kwargs.pop("fallbacks", self.fallbacks) + fallbacks: Final = kwargs.pop("fallbacks", self._core_fallbacks) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) - context_window_fallbacks: Final = kwargs.pop("context_window_fallbacks", self.context_window_fallbacks) - content_policy_fallbacks: Final = kwargs.pop("content_policy_fallbacks", self.content_policy_fallbacks) + context_window_fallbacks: Final = kwargs.pop("context_window_fallbacks", self._core_context_window_fallbacks) + content_policy_fallbacks: Final = kwargs.pop("content_policy_fallbacks", self._core_content_policy_fallbacks) # Support per-request model_group_retry_policy override (from key/team settings) model_group_retry_policy: Final = kwargs.pop("model_group_retry_policy", self.model_group_retry_policy) model_group: Final[str | None] = kwargs.get("model") @@ -9126,7 +9305,7 @@ class Router: ## ADD MODEL GROUP SIZE TO METADATA - used for model_group_rate_limit_error tracking _metadata: Final[dict] = kwargs.get("litellm_metadata", kwargs.get("metadata")) or {} if "model_group" in _metadata and isinstance(_metadata["model_group"], str): - model_list: Final = self.get_model_list(model_name=_metadata["model_group"]) + model_list: Final = self._core_get_model_list(model_name=_metadata["model_group"]) if model_list is not None: _metadata.update({"model_group_size": len(model_list)}) @@ -9205,7 +9384,7 @@ class Router: ## LOGGING if num_retries > 0: - kwargs = self.log_retry(kwargs=kwargs, e=original_exception) + kwargs = self._core_log_retry(kwargs=kwargs, e=original_exception) first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry( exception=original_exception, already_skipped=kwargs.get("_retry_skipped_deployment_ids"), @@ -9254,7 +9433,7 @@ class Router: original_exception = e ## LOGGING - kwargs = self.log_retry(kwargs=kwargs, e=e) + kwargs = self._core_log_retry(kwargs=kwargs, e=e) remaining_retries = num_retries - current_attempt - 1 _model: str | None = kwargs.get("model") if _model is not None: @@ -9352,7 +9531,7 @@ class Router: """ mock_testing_rate_limit_error: Final[bool | None] = kwargs.pop("mock_testing_rate_limit_error", None) - available_models: Final = self.get_model_list(model_name=model_group) + available_models: Final = self._core_get_model_list(model_name=model_group) num_retries: int | None = None if available_models is not None and len(available_models) == 1: @@ -9455,9 +9634,9 @@ class Router: """ Returns the first model from the default_fallbacks list, if it exists. """ - if self.fallbacks is None: + if self._core_fallbacks is None: return None - for fallback in self.fallbacks: + for fallback in self._core_fallbacks: if isinstance(fallback, dict) and "*" in fallback: default_list = fallback["*"] if isinstance(default_list, list) and len(default_list) > 0: @@ -9536,7 +9715,11 @@ class Router: model_group: Final = metadata.get("model_group", None) model_info: Final = litellm_params.get("model_info", {}) or {} deployment_id: Final = model_info.get("id", None) - if model_group is None or deployment_id is None or self.get_deployment(model_id=str(deployment_id)) is None: + if ( + model_group is None + or deployment_id is None + or self._core_get_deployment(model_id=str(deployment_id)) is None + ): return if self.session_affinity_repin_on_failover: @@ -9585,12 +9768,12 @@ class Router: model_group: Final = callback.get_repin_model_group(litellm_params) if model_group is None or self._is_priority_routing_group(model_group): return - registered_ids: Final = self.get_candidate_model_ids_for_route( + registered_ids: Final = self._core_get_candidate_model_ids_for_route( model=model_group, team_id=get_request_team_id(litellm_params) ) cooldown_ids: Final = frozenset( model_id - for model_id, _ in await self.cooldown_cache.async_get_active_cooldowns( + for model_id, _ in await self._core_cooldown_cache.async_get_active_cooldowns( model_ids=list(registered_ids), parent_otel_span=None ) ) @@ -9662,22 +9845,22 @@ class Router: RedisPipelineIncrementOperation(key=key, increment_value=increment_value, ttl=RoutingArgs.ttl.value) for key, increment_value in ((tpm_key, total_tokens), (rpm_key, rpm_increment)) ] - post_increment_values: Final = await self.cache.async_increment_cache_pipeline( + post_increment_values: Final = await self._core_cache.async_increment_cache_pipeline( increment_list=pipeline_operations, parent_otel_span=parent_otel_span, ) - if post_increment_values is not None and self.cache.redis_cache is not None: + if post_increment_values is not None and self._core_cache.redis_cache is not None: for operation, value in zip(pipeline_operations, post_increment_values): - await self.cache.async_set_cache( + await self._core_cache.async_set_cache( operation["key"], int(value), local_only=True, ttl=RoutingArgs.ttl.value ) return tpm_key def _deployment_tracks_usage(self, deployment_id: str, model_group: str) -> bool: - deployment_info: Final = self.get_deployment(model_id=deployment_id) + deployment_info: Final = self._core_get_deployment(model_id=deployment_id) if deployment_info is None: return False - deployment_model_info: Final = self.get_router_model_info( + deployment_model_info: Final = self._core_get_router_model_info( deployment=deployment_info, received_model_name=model_group, ) @@ -9795,7 +9978,7 @@ class Router: caller_failure_model_info.get("id") if caller_failure_model_info is not None else None ) caller_failure_deployment: Final = ( - self.get_deployment(model_id=caller_failure_model_id) + self._core_get_deployment(model_id=caller_failure_model_id) if isinstance(caller_failure_model_id, str) else None ) @@ -9831,7 +10014,7 @@ class Router: elif header_cooldown is not None and header_cooldown >= 0: _time_to_cooldown = header_cooldown else: - _time_to_cooldown = self.cooldown_time + _time_to_cooldown = self._core_cooldown_time if isinstance(_model_info, dict): deployment_id: Final[str | None] = _model_info.get("id") @@ -9892,7 +10075,7 @@ class Router: ## RPM rpm_key: Final = RouterCacheEnum.RPM.value.format(id=id, current_minute=current_minute, model=deployment_name) with service_target(ROUTER_USAGE_TARGET): - await self.cache.async_increment_cache( + await self._core_cache.async_increment_cache( key=rpm_key, value=1, parent_otel_span=parent_otel_span, @@ -9913,7 +10096,7 @@ class Router: """ return get_metadata_variable_name_from_kwargs(kwargs) - def log_retry(self, kwargs: dict, e: Exception) -> dict: + def _core_log_retry(self, kwargs: dict, e: Exception) -> dict: """ When a retry or fallback happens, record which model group, deployment and attempt just failed and why, and count it toward the request-wide num_retries_per_request cap @@ -9969,7 +10152,7 @@ class Router: def _deployment_params_base_model(self, deployment_id: str | None) -> str | None: """The ``base_model`` a deployment declares under ``litellm_params`` (the Azure named-deployment shape), the same key the success path's ``get_base_model_from_metadata`` reads.""" - deployment: Final = self.get_deployment(model_id=deployment_id) if deployment_id is not None else None + deployment: Final = self._core_get_deployment(model_id=deployment_id) if deployment_id is not None else None if deployment is None: return None extra_params: Final = cast( # cast-ok: pydantic extras are untyped, base_model is isinstance-checked below @@ -9987,20 +10170,20 @@ class Router: """ rpm_key: Final = deployment_id - request_count = self.cache.get_cache(key=rpm_key, parent_otel_span=parent_otel_span, local_only=True) + request_count = self._core_cache.get_cache(key=rpm_key, parent_otel_span=parent_otel_span, local_only=True) if request_count is None: request_count = 1 - self.cache.set_cache(key=rpm_key, value=request_count, local_only=True, ttl=60) # only store for 60s + self._core_cache.set_cache(key=rpm_key, value=request_count, local_only=True, ttl=60) # only store for 60s else: request_count += 1 - self.cache.set_cache(key=rpm_key, value=request_count, local_only=True) # don't change existing ttl + self._core_cache.set_cache(key=rpm_key, value=request_count, local_only=True) # don't change existing ttl return request_count def _has_default_fallbacks(self) -> bool: - if self.fallbacks is None: + if self._core_fallbacks is None: return False - for fallback in self.fallbacks: + for fallback in self._core_fallbacks: if isinstance(fallback, dict): if "*" in fallback: return True @@ -10013,7 +10196,7 @@ class Router: selected wins over the requested group. Raising without this returning True would turn a deliverable response into an error the fallback chain cannot recover from. """ - content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) + content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self._core_content_policy_fallbacks) if content_policy_fallbacks is not None: return has_unattempted_fallback_target( self._get_fallback_model_group_for_lookup_groups( @@ -10040,12 +10223,12 @@ class Router: """ if fallbacks_disabled_for_request(kwargs): return False - content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) + content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self._core_content_policy_fallbacks) if content_policy_fallbacks is not None: return self._has_content_policy_fallback(model_group, kwargs) if self._has_default_fallbacks(): return True - fallbacks: Final = kwargs.get("fallbacks", self.fallbacks) + fallbacks: Final = kwargs.get("fallbacks", self._core_fallbacks) if fallbacks is None: return False resolved, _ = get_fallback_model_group_for_lookup_groups( @@ -10061,7 +10244,7 @@ class Router: """ request_team_id: Final[str | None] = (kwargs.get("metadata", {}) or {}).get("user_api_key_team_id") order_model_group: Final = get_pre_routing_selection(kwargs) or model_group - all_deployments: Final = self.get_model_list(model_name=order_model_group, team_id=request_team_id) or () + all_deployments: Final = self._core_get_model_list(model_name=order_model_group, team_id=request_team_id) or () return tuple( sorted( { @@ -10090,10 +10273,10 @@ class Router: skip_up_to: Final = current_target if current_target is not None else order_levels[0] if any(o > skip_up_to for o in order_levels): return True - content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) + content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self._core_content_policy_fallbacks) if content_policy_fallbacks is not None and self._has_content_policy_fallback(model_group, kwargs): return True - fallbacks: Final = kwargs.get("fallbacks", self.fallbacks) + fallbacks: Final = kwargs.get("fallbacks", self._core_fallbacks) if not fallbacks: return False if check_non_standard_fallback_format(fallbacks=fallbacks): @@ -10240,7 +10423,7 @@ class Router: exception_status=e.status_code, original_exception=e, deployment=deployment["model_info"]["id"], - time_to_cooldown=self.cooldown_time, + time_to_cooldown=self._core_cooldown_time, ) raise e except Exception as e: @@ -10260,14 +10443,14 @@ class Router: model_id: Final = ( _MODEL_INFO_ADAPTER.validate_python(model_info).get("id") if isinstance(model_info, dict) else None ) - current: Final = self.get_deployment(model_id) if isinstance(model_id, str) else None + current: Final = self._core_get_deployment(model_id) if isinstance(model_id, str) else None return deployment if current is None else current.model_dump(exclude_none=True) async def async_admit_deployment_turn( self, model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object] ) -> None: deployment: Final = self._deployment_as_configured_now(opened_on) - budget_limiter: Final = self.get_router_deployment_budget_limiter() + budget_limiter: Final = self._core_get_router_deployment_budget_limiter() if budget_limiter is not None: try: await budget_limiter.async_filter_deployments( # pyright: ignore[reportUnknownMemberType] # legacy signature takes untyped deployment dicts @@ -10390,36 +10573,9 @@ class Router: return f"{type(value).__module__}.{type(value).__qualname__}" @staticmethod - def generate_model_id(model_group: str, litellm_params: dict) -> str: # mutable-ok: hashed read-only - """ - Helper function to consistently generate the same id for a deployment - - - create a string from all the litellm params - - hash - - use hash as id - """ - # Optimized: Use list and join instead of string concatenation in loop - # This avoids creating many temporary string objects (O(n) vs O(n²) complexity) - parts: Final = [model_group] - for k, v in litellm_params.items(): - if isinstance(k, str): - parts.append(k) - elif isinstance(k, dict): - parts.append(json.dumps(k, default=Router._json_default_stable_id)) - else: - parts.append(str(k)) - - if isinstance(v, str): - parts.append(v) - elif isinstance(v, dict): - parts.append(json.dumps(v, default=Router._json_default_stable_id)) - else: - parts.append(str(v)) - - concat_str: Final = "".join(parts) - hash_object: Final = hashlib.sha256(concat_str.encode()) - - return hash_object.hexdigest() + def _core_generate_model_id(model_group: str, litellm_params: dict) -> str: # mutable-ok: hashed read-only + """The stable id of a deployment without ``model_info.id``; see ``litellm.router_api.deployment_id``.""" + return _deployment_id(model_group, litellm_params) @staticmethod def _inherit_builtin_cache_pricing(model_info: dict, backend_model: str, custom_llm_provider: str | None) -> None: @@ -10770,7 +10926,7 @@ class Router: max_input_chars=(max_input_chars if max_input_chars is not None else DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS), ) self._register_pre_routing_strategy( - registry=self.auto_routers, + registry=self._core_auto_routers, deployment=deployment, strategy=autor_router, strategy_label="Auto-router", @@ -10784,9 +10940,9 @@ class Router: """ return classify_strategy_router_model(litellm_params.model) == "complexity" - def config_deployments(self) -> Iterator[Mapping[str, object]]: + def _core_config_deployments(self) -> Iterator[Mapping[str, object]]: """The model_list rows that came from config.yaml rather than the DB (``model_info.db_model`` unset).""" - for deployment in self.model_list: + for deployment in self._core_model_list: if not isinstance(deployment, Mapping): continue model_info = deployment.get("model_info") @@ -10804,7 +10960,7 @@ class Router: """ limit: Final = self.auto_router_capability_limit() if self.auto_router_capability_limit is not None else None others: Final = count_capability_routers( - (deployment for deployment in self.model_list if isinstance(deployment, Mapping)), + (deployment for deployment in self._core_model_list if isinstance(deployment, Mapping)), capability=capability, ) return capability_limit_violation(capability=capability, held=others + 1, limit=limit) @@ -10851,7 +11007,7 @@ class Router: complexity_router_config=complexity_router_config, ) self._register_pre_routing_strategy( - registry=self.complexity_routers, + registry=self._core_complexity_routers, deployment=deployment, strategy=complexity_router, strategy_label="Complexity-router", @@ -10928,7 +11084,7 @@ class Router: ) except (TypeError, ValueError): return None - return Router.generate_model_id(deployment.model_name, {"routing_definition": definition}) + return Router._core_generate_model_id(deployment.model_name, {"routing_definition": definition}) @staticmethod def _unregister_pre_routing_strategy( @@ -10970,9 +11126,9 @@ class Router: return model_name: Final = deployment.model_name tags: Final = self._deployment_tags(deployment) - for registry in (self.auto_routers, self.complexity_routers, self.quality_routers): + for registry in (self._core_auto_routers, self._core_complexity_routers, self._core_quality_routers): self._unregister_pre_routing_strategy(registry, model_name, tags) - if self._unregister_pre_routing_strategy(self.adaptive_routers, model_name, tags): + if self._unregister_pre_routing_strategy(self._core_adaptive_routers, model_name, tags): self._sync_adaptive_router_hooks() def _finalize_adaptive_router_if_configured(self) -> None: @@ -10980,7 +11136,7 @@ class Router: build an AdaptiveRouter for each. Safe no-op when none are configured. Idempotent: skips any deployment whose (model_name, tags) pair is already initialized, so hot-reloads don't rebuild routers that would lose state.""" - for entry in self.model_list or []: + for entry in self._core_model_list or []: lp = entry.get("litellm_params") if isinstance(entry, dict) else entry.litellm_params lp_model = (lp.get("model") if isinstance(lp, dict) else lp.model) if lp else None if not (lp_model and lp_model.startswith("auto_router/adaptive_router")): @@ -10993,21 +11149,23 @@ class Router: litellm_params=(lp if not isinstance(lp, dict) else LiteLLM_Params.model_validate(lp)), model_info=(entry.get("model_info") if isinstance(entry, dict) else entry.model_info), ) - if self._has_registered_strategy(self.adaptive_routers, model_name, self._deployment_tags(deployment)): + if self._has_registered_strategy( + self._core_adaptive_routers, model_name, self._deployment_tags(deployment) + ): continue self.init_adaptive_router_deployment(deployment=deployment) - for model_name, tagged_complexity_routers in self.complexity_routers.items(): + for model_name, tagged_complexity_routers in self._core_complexity_routers.items(): for tagged in tagged_complexity_routers: complexity_router = tagged.strategy if not complexity_router.config.adaptive: continue - if self._has_registered_strategy(self.adaptive_routers, model_name, tagged.tags): + if self._has_registered_strategy(self._core_adaptive_routers, model_name, tagged.tags): continue adaptive_router: AdaptiveRouter | None = complexity_router._ensure_adaptive_router() if adaptive_router is not None: - self.adaptive_routers[model_name] = [ - *self.adaptive_routers.get(model_name, []), + self._core_adaptive_routers[model_name] = [ + *self._core_adaptive_routers.get(model_name, []), TaggedPreRoutingStrategy( tags=tagged.tags, strategy=adaptive_router, @@ -11029,12 +11187,12 @@ class Router: for callback in litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook): litellm.logging_callback_manager.remove_callback_from_all_lists(callback) - for tagged_adaptive_routers in self.adaptive_routers.values(): + for tagged_adaptive_routers in self._core_adaptive_routers.values(): for tagged in tagged_adaptive_routers: litellm.logging_callback_manager.add_litellm_callback( AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy) ) - if self.adaptive_routers: + if self._core_adaptive_routers: litellm.logging_callback_manager.add_litellm_callback(AdaptiveRouterResponseHeaderHook()) def init_adaptive_router_deployment(self, deployment: Deployment) -> None: @@ -11044,7 +11202,7 @@ class Router: keyed by `deployment.model_name`. `model_to_prefs` and `model_to_cost` are derived from the OTHER models - already registered in `self.model_list` whose `model_name` appears in + already registered in `self._core_model_list` whose `model_name` appears in `available_models`. Models not yet registered fall back to defaults. """ # Local import: AdaptiveRouter -> hooks -> classifier all import litellm @@ -11071,7 +11229,7 @@ class Router: # O(k) via the name→indices map: only touch deployments whose name # is listed in `available_models`, instead of scanning model_list. for name in config.available_models: - deployments = self.deployments_named(name) + deployments = self._core_deployments_named(name) if not deployments: continue d = deployments[0] @@ -11094,7 +11252,7 @@ class Router: model_to_cost=model_to_cost, ) self._register_pre_routing_strategy( - registry=self.adaptive_routers, + registry=self._core_adaptive_routers, deployment=deployment, strategy=adaptive_router, strategy_label="Adaptive-router", @@ -11122,7 +11280,7 @@ class Router: Resolves the default model from either `quality_router_default_model` or `quality_router_config["default_model"]`, then instantiates the - QualityRouter and stores it in `self.quality_routers`. + QualityRouter and stores it in `self._core_quality_routers`. """ # Import here to mirror the AutoRouter / ComplexityRouter init pattern # and avoid circular imports. @@ -11149,7 +11307,7 @@ class Router: quality_router_config=quality_router_config, ) self._register_pre_routing_strategy( - registry=self.quality_routers, + registry=self._core_quality_routers, deployment=deployment, strategy=quality_router, strategy_label="Quality-router", @@ -11174,17 +11332,17 @@ class Router: original_model_list: Final = copy.deepcopy(model_list) self._discovered_model_info_cache.flush_cache() with self._deployment_index_lock: - self.model_list = [] + self._core_model_list = [] self.model_id_to_deployment_index_map = {} # Reset the index - self.model_name_to_deployment_indices = {} # Reset the model_name index - self.team_model_to_deployment_indices = {} # Reset the team_model index - self.team_pattern_routers = {} - self.team_public_model_names = frozenset() + self._core_model_name_to_deployment_indices = {} # Reset the model_name index + self._core_team_model_to_deployment_indices = {} # Reset the team_model index + self._core_team_pattern_routers = {} + self._core_team_public_model_names = frozenset() # Reset per-strategy router registries so hot-reload doesn't leave # stale routers pointing at the old model_list. - self.quality_routers = {} - self.complexity_routers = {} - self.auto_routers = {} + self._core_quality_routers = {} + self._core_complexity_routers = {} + self._core_auto_routers = {} self._provider_unresolved_deployments = () self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() @@ -11227,7 +11385,7 @@ class Router: # check if model info has id if "id" not in _model_info: - _id = self.generate_model_id(_model_name, _litellm_params) + _id = self._core_generate_model_id(_model_name, _litellm_params) _model_info["id"] = _id if _litellm_params.get("organization", None) is not None and isinstance( @@ -11253,8 +11411,8 @@ class Router: duplicate_ids=duplicate_ids, ) - verbose_router_logger.debug("\nInitialized Model List %s", self.get_model_names()) - self.model_names = {m["model_name"] for m in model_list} + verbose_router_logger.debug("\nInitialized Model List %s", self._core_get_model_names()) + self._core_model_names = {m["model_name"] for m in model_list} # Note: model_name_to_deployment_indices is already built incrementally # by _create_deployment -> _add_model_to_list_and_index_map @@ -11316,7 +11474,7 @@ class Router: raise Exception(f"Unsupported provider - {custom_llm_provider}") #### DEPLOYMENT NAMES INIT ######## - self.deployment_names.append(deployment.litellm_params.model) + self._core_deployment_names.append(deployment.litellm_params.model) ############ Users can either pass tpm/rpm as a litellm_param or a router param ########### # for get_available_deployment, we use the litellm_param["rpm"] # in this snippet we also set rpm to be a litellm_param @@ -11331,24 +11489,24 @@ class Router: # if deployment.model_name == "*": # if deployment.litellm_params.model == "*": # # user wants to pass through all requests to litellm.acompletion for unknown deployments - # self.router_general_settings.pass_through_all_models = True + # self._core_router_general_settings.pass_through_all_models = True # else: - # self.default_deployment = deployment.to_json(exclude_none=True) + # self._core_default_deployment = deployment.to_json(exclude_none=True) # Check if user is using provider specific wildcard routing # example model_name = "databricks/*" or model_name = "anthropic/*" if "*" in deployment.model_name: # store this as a regex pattern - all deployments matching this pattern will be sent to this deployment # Store deployment.model_name as a regex pattern - self.pattern_router.add_pattern(deployment.model_name, deployment.to_json(exclude_none=True)) + self._core_pattern_router.add_pattern(deployment.model_name, deployment.to_json(exclude_none=True)) if deployment.model_info.id: - self.provider_default_deployment_ids.append(deployment.model_info.id) + self._core_provider_default_deployment_ids.append(deployment.model_info.id) _team_id: Final = deployment.model_info.get("team_id") _team_public_model_name: Final[str | None] = deployment.model_info.get("team_public_model_name") if _team_id is not None and _team_public_model_name is not None and "*" in _team_public_model_name: - if _team_id not in self.team_pattern_routers: - self.team_pattern_routers[_team_id] = PatternMatchRouter() - self.team_pattern_routers[_team_id].add_pattern( + if _team_id not in self._core_team_pattern_routers: + self._core_team_pattern_routers[_team_id] = PatternMatchRouter() + self._core_team_pattern_routers[_team_id].add_pattern( _team_public_model_name, deployment.to_json(exclude_none=True) ) @@ -11455,7 +11613,7 @@ class Router: # check if deployment already exists _deployment_model_id: Final = deployment.model_info.id - if _deployment_model_id and self.has_model_id(_deployment_model_id): + if _deployment_model_id and self._core_has_model_id(_deployment_model_id): return None declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True) @@ -11475,7 +11633,7 @@ class Router: # add to model names self._add_model_to_list_and_index_map(model=_deployment, model_id=served.model_info.id) - self.model_names.add(served.model_name) + self._core_model_names.add(served.model_name) self._sync_deployment_budget_config(deployment=served) return served @@ -11497,7 +11655,7 @@ class Router: del self.model_id_to_deployment_index_map[model_id] # Update model_name_to_deployment_indices - for model_name, indices in list(self.model_name_to_deployment_indices.items()): + for model_name, indices in list(self._core_model_name_to_deployment_indices.items()): # Build new list without mutating the original updated_indices = [] for idx in indices: @@ -11513,13 +11671,13 @@ class Router: # Update or remove the entry if len(updated_indices) > 0: - self.model_name_to_deployment_indices[model_name] = updated_indices + self._core_model_name_to_deployment_indices[model_name] = updated_indices else: - del self.model_name_to_deployment_indices[model_name] - self.model_names.discard(model_name) + del self._core_model_name_to_deployment_indices[model_name] + self._core_model_names.discard(model_name) # Update team_model_to_deployment_indices - for key, indices in list(self.team_model_to_deployment_indices.items()): + for key, indices in list(self._core_team_model_to_deployment_indices.items()): # Build new list without mutating the original updated_indices = [] for idx in indices: @@ -11535,19 +11693,19 @@ class Router: # Update or remove the entry if len(updated_indices) > 0: - self.team_model_to_deployment_indices[key] = updated_indices + self._core_team_model_to_deployment_indices[key] = updated_indices else: - del self.team_model_to_deployment_indices[key] - self.team_public_model_names = frozenset( - public_model_name for _, public_model_name in self.team_model_to_deployment_indices + del self._core_team_model_to_deployment_indices[key] + self._core_team_public_model_names = frozenset( + public_model_name for _, public_model_name in self._core_team_model_to_deployment_indices ) - self.pattern_router.remove_deployment(model_id) - for team_id in list(self.team_pattern_routers.keys()): - team_pattern_router = self.team_pattern_routers[team_id] + self._core_pattern_router.remove_deployment(model_id) + for team_id in list(self._core_team_pattern_routers.keys()): + team_pattern_router = self._core_team_pattern_routers[team_id] team_pattern_router.remove_deployment(model_id) if not team_pattern_router.patterns: - del self.team_pattern_routers[team_id] + del self._core_team_pattern_routers[team_id] def _update_team_model_index(self, model: dict, idx: int) -> None: """ @@ -11561,11 +11719,13 @@ class Router: team_public_model_name: Final = (model.get("model_info") or {}).get("team_public_model_name") if team_id and team_public_model_name: key: Final = (team_id, team_public_model_name) - self.team_public_model_names = self.team_public_model_names | frozenset({team_public_model_name}) - if key not in self.team_model_to_deployment_indices: - self.team_model_to_deployment_indices[key] = [] - if idx not in self.team_model_to_deployment_indices[key]: - self.team_model_to_deployment_indices[key].append(idx) + self._core_team_public_model_names = self._core_team_public_model_names | frozenset( + {team_public_model_name} + ) + if key not in self._core_team_model_to_deployment_indices: + self._core_team_model_to_deployment_indices[key] = [] + if idx not in self._core_team_model_to_deployment_indices[key]: + self._core_team_model_to_deployment_indices[key].append(idx) def _add_model_to_list_and_index_map(self, model: dict, model_id: str | None = None) -> None: """ @@ -11576,8 +11736,8 @@ class Router: - model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"] """ with self._deployment_index_lock: - idx: Final = len(self.model_list) - self.model_list.append(model) + idx: Final = len(self._core_model_list) + self._core_model_list.append(model) self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() @@ -11590,14 +11750,14 @@ class Router: # Update model_name index for O(1) lookup model_name: Final = model.get("model_name") if model_name: - if model_name not in self.model_name_to_deployment_indices: - self.model_name_to_deployment_indices[model_name] = [] - self.model_name_to_deployment_indices[model_name].append(idx) + if model_name not in self._core_model_name_to_deployment_indices: + self._core_model_name_to_deployment_indices[model_name] = [] + self._core_model_name_to_deployment_indices[model_name].append(idx) # Update team_model index for O(1) team-scoped lookup self._update_team_model_index(model, idx) - def upsert_deployment(self, deployment: Deployment) -> Deployment | None: + def _core_upsert_deployment(self, deployment: Deployment) -> Deployment | None: """ Add or update deployment Parameters: @@ -11607,7 +11767,7 @@ class Router: - The added/updated deployment """ _deployment_model_id: Final = deployment.model_info.id or "" - _deployment_on_router: Final[Deployment | None] = self.get_deployment(model_id=_deployment_model_id) + _deployment_on_router: Final[Deployment | None] = self._core_get_deployment(model_id=_deployment_model_id) served: Final = without_model_capability_flags(deployment) try: if _deployment_on_router is not None: @@ -11625,7 +11785,7 @@ class Router: with self._deployment_index_lock: removal_idx: Final = self.model_id_to_deployment_index_map.get(_deployment_model_id) if removal_idx is not None: - self.model_list.pop(removal_idx) + self._core_model_list.pop(removal_idx) self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() self._update_deployment_indices_after_removal( @@ -11672,7 +11832,7 @@ class Router: capability ceiling a newcomer gets: with the ceiling tightened since the deployment first registered, judging the rollback would drop a serving router over an unrelated failed edit. """ - if previous_deployment is None or self.has_model_id(model_id): + if previous_deployment is None or self._core_has_model_id(model_id): return limit_resolver: Final = self.auto_router_capability_limit self.auto_router_capability_limit = None @@ -11765,7 +11925,7 @@ class Router: """Register a deployment's custom/inherited pricing in ``litellm.model_cost``. Takes only a ``Deployment``, so it registers pricing for a deployment that - is never added to ``self.model_list`` (a per-request client-side-credential + is never added to ``self._core_model_list`` (a per-request client-side-credential deployment) just as readily as one that is. """ Router._register_deployment_in_model_cost( @@ -11836,7 +11996,7 @@ class Router: self._provider_unresolved_deployments = () for create_deployment in provider_unresolved: create_deployment() - for entry in tuple(self.model_list): + for entry in tuple(self._core_model_list): try: deployment = entry if isinstance(entry, Deployment) else Deployment(**entry) except Exception: # noqa: BLE001 # a malformed entry must not abort the rest of the rebuild @@ -11855,7 +12015,7 @@ class Router: _replay_model_cost_registrations = replay_model_cost_registrations - def delete_deployment(self, id: str) -> Deployment | None: + def _core_delete_deployment(self, id: str) -> Deployment | None: """ Parameters: - id: str - the id of the deployment to be deleted @@ -11870,11 +12030,11 @@ class Router: if deployment_idx is None: return None # Pop the item from the list first - item: Final = self.model_list.pop(deployment_idx) + item: Final = self._core_model_list.pop(deployment_idx) self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() self._update_deployment_indices_after_removal(model_id=id, removal_idx=deployment_idx) - _budget_limiter: Final = self.get_router_deployment_budget_limiter() + _budget_limiter: Final = self._core_get_router_deployment_budget_limiter() if _budget_limiter is not None: _budget_limiter.unregister_deployment_budget(model_id=id) try: @@ -11891,7 +12051,7 @@ class Router: return None return item - def get_router_deployment_budget_limiter( + def _core_get_router_deployment_budget_limiter( self, ) -> RouterBudgetLimiting | None: """ @@ -11910,8 +12070,6 @@ class Router: return _cb return None - _get_router_deployment_budget_limiter = get_router_deployment_budget_limiter - def _deployment_has_budget_limits(self, deployment: Deployment) -> bool: return ( deployment.litellm_params.get("max_budget") is not None @@ -11924,7 +12082,7 @@ class Router: if model_id is None: return - _budget_limiter = self.get_router_deployment_budget_limiter() + _budget_limiter = self._core_get_router_deployment_budget_limiter() if not self._deployment_has_budget_limits(deployment=deployment): if _budget_limiter is not None: @@ -11933,32 +12091,32 @@ class Router: if _budget_limiter is None: self.add_optional_pre_call_checks(optional_pre_call_checks=["router_budget_limiting"]) - _budget_limiter = self.get_router_deployment_budget_limiter() + _budget_limiter = self._core_get_router_deployment_budget_limiter() if _budget_limiter is not None: _budget_limiter.register_deployment_budget(deployment=deployment.to_json(exclude_none=True)) - def deployments_named(self, model_name: str) -> tuple[DeploymentTypedDict, ...] | None: + def _core_deployments_named(self, model_name: str) -> tuple[DeploymentTypedDict, ...] | None: """The deployments indexed under `model_name`, read under the index lock, or None when the name is not indexed.""" with self._deployment_index_lock: - indices: Final = self.model_name_to_deployment_indices.get(model_name) - return None if indices is None else tuple(self.model_list[idx] for idx in indices) + indices: Final = self._core_model_name_to_deployment_indices.get(model_name) + return None if indices is None else tuple(self._core_model_list[idx] for idx in indices) def _served_openai_models(self, model_name: str) -> tuple[str | None, ...]: - return served_openai_models(self.deployments_named(model_name) or ()) + return served_openai_models(self._core_deployments_named(model_name) or ()) def team_public_model_names_of(self, team_id: str) -> frozenset[str]: """The public names `team_id` serves its own deployments under.""" with self._deployment_index_lock: - return frozenset(name for owner, name in self.team_model_to_deployment_indices if owner == team_id) + return frozenset(name for owner, name in self._core_team_model_to_deployment_indices if owner == team_id) def _team_deployments_named(self, team_id: str, public_model_name: str) -> tuple[DeploymentTypedDict, ...] | None: """The deployments `team_id` serves under `public_model_name`, or None when that pair is not indexed.""" with self._deployment_index_lock: - indices: Final = self.team_model_to_deployment_indices.get((team_id, public_model_name)) - return None if indices is None else tuple(self.model_list[idx] for idx in indices) + indices: Final = self._core_team_model_to_deployment_indices.get((team_id, public_model_name)) + return None if indices is None else tuple(self._core_model_list[idx] for idx in indices) - def get_deployment(self, model_id: str) -> Deployment | None: + def _core_get_deployment(self, model_id: str) -> Deployment | None: """ Returns -> Deployment or None @@ -11967,7 +12125,7 @@ class Router: # Use O(1) lookup via model_id_to_deployment_index_map only with self._deployment_index_lock: idx: Final = self.model_id_to_deployment_index_map.get(model_id) - model: Final = None if idx is None else self.model_list[idx] + model: Final = None if idx is None else self._core_model_list[idx] if model is not None: if isinstance(model, dict): return Deployment(**model) @@ -11978,7 +12136,7 @@ class Router: return None - def get_deployment_credentials(self, model_id: str) -> dict | None: + def _core_get_deployment_credentials(self, model_id: str) -> dict | None: """ Returns -> dict of credentials for a given model id. @@ -11986,14 +12144,14 @@ class Router: so file/batch/passthrough callers that resolve credentials directly cannot keep using a paused deployment. """ - deployment: Final = self.get_deployment(model_id=model_id) - if deployment is None or self.is_deployment_blocked(deployment): + deployment: Final = self._core_get_deployment(model_id=model_id) + if deployment is None or self._core_is_deployment_blocked(deployment): return None return CredentialLiteLLMParams.model_validate( deployment.litellm_params.model_dump(exclude_none=True) ).model_dump(exclude_none=True) - def get_deployment_by_model_group_name(self, model_group_name: str) -> Deployment | None: + def _core_get_deployment_by_model_group_name(self, model_group_name: str) -> Deployment | None: """ Returns -> Deployment or None @@ -12003,9 +12161,9 @@ class Router: """ # O(1) lookup in model_name index with self._deployment_index_lock: - indices: Final = self.model_name_to_deployment_indices.get(model_group_name) + indices: Final = self._core_model_name_to_deployment_indices.get(model_group_name) # Return first deployment for this model_name - model: Final = self.model_list[indices[0]] if indices else None + model: Final = self._core_model_list[indices[0]] if indices else None if model is not None: if isinstance(model, dict): return Deployment(**model) @@ -12016,7 +12174,7 @@ class Router: return None @staticmethod - def deployment_usable_by_team(model: Mapping[str, object] | Deployment, team_id: str | None) -> bool: + def _core_deployment_usable_by_team(model: Mapping[str, object] | Deployment, team_id: str | None) -> bool: """ A team-scoped deployment (``model_info.team_id`` set) is only usable by callers from that same team; deployments without a team owner are shared. @@ -12025,8 +12183,6 @@ class Router: owner_team_id: Final = cast(Mapping[str, object], model_info).get("team_id") if model_info is not None else None return team_may_use_deployment(owner_team_id, team_id) - _deployment_usable_by_team = deployment_usable_by_team - def _get_model_group_deployment_usable_by_team( self, model_group_name: str, team_id: str | None ) -> Deployment | None: @@ -12036,18 +12192,20 @@ class Router: credentials. """ with self._deployment_index_lock: - indices: Final = self.model_name_to_deployment_indices.get(model_group_name) or () + indices: Final = self._core_model_name_to_deployment_indices.get(model_group_name) or () usable: Final = ( - self.model_list[idx] for idx in indices if self.deployment_usable_by_team(self.model_list[idx], team_id) + self._core_model_list[idx] + for idx in indices + if self._core_deployment_usable_by_team(self._core_model_list[idx], team_id) ) first_usable: Final = next(usable, None) if first_usable is None: return None return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable - async def arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None: + async def _core_arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None: """Refresh token limits advertised by configured OpenAI-compatible deployments.""" - deployments: Final = iter(tuple(self.model_list)) + deployments: Final = iter(tuple(self._core_model_list)) async def refresh_worker() -> None: for raw_deployment in deployments: @@ -12068,7 +12226,7 @@ class Router: { **deployment.litellm_params.model_dump(exclude_none=True), **( - self.get_deployment_credentials_with_provider(deployment.model_info.id or "") + self._core_get_deployment_credentials_with_provider(deployment.model_info.id or "") or MappingProxyType({}) ), } @@ -12101,30 +12259,30 @@ class Router: } ), client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), - cache=self.cache.in_memory_cache, + cache=self._core_cache.in_memory_cache, ) model_id: Final = deployment.model_info.id - if not limits or model_id is None or self.get_model_info(model_id) is not raw_deployment: + if not limits or model_id is None or self._core_get_model_info(model_id) is not raw_deployment: return - self._discovered_model_info_cache.max_size_in_memory = max(len(self.model_list), 1) + self._discovered_model_info_cache.max_size_in_memory = max(len(self._core_model_list), 1) self._discovered_model_info_cache.delete_cache(model_id) self._discovered_model_info_cache.set_cache( model_id, DiscoveredDeploymentModelInfo(deployment=raw_deployment, limits=limits) ) self._invalidate_model_group_info_cache() - def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: + def _core_get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: cached: Final[object] = self._discovered_model_info_cache.get_cache(model_id) if ( model_id is not None and isinstance(cached, DiscoveredDeploymentModelInfo) - and cached.deployment is self.get_model_info(model_id) + and cached.deployment is self._core_get_model_info(model_id) ): configured: Final = TypeAdapter(Mapping[str, object]).validate_python(cached.deployment["model_info"]) return MappingProxyType({key: value for key, value in cached.limits.items() if configured.get(key) is None}) return MappingProxyType({}) - def get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: + def _core_get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: """ Return what the concrete deployments behind model_name contribute to its /v1/models entry: the cost-map keys for their underlying models, plus the widest @@ -12143,14 +12301,16 @@ class Router: this never triggers pattern matching or deep copies, so it is safe to call per listed model on the /v1/models hot path. """ - deployments: Final = self.deployments_named(model_name) + deployments: Final = self._core_deployments_named(model_name) if not deployments: return None model_infos: Final = tuple( MappingProxyType( { - **self.get_discovered_model_info((deployment.get("model_info") or MappingProxyType({})).get("id")), + **self._core_get_discovered_model_info( + (deployment.get("model_info") or MappingProxyType({})).get("id") + ), **MappingProxyType( { k: v @@ -12181,7 +12341,7 @@ class Router: ) return max(limits) if limits else None - def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": + def _core_get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete deployment of model_name, via O(1) index lookup. @@ -12193,13 +12353,13 @@ class Router: get_model_listing_info does: its caller truncates an embedding input to this value, so the widest window in a mixed group would be the wrong answer there. """ - deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name) if deployment is None: return (None, None) model_info: Final = MappingProxyType( { - **self.get_discovered_model_info(deployment.model_info.id), + **self._core_get_discovered_model_info(deployment.model_info.id), **deployment.model_info.model_dump(exclude_none=True), } ) @@ -12208,9 +12368,9 @@ class Router: coerce_token_limit(model_info.get("max_output_tokens")), ) - def get_configured_mode(self, model_name: str) -> "str | None": + def _core_get_configured_mode(self, model_name: str) -> "str | None": """Return the mode explicitly configured for a concrete deployment.""" - deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name) if deployment is None: return None @@ -12219,7 +12379,7 @@ class Router: return mode return None - def get_configured_display_name(self, model_name: str) -> "str | None": + def _core_get_configured_display_name(self, model_name: str) -> "str | None": """ Return the display_name explicitly configured in a concrete deployment's model_info for model_name, via O(1) index lookup. @@ -12230,7 +12390,7 @@ class Router: matching or deep copies, so it is safe to call per listed model on the /v1/models hot path. """ - deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name) + deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name) if deployment is None: return None @@ -12239,12 +12399,12 @@ class Router: return display_name return None - def routable_model_group(self, model_name: str) -> str: + def _core_routable_model_group(self, model_name: str) -> str: """ The model group a request to model_name routes to: its target when model_name is a `model_group_alias`, else model_name itself. """ - target: Final = self.get_model_from_alias(model_name) + target: Final = self._core_get_model_from_alias(model_name) return target if target is not None else model_name def _routable_deployments(self, model_name: str, team_id: str | None) -> tuple[DeploymentTypedDict, ...]: @@ -12260,15 +12420,19 @@ class Router: Returns an empty tuple for wildcard-expanded or unknown names. """ - named: Final = self.get_all_deployments(model_name=self.routable_model_group(model_name), team_id=team_id) - usable: Final = tuple(deployment for deployment in named if self.deployment_usable_by_team(deployment, team_id)) + named: Final = self._core_get_all_deployments( + model_name=self._core_routable_model_group(model_name), team_id=team_id + ) + usable: Final = tuple( + deployment for deployment in named if self._core_deployment_usable_by_team(deployment, team_id) + ) return tuple( deployment for deployment in (usable or named) if _configured_model_info(deployment).get("blocked") is not True ) - def get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: + def _core_get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: """ Return the service_tiers value each deployment a request from team_id to model_name can route to (see _routable_deployments) configures in @@ -12283,7 +12447,7 @@ class Router: for deployment in self._routable_deployments(model_name, team_id) ) - def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: + def _core_get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: """ Whether each deployment a request from team_id to model_name can route to (see _routable_deployments) answers Codex's client `tool_search`: @@ -12292,7 +12456,7 @@ class Router: """ return tuple(_search_tool_support(deployment) for deployment in self._routable_deployments(model_name, team_id)) - def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: + def _core_get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: """ Return the `litellm_params.model` of the first deployment a request from team_id to model_name can route to (see _routable_deployments). @@ -12303,7 +12467,7 @@ class Router: deployment: Final = next(iter(self._routable_deployments(model_name, team_id)), None) return deployment["litellm_params"].get("model") if deployment is not None else None - def get_credential_deployment(self, model_id: str, team_id: str | None = None) -> Deployment | None: + def _core_get_credential_deployment(self, model_id: str, team_id: str | None = None) -> Deployment | None: """ The deployment a passthrough endpoint (files, batches, etc.) resolves for a model id or model name: by deployment id first, then by model_name, then by @@ -12316,12 +12480,12 @@ class Router: by resolving the deployment directly. """ deployment: Final = ( - self.get_deployment(model_id=model_id) + self._core_get_deployment(model_id=model_id) or self._get_model_group_deployment_usable_by_team(model_group_name=model_id, team_id=team_id) or self._get_team_public_name_deployment(model_id=model_id, team_id=team_id) or self._get_wildcard_deployment_usable_by_team(model_id=model_id, team_id=team_id) ) - if deployment is None or self.is_deployment_blocked(deployment): + if deployment is None or self._core_is_deployment_blocked(deployment): return None return deployment @@ -12329,19 +12493,19 @@ class Router: if team_id is None: return None with self._deployment_index_lock: - team_indices: Final = self.team_model_to_deployment_indices.get((team_id, model_id)) - team_model: Final = self.model_list[team_indices[0]] if team_indices else None + team_indices: Final = self._core_team_model_to_deployment_indices.get((team_id, model_id)) + team_model: Final = self._core_model_list[team_indices[0]] if team_indices else None if team_model is None: return None return Deployment(**team_model) if isinstance(team_model, dict) else team_model def _get_wildcard_deployment_usable_by_team(self, model_id: str, team_id: str | None) -> Deployment | None: - team_pattern_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None + team_pattern_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None team_wildcard_models: Final = team_pattern_router.route(model_id) if team_pattern_router else None global_wildcard_models: Final = tuple( wildcard_model - for wildcard_model in (self.pattern_router.route(model_id) or ()) - if self.deployment_usable_by_team(wildcard_model, team_id) + for wildcard_model in (self._core_pattern_router.route(model_id) or ()) + if self._core_deployment_usable_by_team(wildcard_model, team_id) ) potential_wildcard_models: Final = team_wildcard_models or global_wildcard_models if not potential_wildcard_models: @@ -12353,7 +12517,7 @@ class Router: return wildcard_deployment return None - def get_deployment_credentials_with_provider( + def _core_get_deployment_credentials_with_provider( self, model_id: str, team_id: str | None = None ) -> dict[str, Any] | None: """ @@ -12382,7 +12546,7 @@ class Router: credentials = router.get_deployment_credentials_with_provider("gpt-4o-litellm") # Returns: {"api_key": "sk-...", "custom_llm_provider": "openai", "model": "gpt-4o", ...} """ - deployment: Final = self.get_credential_deployment(model_id=model_id, team_id=team_id) + deployment: Final = self._core_get_credential_deployment(model_id=model_id, team_id=team_id) if deployment is None: return None @@ -12418,7 +12582,7 @@ class Router: return credentials @overload - def get_router_model_info( + def _core_get_router_model_info( self, deployment: Union[dict, "Deployment"], received_model_name: str, @@ -12427,10 +12591,10 @@ class Router: pass @overload - def get_router_model_info(self, deployment: None, received_model_name: str, id: str) -> ModelMapInfo: + def _core_get_router_model_info(self, deployment: None, received_model_name: str, id: str) -> ModelMapInfo: pass - def get_router_model_info( + def _core_get_router_model_info( self, deployment: Union[dict, "Deployment"] | None, received_model_name: str, @@ -12451,7 +12615,7 @@ class Router: - ValueError -> If model is not mapped yet """ if id is not None: - _deployment: Final = self.get_deployment(model_id=id) + _deployment: Final = self._core_get_deployment(model_id=id) if _deployment is not None: deployment = _deployment @@ -12509,7 +12673,7 @@ class Router: model = _model if "*" in model: # only call pattern_router for wildcard models - potential_models: Final = self.pattern_router.route(received_model_name) + potential_models: Final = self._core_pattern_router.route(received_model_name) if potential_models is not None: for potential_model in potential_models: try: @@ -12553,13 +12717,13 @@ class Router: _declared_base_model(user_model_info) or _declared_base_model(litellm_params.model_extra or {}), custom_llm_provider, ), - **self.get_discovered_model_info((deployment.get("model_info") or {}).get("id")), + **self._core_get_discovered_model_info((deployment.get("model_info") or {}).get("id")), **MappingProxyType({key: value for key, value in user_model_info.items() if value is not None}), } return merged_model_info - def get_model_info(self, id: str) -> dict | None: + def _core_get_model_info(self, id: str) -> dict | None: """ For a given model id, return the model info @@ -12572,21 +12736,21 @@ class Router: # O(1) lookup via model_id_to_deployment_index_map with self._deployment_index_lock: idx: Final = self.model_id_to_deployment_index_map.get(id) - return None if idx is None else self.model_list[idx] + return None if idx is None else self._core_model_list[idx] - def get_model_group(self, id: str) -> list | None: + def _core_get_model_group(self, id: str) -> list | None: """ Return list of all models in the same model group as that model id """ - model_info: Final = self.get_model_info(id=id) + model_info: Final = self._core_get_model_info(id=id) if model_info is None: return None model_name: Final = model_info["model_name"] - return self.get_model_list(model_name=model_name) + return self._core_get_model_list(model_name=model_name) - def get_deployment_model_info(self, model_id: str, model_name: str) -> ModelInfo | None: + def _core_get_deployment_model_info(self, model_id: str, model_name: str) -> ModelInfo | None: """ For a given model id, return the model info @@ -12605,7 +12769,7 @@ class Router: custom_model_info = ( { **copy.deepcopy(litellm.model_cost.get(model_id) or MappingProxyType({})), - **self.get_discovered_model_info(model_id), + **self._core_get_discovered_model_info(model_id), } if model_id in litellm.model_cost else None @@ -12695,7 +12859,7 @@ class Router: configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None reasoning_efforts_initialized = False reasoning_efforts_unknown = False - model_list: Final = self.get_model_list_of_routed_group(model_group) + model_list: Final = self._core_get_model_list_of_routed_group(model_group) model_group_description: Final[str | None] = next( (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)), None, @@ -12706,7 +12870,7 @@ class Router: "model_name" in model and model["model_name"] == model_group or "model_name" in model - and self.pattern_router.route(model_group) is not None + and self._core_pattern_router.route(model_group) is not None ): # exact match is_match = True @@ -12755,7 +12919,9 @@ class Router: try: model_id = model_info_dict.get("id", None) if model_id is not None: - model_info = self.get_deployment_model_info(model_id=model_id, model_name=litellm_params.model) + model_info = self._core_get_deployment_model_info( + model_id=model_id, model_name=litellm_params.model + ) else: model_info = None except Exception: @@ -12954,7 +13120,7 @@ class Router: return model_group_info - def get_model_group_info( + def _core_get_model_group_info( self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None ) -> ModelGroupInfo | None: """ @@ -12965,8 +13131,8 @@ class Router: - None if error constructing model group info or hidden model group (unless include_hidden) """ ## Check if model group alias - if model_group in self.model_group_alias: - item: Final = self.model_group_alias[model_group] + if model_group in self._core_model_group_alias: + item: Final = self._core_model_group_alias[model_group] if isinstance(item, str): _router_model_group = item elif isinstance(item, dict): @@ -12988,7 +13154,7 @@ class Router: model_group=model_group, user_facing_model_group_name=model_group, team_id=team_id ) - async def get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: + async def _core_get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: """ Returns current tpm/rpm usage for model group @@ -13003,7 +13169,7 @@ class Router: tpm_keys: Final[list[str]] = [] rpm_keys: Final[list[str]] = [] - model_list: Final = self.get_model_list(model_name=model_group) + model_list: Final = self._core_get_model_list(model_name=model_group) if model_list is None: # no matching deployments return None, None @@ -13030,7 +13196,7 @@ class Router: ) combined_tpm_rpm_keys: Final = tpm_keys + rpm_keys - combined_tpm_rpm_values: Final = await self.cache.async_batch_get_cache(keys=combined_tpm_rpm_keys) + combined_tpm_rpm_values: Final = await self._core_cache.async_batch_get_cache(keys=combined_tpm_rpm_keys) if combined_tpm_rpm_values is None: return None, None @@ -13064,7 +13230,7 @@ class Router: itpm_keys: Final[list[str]] = [] otpm_keys: Final[list[str]] = [] - model_list: Final = self.get_model_list(model_name=model_group) + model_list: Final = self._core_get_model_list(model_name=model_group) if model_list is None: return None, None @@ -13088,7 +13254,7 @@ class Router: ) ) - combined_values: Final = await self.cache.async_batch_get_cache(keys=itpm_keys + otpm_keys) + combined_values: Final = await self._core_cache.async_batch_get_cache(keys=itpm_keys + otpm_keys) if combined_values is None: return None, None @@ -13114,12 +13280,12 @@ class Router: This is a speed optimization, since set_response_headers makes a call to get_model_group_info on every request """ - return self.get_model_group_info(model_group) + return self._core_get_model_group_info(model_group) - def cached_model_group_info(self, model_group: str) -> ModelGroupInfo | None: + def _core_cached_model_group_info(self, model_group: str) -> ModelGroupInfo | None: return self._cached_get_model_group_info(model_group) - async def get_remaining_model_group_usage(self, model_group: str) -> dict[str, int]: + async def _core_get_remaining_model_group_usage(self, model_group: str) -> dict[str, int]: model_group_info: Final = self._cached_get_model_group_info(model_group) returned_dict: Final[dict[str, int]] = {} @@ -13144,7 +13310,7 @@ class Router: if tpm_limit is None and rpm_limit is None: return returned_dict - current_tpm, current_rpm = await self.get_model_group_usage(model_group) + current_tpm, current_rpm = await self._core_get_model_group_usage(model_group) if tpm_limit is not None: returned_dict["x-ratelimit-remaining-tokens"] = tpm_limit - (current_tpm or 0) @@ -13181,7 +13347,7 @@ class Router: additional_headers.update(complexity_router_decision_headers(request_kwargs)) if model_group is not None: - remaining_usage: Final = await self.get_remaining_model_group_usage(model_group) + remaining_usage: Final = await self._core_get_remaining_model_group_usage(model_group) apply_remaining_usage_headers(additional_headers, remaining_usage) return response @@ -13192,16 +13358,16 @@ class Router: This index allows us to find all deployments for a given model_name in O(1) time instead of O(n) linear scan through the entire model_list. """ - self.model_name_to_deployment_indices.clear() - self.team_model_to_deployment_indices.clear() - self.team_public_model_names = frozenset() + self._core_model_name_to_deployment_indices.clear() + self._core_team_model_to_deployment_indices.clear() + self._core_team_public_model_names = frozenset() for idx, model in enumerate(model_list): model_name = model.get("model_name") if model_name: - if model_name not in self.model_name_to_deployment_indices: - self.model_name_to_deployment_indices[model_name] = [] - self.model_name_to_deployment_indices[model_name].append(idx) + if model_name not in self._core_model_name_to_deployment_indices: + self._core_model_name_to_deployment_indices[model_name] = [] + self._core_model_name_to_deployment_indices[model_name].append(idx) self._update_team_model_index(model, idx) @@ -13212,7 +13378,7 @@ class Router: requests arrive before model_id_to_deployment_index_map is populated. """ # First populate the model_list - self.model_list = [] + self._core_model_list = [] self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() for _, model in enumerate(model_list): @@ -13224,7 +13390,7 @@ class Router: if model_id is None: model_name = model.get("model_name", "") litellm_params = model.get("litellm_params", {}) - model_id = self.generate_model_id(model_name, litellm_params) + model_id = self._core_generate_model_id(model_name, litellm_params) # Update the model_info in the original list if "model_info" not in model: model["model_info"] = {} @@ -13232,7 +13398,7 @@ class Router: self._add_model_to_list_and_index_map(model=model, model_id=model_id) - def get_model_ids(self, model_name: str | None = None, exclude_team_models: bool = False) -> list[str]: + def _core_get_model_ids(self, model_name: str | None = None, exclude_team_models: bool = False) -> list[str]: """ if 'model_name' is none, returns all. @@ -13245,7 +13411,7 @@ class Router: if model_name is not None: # O(1) lookup in model_name index, then O(k) iteration where k = deployments for this model_name - for model in self.deployments_named(model_name) or (): + for model in self._core_deployments_named(model_name) or (): if "model_info" in model and "id" in model["model_info"]: if exclude_team_models and model["model_info"].get("team_id"): continue @@ -13255,7 +13421,7 @@ class Router: # Use the index map keys for O(n) where n = total deployments with self._deployment_index_lock: id_indices: Final = tuple(self.model_id_to_deployment_index_map.items()) - models: Final = tuple(self.model_list) + models: Final = tuple(self._core_model_list) for model_id, idx in id_indices: model = models[idx] if "model_info" in model and "id" in model["model_info"]: @@ -13265,7 +13431,7 @@ class Router: return ids - def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: + def _core_get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: """ Deployment ids that could serve ``model`` for ``team_id``, following the same precedence ``_common_checks_available_deployment`` uses to build a candidate pool: @@ -13279,7 +13445,7 @@ class Router: check tell a genuine cross-group route from same-group unavailability without leaking deployment ids into request kwargs bound for the provider. """ - resolved: Final = self.get_model_from_alias(model=model) or model + resolved: Final = self._core_get_model_from_alias(model=model) or model routing_group_members: Final = self._get_routing_group_deployments(model=resolved, team_id=team_id) if routing_group_members is not None: return self._deployment_ids(routing_group_members) @@ -13291,7 +13457,7 @@ class Router: return self._deployment_ids( (early_deployments,) if isinstance(early_deployments, Mapping) else early_deployments ) - return self._deployment_ids(self.get_all_deployments(model_name=resolved, team_id=team_id)) + return self._deployment_ids(self._core_get_all_deployments(model_name=resolved, team_id=team_id)) @staticmethod def _deployment_ids(deployments: Sequence[Mapping[str, object]]) -> frozenset[str]: @@ -13313,7 +13479,7 @@ class Router: region, typed for the tag routing filter """ allowed_model_region: Final = None if request_kwargs is None else request_kwargs.get("allowed_model_region") - if allowed_model_region is None or not self.enable_pre_call_checks or not has_request_input: + if allowed_model_region is None or not self._core_enable_pre_call_checks or not has_request_input: return tuple(deployments) if not isinstance(allowed_model_region, str): return () @@ -13377,7 +13543,7 @@ class Router: remaining = rest return routable_ids - def has_model_id(self, candidate_id: str) -> bool: + def _core_has_model_id(self, candidate_id: str) -> bool: """ O(1) membership check for a deployment ID without allocating large lists. @@ -13390,7 +13556,7 @@ class Router: """ return candidate_id in self.model_id_to_deployment_index_map - def resolve_model_name_from_model_id( + def _core_resolve_model_name_from_model_id( self, model_id: str | None, custom_llm_provider: str | None = None ) -> str | None: """ @@ -13420,16 +13586,16 @@ class Router: return None # Strategy 1: Check if model_id directly matches a model_name or deployment ID - if model_id in self.model_names: + if model_id in self._core_model_names: return model_id - if self.has_model_id(model_id): - deployment = self.get_deployment(model_id=model_id) + if self._core_has_model_id(model_id): + deployment = self._core_get_deployment(model_id=model_id) if deployment is not None and deployment.model_name: return deployment.model_name return model_id # Strategy 2: Search through router's model_list to find by litellm_params.model - all_models: Final = self.get_model_list(model_name=None) + all_models: Final = self._core_get_model_list(model_name=None) if not all_models: return None @@ -13450,30 +13616,30 @@ class Router: if model_name: return model_name - if self.is_recognized_model(model_id): + if self._core_is_recognized_model(model_id): return model_id if custom_llm_provider is None: return None - if self.pattern_router.routes_to_provider(model_id, custom_llm_provider): + if self._core_pattern_router.routes_to_provider(model_id, custom_llm_provider): return model_id provider_prefixed: Final = f"{custom_llm_provider}/{model_id}" - if not model_id.startswith(f"{custom_llm_provider}/") and self.pattern_router.routes_to_provider( + if not model_id.startswith(f"{custom_llm_provider}/") and self._core_pattern_router.routes_to_provider( provider_prefixed, custom_llm_provider ): return provider_prefixed return None - def resolve_served_model_name(self, model_name: str, served_model: str) -> str: + def _core_resolve_served_model_name(self, model_name: str, served_model: str) -> str: """ `served_model` when `model_name` routes to it (the alias target, the routing group member, or the name itself), so a request can return to the group that served an earlier one, else `model_name`. """ - target: Final = self.get_model_from_alias(model_name) or model_name + target: Final = self._core_get_model_from_alias(model_name) or model_name routing_group: Final = self.get_routing_group(target) members: Final = routing_group.models if routing_group is not None else () return served_model if served_model == target or served_model in members else model_name - def map_team_model( + def _core_map_team_model( self, team_model_name: str | None, team_id: str, custom_llm_provider: str | None = None ) -> str | None: """ @@ -13483,7 +13649,7 @@ class Router: use the first team deployment of that provider, pinned by ID when its public name also covers another provider. """ - models: Final = self.get_model_list(model_name=team_model_name, team_id=team_id) + models: Final = self._core_get_model_list(model_name=team_model_name, team_id=team_id) if not models: return None if team_model_name is None and custom_llm_provider is not None: @@ -13536,7 +13702,7 @@ class Router: # handled downstream by the pattern_router in _common_checks_available_deployment. return None - def should_include_deployment( + def _core_should_include_deployment( self, model_name: str, model: DeploymentTypedDict | dict, team_id: str | None = None ) -> bool: """ @@ -13561,7 +13727,7 @@ class Router: # No match: deployment is for a different team or doesn't match the requested model return False - def get_all_deployments( + def _core_get_all_deployments( self, model_name: str, model_alias: str | None = None, @@ -13590,7 +13756,7 @@ class Router: if team_models is not None: # O(k) where k = team deployments for this model_name (typically 1-10) for model in team_models: - if not self.should_include_deployment(model_name=model_name, model=model, team_id=team_id): + if not self._core_should_include_deployment(model_name=model_name, model=model, team_id=team_id): continue if model_alias is not None: alias_model = model.copy() @@ -13602,11 +13768,11 @@ class Router: return returned_models # O(1) lookup in model_name index - named_models: Final = self.deployments_named(model_name) + named_models: Final = self._core_deployments_named(model_name) if named_models is not None: # O(k) where k = deployments for this model_name (typically 1-10) for model in named_models: - if self.should_include_deployment(model_name=model_name, model=model, team_id=team_id): + if self._core_should_include_deployment(model_name=model_name, model=model, team_id=team_id): if model_alias is not None: # Optimized: Use shallow copy since we only modify top-level model_name # This is much faster than deepcopy for nested dict structures @@ -13619,8 +13785,8 @@ class Router: # Fallback: if team_id is provided and model_name not in index, # check if model_name matches any team_public_model_name # O(n) scan but only when team_id lookup fails - for _idx, model in enumerate(self.model_list): - if self.should_include_deployment(model_name=model_name, model=model, team_id=team_id): + for _idx, model in enumerate(self._core_model_list): + if self._core_should_include_deployment(model_name=model_name, model=model, team_id=team_id): if model_alias is not None: # Optimized: Use shallow copy since we only modify top-level model_name alias_model = model.copy() @@ -13631,16 +13797,14 @@ class Router: return returned_models - _get_all_deployments = get_all_deployments - - def get_model_names(self, team_id: str | None = None) -> list[str]: + def _core_get_model_names(self, team_id: str | None = None) -> list[str]: """ Returns all possible model names for the router, including models defined via model_group_alias. If a team_id is provided, only deployments configured with that team_id (i.e. team‐specific models) will yield their team public name. """ - deployments: Final = self.get_model_list() or [] + deployments: Final = self._core_get_model_list() or [] model_names: Final = [] for deployment in deployments: @@ -13654,7 +13818,7 @@ class Router: return model_names - def get_fully_blocked_model_names(self) -> set[str]: + def _core_get_fully_blocked_model_names(self) -> set[str]: """ Returns the set of model_names where every backing deployment has `blocked=True`. @@ -13662,7 +13826,7 @@ class Router: surfacing them on admin endpoints (e.g. `/model/info`). A model with at least one non-blocked deployment is still serviceable and remains visible. """ - deployments: Final = self.get_model_list() or [] + deployments: Final = self._core_get_model_list() or [] blocked_by_name: Final[dict[str, bool]] = {} for deployment in deployments: name = deployment.get("model_name") or "" @@ -13676,20 +13840,18 @@ class Router: return {name for name, fully_blocked in blocked_by_name.items() if fully_blocked} @staticmethod - def are_all_deployments_blocked( - deployments: list[DeploymentTypedDict], + def _core_are_all_deployments_blocked( + deployments: Sequence[DeploymentTypedDict], ) -> bool: return len(deployments) > 0 and all( (deployment.get("model_info") or {}).get("blocked") is True for deployment in deployments ) - _are_all_deployments_blocked = are_all_deployments_blocked - def _is_model_fully_blocked(self, model: str) -> bool: - deployments: Final = self.get_model_list(model_name=model) or [] - return self.are_all_deployments_blocked(deployments=deployments) + deployments: Final = self._core_get_model_list(model_name=model) or [] + return self._core_are_all_deployments_blocked(deployments=deployments) - async def async_get_fully_unhealthy_model_names(self) -> set[str]: + async def _core_async_get_fully_unhealthy_model_names(self) -> set[str]: """ Returns the set of model names where every backing deployment is currently marked unhealthy by background health checks (and the health state is not stale). @@ -13717,12 +13879,12 @@ class Router: request for it still be attempted?". A hidden model can still be called directly. """ - if self.allowed_fails_policy is not None: + if self._core_allowed_fails_policy is not None: return set() - unhealthy_ids: Final = await self.health_state_cache.async_get_unhealthy_deployment_ids() + unhealthy_ids: Final = await self._core_health_state_cache.async_get_unhealthy_deployment_ids() if not unhealthy_ids: return set() - deployments: Final = self.get_model_list() or [] + deployments: Final = self._core_get_model_list() or [] unhealthy_by_name: Final[dict[str, bool]] = {} for deployment in deployments: model_info = deployment.get("model_info") or {} @@ -13771,7 +13933,7 @@ class Router: """ return bool(model_info and model_info.get("team_id")) - def get_model_list_from_model_alias(self, model_name: str | None = None) -> list[DeploymentTypedDict]: + def _core_get_model_list_from_model_alias(self, model_name: str | None = None) -> list[DeploymentTypedDict]: """ Helper function to get model list from model alias. @@ -13781,11 +13943,11 @@ class Router: if model_name is not None: # Fast path: direct dict lookup avoids scanning all aliases for non-alias model names. - if model_name not in self.model_group_alias: + if model_name not in self._core_model_group_alias: return returned_models - alias_items = [(model_name, self.model_group_alias[model_name])] + alias_items = [(model_name, self._core_model_group_alias[model_name])] else: - alias_items = list(self.model_group_alias.items()) + alias_items = list(self._core_model_group_alias.items()) for model_alias, model_value in alias_items: if isinstance(model_value, str): @@ -13803,11 +13965,13 @@ class Router: {**row, "model_name": model_alias} for row in self._materialize_routing_group_rows((alias_group,)) ) else: - returned_models.extend(self.get_all_deployments(model_name=_router_model_name, model_alias=model_alias)) + returned_models.extend( + self._core_get_all_deployments(model_name=_router_model_name, model_alias=model_alias) + ) return returned_models - def get_model_list_from_routing_groups(self, model_name: str | None = None) -> Sequence[DeploymentTypedDict]: + def _core_get_model_list_from_routing_groups(self, model_name: str | None = None) -> Sequence[DeploymentTypedDict]: """ Callable routing groups materialized as model-list rows, mirroring `get_model_list_from_model_alias`: each member deployment is emitted @@ -13825,7 +13989,7 @@ class Router: rows: Final = self._materialize_routing_group_rows( tuple( callable_group - for name in self.routing_groups + for name in self._core_routing_groups if (callable_group := self.get_routing_group(name)) is not None ) ) @@ -13837,7 +14001,7 @@ class Router: self._as_routing_group_row(apply_routing_group_priority(group, member, deployment)) for group in groups for member in group.models - for deployment in self.get_all_deployments(model_name=member, model_alias=group.group_name) + for deployment in self._core_get_all_deployments(model_name=member, model_alias=group.group_name) ) @staticmethod @@ -13957,7 +14121,7 @@ class Router: comparing. The filter asks the same question, so a param named by the allowlist on the tier overlay, the request, or a deployment's own litellm_params is never a drop candidate. """ - deployments: Final = self.get_model_list(model_name=model) or () + deployments: Final = self._core_get_model_list(model_name=model) or () if not deployments: return tier_params allowlisted: Final = self._declared_param_allowlist(tier_params) | self._declared_param_allowlist( @@ -13978,7 +14142,7 @@ class Router: ) return MappingProxyType({key: value for key, value in tier_params.items() if key not in unsupported}) - def get_model_list( + def _core_get_model_list( self, model_name: str | None = None, team_id: str | None = None ) -> list[DeploymentTypedDict] | None: """ @@ -13991,16 +14155,16 @@ class Router: returned_models: list[DeploymentTypedDict] = [] if model_name is not None: - returned_models.extend(self.get_all_deployments(model_name=model_name, team_id=team_id)) + returned_models.extend(self._core_get_all_deployments(model_name=model_name, team_id=team_id)) - returned_models.extend(self.get_model_list_from_model_alias(model_name=model_name)) - returned_models.extend(self.get_model_list_from_routing_groups(model_name=model_name)) + returned_models.extend(self._core_get_model_list_from_model_alias(model_name=model_name)) + returned_models.extend(self._core_get_model_list_from_routing_groups(model_name=model_name)) if len(returned_models) == 0 and model_name is not None: returned_models.extend(self._get_wildcard_deployments(model_name=model_name, team_id=team_id)) if model_name is None: - returned_models += self.model_list + returned_models += self._core_model_list return returned_models @@ -14009,31 +14173,31 @@ class Router: The deployments of the wildcard routes matching model_name (the proxy-wide ones, plus team_id's own when given), each emitted under model_name. """ - team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None matches: Final = [ - *self.pattern_router.get_deployments_by_pattern(model=model_name), + *self._core_pattern_router.get_deployments_by_pattern(model=model_name), *(team_router.get_deployments_by_pattern(model=model_name) if team_router is not None else ()), ] return [{**DeploymentTypedDict(**m), "model_name": model_name} for m in matches] - def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: + def _core_wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: """The deployments routing serves ``model`` from through a wildcard: ``team_id``'s own pattern matching it, else the most specific proxy-wide pattern, never both.""" - team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None team_owned: Final[Sequence[Mapping[str, object]]] = ( team_router.get_deployments_by_pattern(model=model) if team_router is not None else () ) - return team_owned or self.pattern_router.get_deployments_by_pattern(model=model) + return team_owned or self._core_pattern_router.get_deployments_by_pattern(model=model) - def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: + def _core_wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: """The models wildcard_route_deployments would route ``model`` to, without copying the deployments.""" - team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None + team_router: Final = self._core_team_pattern_routers.get(team_id) if team_id is not None else None team_owned: Final = ( team_router.get_models_by_pattern(model=model) if team_router is not None else frozenset[str]() ) - return team_owned or self.pattern_router.get_models_by_pattern(model=model) + return team_owned or self._core_pattern_router.get_models_by_pattern(model=model) - def team_ptu_ceiling(self, team_id: str, model_group: str) -> PTUTeamCeiling | None: + def _core_team_ptu_ceiling(self, team_id: str, model_group: str) -> PTUTeamCeiling | None: """The per-minute ceiling ``team_id``'s PTU shares set on the group serving ``model_group``, else None, kept until the model list next changes.""" if team_id not in self._ptu_share_teams(): @@ -14041,18 +14205,18 @@ class Router: return self._cached_team_ptu_ceiling(team_id, model_group) def _collect_ptu_share_teams(self) -> frozenset[str]: - return ptu_share_teams(self.get_model_list() or ()) + return ptu_share_teams(self._core_get_model_list() or ()) def _resolve_team_ptu_ceiling(self, team_id: str, model_group: str) -> PTUTeamCeiling | None: return team_ptu_ceiling( - self.get_model_list() or (), - self.model_list, + self._core_get_model_list() or (), + self._core_model_list, team_id, model_group, - lambda name: self.wildcard_route_deployments(name, team_id), + lambda name: self._core_wildcard_route_deployments(name, team_id), ) - def get_model_list_of_routed_group(self, model_group: str) -> list[DeploymentTypedDict]: + def _core_get_model_list_of_routed_group(self, model_group: str) -> list[DeploymentTypedDict]: """ The deployments a request the router has already resolved to model_group is served from: the ones named model_group, the routing group of that name, or @@ -14063,13 +14227,13 @@ class Router: an alias target is served by its own deployments, never by a second hop: in the chain X -> T -> U a request to X is served from T's deployments. """ - named: Final = [ - *self._get_all_deployments(model_name=model_group), - *self.get_model_list_from_routing_groups(model_name=model_group), - ] + own: Final = cast( # cast-ok: the private alias returns what get_all_deployments returns + list[DeploymentTypedDict], self._get_all_deployments(model_name=model_group) + ) + named: Final = [*own, *self._core_get_model_list_from_routing_groups(model_name=model_group)] return named or self._get_wildcard_deployments(model_name=model_group) - def resolved_litellm_models( + def _core_resolved_litellm_models( self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False ) -> tuple[str, ...]: """The provider model strings `model_name` can actually be served by on this proxy. @@ -14087,26 +14251,26 @@ class Router: if use_base_model: return tuple( backend_model - for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or () + for deployment in self._core_get_model_list(model_name=model_name, team_id=team_id) or () if (backend_model := self._cost_map_backend_model(deployment)) ) return tuple( litellm_model - for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or () + for deployment in self._core_get_model_list(model_name=model_name, team_id=team_id) or () if isinstance(litellm_model := deployment.get("litellm_params", {}).get("model"), str) and litellm_model ) def _invalidate_model_group_info_cache(self) -> None: """Invalidate the cached model group info. - Call this whenever self.model_list is modified to ensure the cache is rebuilt. + Call this whenever self._core_model_list is modified to ensure the cache is rebuilt. Also clears the auth-layer zero-cost cache, which depends on the same ``ModelGroupInfo`` data — without this, an in-place pricing update on an existing deployment (same model count) would keep a stale ``True`` result and bypass budget enforcement. """ self._cached_get_model_group_info.cache_clear() - self.cached_deployment_model_info.cache_clear() + self._core_cached_deployment_model_info.cache_clear() self._ptu_share_teams.cache_clear() self._cached_team_ptu_ceiling.cache_clear() self._cached_deployment_tracks_usage.cache_clear() @@ -14117,11 +14281,14 @@ class Router: def _invalidate_access_groups_cache(self) -> None: """Invalidate the cached access groups. - Call this whenever self.model_list is modified to ensure the cache is rebuilt. + Call this whenever self._core_model_list is modified to ensure the cache is rebuilt. It also + advances ``registry().version``, so every write path (``apply``, the old ``set_model_list`` + and ``add_deployment``) moves it. """ self._access_groups_cache = None + self._registry_version += 1 - def get_model_access_groups( + def _core_get_model_access_groups( self, model_name: str | None = None, model_access_group: str | None = None, @@ -14146,7 +14313,7 @@ class Router: access_groups: Final = defaultdict(list) - model_list: Final = self.get_model_list(model_name=model_name, team_id=team_id) + model_list: Final = self._core_get_model_list(model_name=model_name, team_id=team_id) if model_list: for m in model_list: _model_info = m.get("model_info") @@ -14167,12 +14334,12 @@ class Router: return access_groups - def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: + def _core_is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: """ Return True if model access group is a wildcard route """ # GET ACCESS GROUPS - access_groups: Final = self.get_model_access_groups(model_access_group=model_access_group) + access_groups: Final = self._core_get_model_access_groups(model_access_group=model_access_group) if len(access_groups) == 0: return False @@ -14181,14 +14348,12 @@ class Router: for model in models: # CHECK IF MODEL ACCESS GROUP IS A WILDCARD ROUTE - if self.pattern_router.route(request=model) is not None: + if self._core_pattern_router.route(request=model) is not None: return True return False - _is_model_access_group_for_wildcard_route = is_model_access_group_for_wildcard_route - - def get_settings(self): + def _core_get_settings(self) -> dict[str, object]: # mutable-ok: the settings snapshot is built fresh per call """ Get router settings method, returns a dictionary of the settings and their values. For example get the set values for routing_strategy_args, routing_strategy, allowed_fails, cooldown_time, num_retries, timeout, max_retries, retry_after @@ -14217,18 +14382,19 @@ class Router: ] for var in vars_to_include: - if var in _all_vars: - _settings_to_return[var] = _all_vars[var] + stored = var if var in _all_vars else "_core_" + var + if stored in _all_vars: + _settings_to_return[var] = _all_vars[stored] if ( var == "routing_strategy_args" - and self.routing_strategy == "latency-based-routing" + and self._core_routing_strategy == "latency-based-routing" and self.lowestlatency_logger is not None ): _settings_to_return[var] = self.lowestlatency_logger.routing_args.json() _settings_to_return["routing_groups"] = [ group.model_dump(exclude=frozenset(("model_priorities",)) if group.model_priorities is None else None) - for group in self.routing_groups.values() + for group in self._core_routing_groups.values() ] return _settings_to_return @@ -14242,7 +14408,7 @@ class Router: routing_strategy=routing_strategy, routing_strategy_args=kwargs.get("routing_strategy_args", {}) ) - def update_settings(self, **kwargs): + def _core_update_settings(self, **kwargs) -> None: """ Update the router settings. """ @@ -14254,8 +14420,8 @@ class Router: "cooldown_time", ] - _existing_router_settings: Final = self.get_settings() - model_group_alias_before: Final = self.model_group_alias + _existing_router_settings: Final = self._core_get_settings() + model_group_alias_before: Final = self._core_model_group_alias rebuild_routing_groups = False routing_args_updated = False for var in kwargs: @@ -14288,14 +14454,14 @@ class Router: if routing_args_updated: self._apply_updated_routing_strategy_args() - if self.model_group_alias != model_group_alias_before: + if self._core_model_group_alias != model_group_alias_before: self._invalidate_model_group_info_cache() if rebuild_routing_groups: routing_groups_input: Final = kwargs.get("routing_groups", self._routing_groups_input) self._init_routing_groups(routing_groups_input) self._routing_groups_input = routing_groups_input - verbose_router_logger.debug("Updated Router settings: %s", self.get_settings()) + verbose_router_logger.debug("Updated Router settings: %s", self._core_get_settings()) @staticmethod def _is_flag_enabled(value: object) -> bool: @@ -14327,10 +14493,10 @@ class Router: parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(kwargs) if client_type == "max_parallel_requests": cache_key = f"{model_id}_max_parallel_requests_client" - client = self.cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) + client = self._core_cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) if client is None: InitalizeCachedClient.set_max_parallel_requests_client(litellm_router_instance=self, model=deployment) - client = self.cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) + client = self._core_cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) return client elif client_type == "async": if ( @@ -14342,11 +14508,11 @@ class Router: return None if kwargs.get("stream") is True: cache_key = f"{model_id}_stream_async_client" - client = self.cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) + client = self._core_cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) return client else: cache_key = f"{model_id}_async_client" - client = self.cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) + client = self._core_cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) return client else: if ( @@ -14358,11 +14524,11 @@ class Router: return None if kwargs.get("stream") is True: cache_key = f"{model_id}_stream_client" - client = self.cache.get_cache(key=cache_key, parent_otel_span=parent_otel_span) + client = self._core_cache.get_cache(key=cache_key, parent_otel_span=parent_otel_span) return client else: cache_key = f"{model_id}_client" - client = self.cache.get_cache(key=cache_key, parent_otel_span=parent_otel_span) + client = self._core_cache.get_cache(key=cache_key, parent_otel_span=parent_otel_span) return client def _count_pre_call_check_tokens( @@ -14425,7 +14591,7 @@ class Router: def _deployment_max_input_tokens(self, model: str, deployment: Mapping[str, object]) -> int | None: """The deployment's declared context window, or None when it declares none or cannot be resolved.""" try: - model_info: Final = self.get_router_model_info( + model_info: Final = self._core_get_router_model_info( deployment=cast(dict, deployment), # cast-ok: router deployments are plain dicts received_model_name=model, ) @@ -14772,7 +14938,7 @@ class Router: group_deployments: Final = ( tuple(request_deployments) if request_deployments is not None - else (tuple(self.get_all_deployments(model_name=model)) or typed_deployments) + else (tuple(self._core_get_all_deployments(model_name=model)) or typed_deployments) ) if ( not _context_window_error @@ -14802,7 +14968,7 @@ class Router: return _returned_deployments - def get_model_from_alias(self, model: str) -> str | None: + def _core_get_model_from_alias(self, model: str) -> str | None: """ Get the model from the alias. @@ -14810,17 +14976,15 @@ class Router: - str, the litellm model name - None, if model is not in model group alias """ - return resolve_model_group_alias(self.model_group_alias, model) - - _get_model_from_alias = get_model_from_alias + return resolve_model_group_alias(self._core_model_group_alias, model) def _get_deployment_by_litellm_model(self, model: str) -> list[DeploymentTypedDict]: """ Get the deployment by litellm model. """ - return [m for m in self.model_list if m["litellm_params"]["model"] == model] + return [m for m in self._core_model_list if m["litellm_params"]["model"] == model] - def get_routable_deployments( + def _core_get_routable_deployments( self, model: str, request_kwargs: Mapping[str, object] = MappingProxyType({}), @@ -14842,16 +15006,16 @@ class Router: include_team_models: bool = False, ) -> tuple[str, list | dict] | None: """ - When ``model`` is not in ``self.model_names``, try team routes, pattern routes, + When ``model`` is not in ``self._core_model_names``, try team routes, pattern routes, team pattern routers, then default deployment. Returns None if none apply. """ - if model in self.model_names: + if model in self._core_model_names: return None # Check for team-specific deployments by team_public_model_name. # This intentionally takes priority over team pattern routers below, # so that named team deployments shadow wildcard/pattern routes. if request_team_id is not None: - team_deployments = self.get_all_deployments(model_name=model, team_id=request_team_id) + team_deployments = self._core_get_all_deployments(model_name=model, team_id=request_team_id) if team_deployments: return model, team_deployments elif include_team_models: @@ -14859,14 +15023,14 @@ class Router: if team_deployments: return model, team_deployments - wildcard_deployments: Final = self.wildcard_route_deployments(model, request_team_id) + wildcard_deployments: Final = self._core_wildcard_route_deployments(model, request_team_id) if wildcard_deployments: return model, list(wildcard_deployments) - if self.default_deployment is not None: + if self._core_default_deployment is not None: # Shallow copy with nested litellm_params copy (100x+ faster than deepcopy) - updated_deployment: Final = self.default_deployment.copy() - updated_deployment["litellm_params"] = self.default_deployment["litellm_params"].copy() + updated_deployment: Final = self._core_default_deployment.copy() + updated_deployment["litellm_params"] = self._core_default_deployment["litellm_params"].copy() updated_deployment["litellm_params"]["model"] = model return model, updated_deployment @@ -14876,8 +15040,8 @@ class Router: """Every team's deployments under public name `model`, for a proxy admin calling without a team.""" with self._deployment_index_lock: team_deployments: Final = [ - self.model_list[index] - for (_, public_model_name), indices in self.team_model_to_deployment_indices.items() + self._core_model_list[index] + for (_, public_model_name), indices in self._core_team_model_to_deployment_indices.items() if public_model_name == model for index in indices ] @@ -14898,16 +15062,16 @@ class Router: ) return team_deployments - def deployments_for_request( + def _core_deployments_for_request( self, model: str, request_kwargs: Mapping[str, object] ) -> Sequence[DeploymentTypedDict]: """The deployments `model` names for this caller, through the same alias, then team-first, then global, then admin-across-teams resolution `_common_checks_available_deployment` applies, so strategy selection and compression policy can never disagree with deployment selection about which marker a name means.""" - registered_name: Final = self.get_model_from_alias(model=model) or model + registered_name: Final = self._core_get_model_from_alias(model=model) or model team_id: Final = get_request_team_id(request_kwargs) - deployments: Final = self.get_all_deployments(model_name=registered_name, team_id=team_id) + deployments: Final = self._core_get_all_deployments(model_name=registered_name, team_id=team_id) if deployments or team_id is not None or not is_proxy_admin_request(request_kwargs): return deployments return self._team_deployments_across_teams(registered_name) @@ -14950,8 +15114,8 @@ class Router: request_team_id=request_team_id, ), ) - elif model not in self.model_names and self.has_model_id(model): - deployment: Final = self.get_deployment(model_id=model) + elif model not in self._core_model_names and self._core_has_model_id(model): + deployment: Final = self._core_get_deployment(model_id=model) if deployment is not None: deployment_model: Final = deployment.litellm_params.model return deployment_model, cast( # cast-ok: contract requires a plain dict for a single deployment @@ -14970,7 +15134,7 @@ class Router: f"LiteLLM Router: Trying to call specific deployment, but Model ID :{model} does not exist in Model ID map" ) - _model_from_alias: Final = self.get_model_from_alias(model=model) + _model_from_alias: Final = self._core_get_model_from_alias(model=model) if _model_from_alias is not None: model = _model_from_alias @@ -15009,7 +15173,7 @@ class Router: healthy_deployments = ( _routing_group_deployments if _routing_group_deployments is not None - else self.get_all_deployments(model_name=model, team_id=request_team_id) + else self._core_get_all_deployments(model_name=model, team_id=request_team_id) ) _pre_model_access_group_filter_len: Final = len(healthy_deployments) healthy_deployments = self._filter_reserved_deployments( @@ -15066,7 +15230,7 @@ class Router: ) # Re-assign model to the fallback and try to get deployments again model = fallback_model - healthy_deployments = self.get_all_deployments(model_name=model, team_id=request_team_id) + healthy_deployments = self._core_get_all_deployments(model_name=model, team_id=request_team_id) healthy_deployments = self._filter_reserved_deployments( model=model, healthy_deployments=self._filter_deployments_by_model_access_groups( @@ -15167,7 +15331,7 @@ class Router: if model in allowed_models or "*" in allowed_models or "all-proxy-models" in allowed_models: return healthy_deployments - access_groups_for_model: Final = self.get_model_access_groups(model_name=model, team_id=request_team_id) + access_groups_for_model: Final = self._core_get_model_access_groups(model_name=model, team_id=request_team_id) if len(access_groups_for_model) == 0: return healthy_deployments @@ -15186,7 +15350,7 @@ class Router: return filtered_deployments - async def async_get_healthy_deployments( + async def _core_async_get_healthy_deployments( self, model: str, request_kwargs: dict, @@ -15280,7 +15444,11 @@ class Router: # Safety net: only bypass cooldown filter when health-check routing is # driving cooldown (i.e. allowed_fails_policy is set). Without a policy, # cooldowns are from real request failures and must not be bypassed. - if not healthy_deployments and self.enable_health_check_routing and self.allowed_fails_policy is not None: + if ( + not healthy_deployments + and self._core_enable_health_check_routing + and self._core_allowed_fails_policy is not None + ): verbose_router_logger.warning( "All deployments in cooldown via health-check routing, bypassing cooldown filter" ) @@ -15296,7 +15464,7 @@ class Router: parent_otel_span=parent_otel_span, ) - if self.enable_pre_call_checks and (messages is not None or input is not None): + if self._core_enable_pre_call_checks and (messages is not None or input is not None): deployments_to_check: Final = healthy_deployments healthy_deployments = self._pre_call_checks( model=model, @@ -15324,7 +15492,7 @@ class Router: request_deployments=request_deployments, ) - # narrow to whatever `self.routing_plugins` left in candidate_models + # narrow to whatever `self._core_routing_plugins` left in candidate_models healthy_deployments = self._filter_by_routing_plugin_candidates( healthy_deployments=healthy_deployments, request_kwargs=request_kwargs, @@ -15544,11 +15712,11 @@ class Router: specific_deployment: bool | None, ): if ( - self.routing_strategy != "usage-based-routing-v2" - and self.routing_strategy != "simple-shuffle" - and self.routing_strategy != "cost-based-routing" - and self.routing_strategy != "latency-based-routing" - and self.routing_strategy != "least-busy" + self._core_routing_strategy != "usage-based-routing-v2" + and self._core_routing_strategy != "simple-shuffle" + and self._core_routing_strategy != "cost-based-routing" + and self._core_routing_strategy != "latency-based-routing" + and self._core_routing_strategy != "least-busy" ): # prevent regressions for other routing strategies, that don't have async get available deployments implemented. return self._get_available_deployment_unresolved( model=model, @@ -15565,7 +15733,7 @@ class Router: # this hook can modify the model, messages before the routing decision is made ######################################################### responses_call: Final = input is not None and messages is None - pre_routing_hook_response: Final = await self.async_pre_routing_hook( + pre_routing_hook_response: Final = await self._core_async_pre_routing_hook( model=model, request_kwargs=request_kwargs, messages=messages, @@ -15594,7 +15762,7 @@ class Router: routing_read_batch: Final = RoutingReadBatch.for_strategy(strategy, strategy_selector) with RoutingReadBatch.scoped(routing_read_batch): - healthy_deployments: Final = await self.async_get_healthy_deployments( + healthy_deployments: Final = await self._core_async_get_healthy_deployments( model=model, request_kwargs=request_kwargs, messages=messages, @@ -15620,7 +15788,7 @@ class Router: start_time: Final = time.time() if strategy == "simple-shuffle": shuffled: Final = simple_shuffle( - resolve_model_alias=self.get_model_from_alias, + resolve_model_alias=self._core_get_model_from_alias, healthy_deployments=healthy_deployments, model=model, request_kwargs=request_kwargs, @@ -15688,7 +15856,7 @@ class Router: ) raise e - async_get_available_deployment = ( # pyright: ignore[reportUnknownVariableType] # legacy selector return + _core_async_get_available_deployment = ( # pyright: ignore[reportUnknownVariableType] # legacy selector return async_resolve_pods_after( _async_get_available_deployment_unresolved # pyright: ignore[reportUnknownArgumentType] # legacy type ) @@ -15729,7 +15897,7 @@ class Router: # 1. Execute pre-routing hook responses_call: Final = input is not None and messages is None - pre_routing_hook_response: Final = await self.async_pre_routing_hook( + pre_routing_hook_response: Final = await self._core_async_pre_routing_hook( model=model, request_kwargs=request_kwargs, messages=messages, @@ -15748,7 +15916,7 @@ class Router: ) # 2. Get healthy deployments - healthy_deployments: Final = await self.async_get_healthy_deployments( + healthy_deployments: Final = await self._core_async_get_healthy_deployments( model=model, request_kwargs=request_kwargs, messages=messages, @@ -15794,7 +15962,7 @@ class Router: start_time: Final = time.perf_counter() if strategy == "simple-shuffle": return simple_shuffle( - resolve_model_alias=self.get_model_from_alias, + resolve_model_alias=self._core_get_model_from_alias, healthy_deployments=pass_through_deployments, model=model, request_kwargs=request_kwargs, @@ -15862,7 +16030,7 @@ class Router: messages: list[dict[str, object]] | None, ) -> RoutingContext: """ - Build a RoutingContext for `model`, run it through `self.routing_plugins` + Build a RoutingContext for `model`, run it through `self._core_routing_plugins` in order, then stash the narrowed candidate list and accumulated signals onto `request_kwargs["metadata"]` so `_filter_by_routing_plugin_candidates` (called later, during healthy-deployment filtering) can consume them. @@ -15871,7 +16039,7 @@ class Router: resolve_structured_messages, ) - candidate_models: Final = list(self.resolved_litellm_models(model)) + candidate_models: Final = list(self._core_resolved_litellm_models(model)) metadata_key: Final = self._get_metadata_variable_name_from_kwargs(request_kwargs) metadata: Final = request_kwargs.setdefault(metadata_key, {}) @@ -15883,7 +16051,7 @@ class Router: metadata=metadata, ) - for plugin in self.routing_plugins: + for plugin in self._core_routing_plugins: context = await plugin.run(context) metadata["routing_plugin_signals"] = context.signals @@ -15898,12 +16066,12 @@ class Router: request_kwargs: dict, ) -> list[dict] | dict: """ - Narrow `healthy_deployments` to whatever `self.routing_plugins` left in + Narrow `healthy_deployments` to whatever `self._core_routing_plugins` left in `context.candidate_models`. Raises rather than silently falling back to the unfiltered pool -- a plugin narrowing to nothing is a policy decision (e.g. no model this tenant's budget allows), not something to bypass. """ - if not self.routing_plugins or not isinstance(healthy_deployments, list): + if not self._core_routing_plugins or not isinstance(healthy_deployments, list): return healthy_deployments metadata_key: Final = self._get_metadata_variable_name_from_kwargs(request_kwargs) @@ -15947,10 +16115,15 @@ class Router: deployments: returning None hands the request to ordinary tag-aware deployment selection. """ - registries: Final = (self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers) + registries: Final = ( + self._core_auto_routers, + self._core_complexity_routers, + self._core_adaptive_routers, + self._core_quality_routers, + ) if not any(registries): return None - deployments: Final = self.deployments_for_request(model, request_kwargs) + deployments: Final = self._core_deployments_for_request(model, request_kwargs) registered_names: Final = tuple(dict.fromkeys(str(d["model_name"]) for d in deployments)) or (model,) candidates: Final = tuple( tagged for registry in registries for name in registered_names for tagged in registry.get(name, []) @@ -15962,7 +16135,7 @@ class Router: if request_tags: for tagged in candidates: if tagged.tags and is_valid_deployment_tag( - list(tagged.tags), request_tags, self.tag_filtering_match_any + list(tagged.tags), request_tags, self._core_tag_filtering_match_any ): return tagged for tagged in candidates: @@ -15970,7 +16143,7 @@ class Router: return tagged request_scoped_filtering: Final = request_kwargs.get("enable_tag_filtering") is True if ( - (self.enable_tag_filtering or request_scoped_filtering) + (self._core_enable_tag_filtering or request_scoped_filtering) and all(tagged.tags for tagged in candidates) and any(not self._is_strategy_marker_deployment(d) for d in deployments) ): @@ -16042,7 +16215,14 @@ class Router: ) -> str: if is_native_compaction_call(): return registered_model_name - if not any((self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers)): + if not any( + ( + self._core_auto_routers, + self._core_complexity_routers, + self._core_adaptive_routers, + self._core_quality_routers, + ) + ): return registered_model_name cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs) if cache_key is None or not isinstance(request_kwargs, dict): @@ -16055,7 +16235,7 @@ class Router: bound_model: Final = await self._get_claude_code_session_router_binding(cache_key) if not isinstance(bound_model, str): return registered_model_name - bound_registered_model: Final = self.get_model_from_alias(model=bound_model) or bound_model + bound_registered_model: Final = self._core_get_model_from_alias(model=bound_model) or bound_model if self._select_pre_routing_strategy(bound_registered_model, request_kwargs) is None: await self._delete_claude_code_session_router_binding(cache_key) return registered_model_name @@ -16076,7 +16256,7 @@ class Router: ) return registered_model_name - async def async_pre_routing_hook( + async def _core_async_pre_routing_hook( self, model: str, request_kwargs: dict[str, object], @@ -16098,7 +16278,7 @@ class Router: """ self._restore_client_output_ceiling(request_kwargs) self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) - requested_registered_model_name: Final = self.get_model_from_alias(model=model) or model + requested_registered_model_name: Final = self._core_get_model_from_alias(model=model) or model registered_model_name: Final = await self._resolve_claude_code_session_router( model=model, registered_model_name=requested_registered_model_name, @@ -16111,7 +16291,7 @@ class Router: # `_filter_by_routing_plugin_candidates`) and may attach signals for # downstream strategies (auto-router, complexity-router, ...) to read. ######################################################### - if self.routing_plugins: + if self._core_routing_plugins: await self._run_routing_plugins( model=registered_model_name, request_kwargs=request_kwargs, messages=messages ) @@ -16297,7 +16477,7 @@ class Router: ) -> DeploymentTypedDict | None: markers: Final = tuple( deployment - for deployment in self.deployments_for_request(model, request_kwargs) + for deployment in self._core_deployments_for_request(model, request_kwargs) if "model" in deployment["litellm_params"] and str(deployment["litellm_params"]["model"]).startswith(AUTO_ROUTER_MODEL_PREFIX) ) @@ -16364,7 +16544,7 @@ class Router: """ if pre_routing_hook_response is None or not selected_strategy.tags or not request_tags: return None - if not is_valid_deployment_tag(selected_strategy.tags, request_tags, self.tag_filtering_match_any): + if not is_valid_deployment_tag(selected_strategy.tags, request_tags, self._core_tag_filtering_match_any): return None return ConsumedRequestTagsStamp(model_group=pre_routing_hook_response.model, tags=selected_strategy.tags) @@ -16479,7 +16659,7 @@ class Router: """ Returns the deployment based on routing strategy """ - if self.routing_plugins: + if self._core_routing_plugins: raise ValueError( "Router(plugins=[...]) is configured, but this call resolved to the synchronous " "deployment-selection path, which never runs the routing-plugin pipeline. This " @@ -16541,7 +16721,11 @@ class Router: healthy_deployments=healthy_deployments, cooldown_deployments=cooldown_deployments, ) - if not healthy_deployments and self.enable_health_check_routing and self.allowed_fails_policy is not None: + if ( + not healthy_deployments + and self._core_enable_health_check_routing + and self._core_allowed_fails_policy is not None + ): verbose_router_logger.warning( "All deployments in cooldown via health-check routing, bypassing cooldown filter" ) @@ -16552,7 +16736,7 @@ class Router: ) # filter pre-call checks - if self.enable_pre_call_checks and (messages is not None or input is not None): + if self._core_enable_pre_call_checks and (messages is not None or input is not None): healthy_deployments = self._pre_call_checks( model=model, healthy_deployments=healthy_deployments, @@ -16626,15 +16810,15 @@ class Router: ) if len(healthy_deployments) == 0: - model_ids = self.get_model_ids(model_name=model) - _cooldown_time = self.cooldown_cache.get_min_cooldown( + model_ids = self._core_get_model_ids(model_name=model) + _cooldown_time = self._core_cooldown_cache.get_min_cooldown( model_ids=model_ids, parent_otel_span=parent_otel_span ) _cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span) raise RouterRateLimitError( model=model, cooldown_time=_cooldown_time, - enable_pre_call_checks=self.enable_pre_call_checks, + enable_pre_call_checks=self._core_enable_pre_call_checks, cooldown_list=_cooldown_list, model_ids=model_ids, cooldown_deployments=get_cooldown_deployment_info( @@ -16654,7 +16838,7 @@ class Router: # if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm ############## Check 'weight' param set for weighted pick ################# shuffled: Final = simple_shuffle( - resolve_model_alias=self.get_model_from_alias, + resolve_model_alias=self._core_get_model_from_alias, healthy_deployments=healthy_deployments, model=model, request_kwargs=request_kwargs, @@ -16673,15 +16857,15 @@ class Router: if deployment is None: verbose_router_logger.info("get_available_deployment for model: %s, No deployment available", model) - model_ids = self.get_model_ids(model_name=model) - _cooldown_time = self.cooldown_cache.get_min_cooldown( + model_ids = self._core_get_model_ids(model_name=model) + _cooldown_time = self._core_cooldown_cache.get_min_cooldown( model_ids=model_ids, parent_otel_span=parent_otel_span ) _cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span) raise RouterRateLimitError( model=model, cooldown_time=_cooldown_time, - enable_pre_call_checks=self.enable_pre_call_checks, + enable_pre_call_checks=self._core_enable_pre_call_checks, cooldown_list=_cooldown_list, model_ids=model_ids, ) @@ -16701,7 +16885,7 @@ class Router: ) ) - def get_available_deployment_for_pass_through( + def _core_get_available_deployment_for_pass_through( # noqa: ANN202 # unannotated before the rename too; its inferred return mixes untyped deployment dicts self, model: str, messages: list[dict[str, str]] | None = None, @@ -16794,7 +16978,7 @@ class Router: pass_through_deployments = self._filter_blocked_deployments(pass_through_deployments) # 5. Apply pre-call checks (if enabled) - if self.enable_pre_call_checks and (messages is not None or input is not None): + if self._core_enable_pre_call_checks and (messages is not None or input is not None): pass_through_deployments = self._pre_call_checks( model=model, healthy_deployments=pass_through_deployments, @@ -16811,15 +16995,15 @@ class Router: ) if len(pass_through_deployments) == 0: - model_ids = self.get_model_ids(model_name=model) - _cooldown_time = self.cooldown_cache.get_min_cooldown( + model_ids = self._core_get_model_ids(model_name=model) + _cooldown_time = self._core_cooldown_cache.get_min_cooldown( model_ids=model_ids, parent_otel_span=parent_otel_span ) _cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span) raise RouterRateLimitError( model=model, cooldown_time=_cooldown_time, - enable_pre_call_checks=self.enable_pre_call_checks, + enable_pre_call_checks=self._core_enable_pre_call_checks, cooldown_list=_cooldown_list, model_ids=pass_through_model_ids, cooldown_deployments=get_cooldown_deployment_info( @@ -16838,7 +17022,7 @@ class Router: # 6. Apply load balancing strategy if strategy == "simple-shuffle": return simple_shuffle( - resolve_model_alias=self.get_model_from_alias, + resolve_model_alias=self._core_get_model_from_alias, healthy_deployments=pass_through_deployments, model=model, request_kwargs=request_kwargs, @@ -16857,15 +17041,15 @@ class Router: verbose_router_logger.info( "get_available_deployment_for_pass_through model: %s, no available deployments", model ) - model_ids = self.get_model_ids(model_name=model) - _cooldown_time = self.cooldown_cache.get_min_cooldown( + model_ids = self._core_get_model_ids(model_name=model) + _cooldown_time = self._core_cooldown_cache.get_min_cooldown( model_ids=model_ids, parent_otel_span=parent_otel_span ) _cooldown_list = get_cooldown_deployments(litellm_router_instance=self, parent_otel_span=parent_otel_span) raise RouterRateLimitError( model=model, cooldown_time=_cooldown_time, - enable_pre_call_checks=self.enable_pre_call_checks, + enable_pre_call_checks=self._core_enable_pre_call_checks, cooldown_list=_cooldown_list, model_ids=model_ids, ) @@ -16914,7 +17098,7 @@ class Router: ] @staticmethod - def is_deployment_blocked(deployment: "Deployment") -> bool: + def _core_is_deployment_blocked(deployment: "Deployment") -> bool: """ Returns True when a `Deployment` Pydantic instance carries the admin-paused flag. Used by credential-lookup helpers so passthrough file / batch endpoints @@ -16925,8 +17109,6 @@ class Router: return False return getattr(model_info, "blocked", None) is True - _is_deployment_blocked = is_deployment_blocked - async def _async_filter_health_check_unhealthy_deployments( self, healthy_deployments: list[dict], @@ -16943,18 +17125,18 @@ class Router: Returns all deployments if health state is unavailable, stale, or would exclude every candidate (safety net). """ - if not self.enable_health_check_routing: + if not self._core_enable_health_check_routing: return healthy_deployments # When allowed_fails_policy is set, cooldown is the sole routing exclusion # mechanism -- skip the binary health check filter so the policy threshold # is respected before any deployment is excluded. With a model-group # allowlist the filter is already scoped, so listed groups keep it. - scoped_groups: Final = self.background_health_check_model_groups - if self.allowed_fails_policy is not None and scoped_groups is None: + scoped_groups: Final = self._core_background_health_check_model_groups + if self._core_allowed_fails_policy is not None and scoped_groups is None: return healthy_deployments - unhealthy_ids: Final = await self.health_state_cache.async_get_unhealthy_deployment_ids( + unhealthy_ids: Final = await self._core_health_state_cache.async_get_unhealthy_deployment_ids( parent_otel_span=parent_otel_span ) if not unhealthy_ids: @@ -16978,14 +17160,16 @@ class Router: parent_otel_span: Span | None = None, ) -> list[dict]: """Sync version of _async_filter_health_check_unhealthy_deployments.""" - if not self.enable_health_check_routing: + if not self._core_enable_health_check_routing: return healthy_deployments - scoped_groups: Final = self.background_health_check_model_groups - if self.allowed_fails_policy is not None and scoped_groups is None: + scoped_groups: Final = self._core_background_health_check_model_groups + if self._core_allowed_fails_policy is not None and scoped_groups is None: return healthy_deployments - unhealthy_ids: Final = self.health_state_cache.get_unhealthy_deployment_ids(parent_otel_span=parent_otel_span) + unhealthy_ids: Final = self._core_health_state_cache.get_unhealthy_deployment_ids( + parent_otel_span=parent_otel_span + ) if not unhealthy_ids: return healthy_deployments @@ -17047,7 +17231,7 @@ class Router: retry_policy=self.retry_policy, ) - def get_allowed_fails_from_policy(self, exception: Exception): + def _core_get_allowed_fails_from_policy(self, exception: Exception) -> int | None: """ BadRequestErrorRetries: Optional[int] = None AuthenticationErrorRetries: Optional[int] = None @@ -17056,7 +17240,7 @@ class Router: ContentPolicyViolationErrorRetries: Optional[int] = None """ # if we can find the exception then in the retry policy -> return the number of retries - allowed_fails_policy: Final[AllowedFailsPolicy | None] = self.allowed_fails_policy + allowed_fails_policy: Final[AllowedFailsPolicy | None] = self._core_allowed_fails_policy if allowed_fails_policy is None: return None @@ -17115,7 +17299,7 @@ class Router: default_webhook_url=router_alerting_config.webhook_url, ) - self.slack_alerting_logger = _slack_alerting_logger + self._core_slack_alerting_logger = _slack_alerting_logger litellm.logging_callback_manager.add_litellm_callback(_slack_alerting_logger) litellm.logging_callback_manager.add_litellm_success_callback( @@ -17123,7 +17307,7 @@ class Router: ) verbose_router_logger.info("\033[94m\nInitialized Alerting for litellm.Router\033[0m\n") - def set_custom_routing_strategy(self, CustomRoutingStrategy: CustomRoutingStrategyBase): + def _core_set_custom_routing_strategy(self, CustomRoutingStrategy: CustomRoutingStrategyBase) -> None: """ Sets get_available_deployment and async_get_available_deployment on an instanced of litellm.Router @@ -17139,30 +17323,30 @@ class Router: "get_available_deployment", resolve_pods_after_bound( strategy.get_available_deployment, # pyright: ignore[reportUnknownArgumentType] # legacy stub - self.kubernetes_pod_discovery, + self._core_kubernetes_pod_discovery, ), ) setattr( self, - "async_get_available_deployment", + "_core_async_get_available_deployment", async_resolve_pods_after_bound( strategy.async_get_available_deployment, # pyright: ignore[reportUnknownArgumentType] # legacy stub - self.kubernetes_pod_discovery, + self._core_kubernetes_pod_discovery, ), ) def _reset_custom_routing_strategy(self) -> None: - for attr in ("get_available_deployment", "async_get_available_deployment"): + for attr in ("get_available_deployment", "_core_async_get_available_deployment"): if attr in self.__dict__: delattr(self, attr) def flush_cache(self): litellm.cache = None - self.cache.flush_cache() + self._core_cache.flush_cache() session_in_memory_cache: Final = self._claude_code_session_router_cache.in_memory_cache if session_in_memory_cache is not None: session_in_memory_cache.flush_cache() - self.prompt_caching_pin_cache.in_memory_cache.flush_cache() + self._core_prompt_caching_pin_cache.in_memory_cache.flush_cache() def reset(self): ## clean up on close @@ -17172,3 +17356,16 @@ class Router: litellm._async_failure_callback = [] self.retry_policy = None self.flush_cache() + + +def _name_core_members_after_old_names() -> None: + # An old member that moved to ``_core_`` can still be reached as the function itself + # (``cached_deployment_model_info`` is an lru_cache over one), so errors that name the method + # keep the name it had before. + for name, member in vars(Router).items(): + old_name = name.removeprefix("_core_") + if old_name != name and inspect.isfunction(member) and hasattr(DeprecatedRouterMembers, old_name): + member.__qualname__ = f"Router.{old_name}" + + +_name_core_members_after_old_names() diff --git a/litellm/router_api.py b/litellm/router_api.py new file mode 100644 index 00000000000..c570bf6de6e --- /dev/null +++ b/litellm/router_api.py @@ -0,0 +1,1296 @@ +""" +The Router's entry points and the typed views they return. + +``litellm.Router`` exposes its endpoint methods (``acompletion``, ``aembedding``, ...) plus eight +entry points, and every other read or write that code outside the router needs goes through one +of them: + +- ``registry()`` -> ``RouterRegistry``: the configured deployments, indexed by id, model group, + alias, wildcard pattern and team, and the plugins they configure. +- ``settings()`` -> ``RouterSettings``: one frozen snapshot of the configuration a caller may read. +- ``health()`` -> ``RouterHealth``: cooldowns, blocked deployments, health checks and usage. +- ``select(model, ...)``: pick a deployment without calling it. +- ``apply(ChangeSet)`` / ``aapply(ChangeSet)``: runtime changes (deployments, settings, plugins), + returning an ``ApplyResult`` with the new registry version. +- ``observe(...)``: bind the router's routing observers to request logging. +- ``register_plugin(plugin)``: a custom routing strategy. +- ``retire()``: release the router's global registrations. + +The views read the router's state when they are called; they hold no copy of it. A Rust core +implements the same views (see litellm-rust/crates/router-core). Each query names the old Router +member it replaces in its docstring; ``DeprecatedRouterMembers`` keeps the old names working, typed +as they were. +""" + +# pyright: reportPrivateUsage=false, reportUnknownMemberType=false, reportUnknownVariableType=false +# pyright: reportMissingTypeArgument=false, reportUnknownParameterType=false +# The views are part of the router: they read and write the Router's ``_core_*`` members by design +# (see the router-core boundary in litellm/router.py), the same way Router's own methods do. Each +# selector query (``deployments(source=...)``, ``names(kind=...)``, ...) is overloaded per selector +# value and returns exactly the type of the ``_core_*`` member that value reads, which is the type +# the old public member had, so moving a caller onto the entry point keeps its types. Some of those +# members are still untyped (bare ``dict`` / ``list``) or mutable (the live objects); the overloads +# repeat them unchanged, and typing the members themselves is the core's job. + +import hashlib +import json +from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Set as AbstractSet +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, overload + +if TYPE_CHECKING: + from typing import Any # noqa: TID251 # only to spell the cores' untyped (Unknown) containers and one Any type + + from litellm.caching.caching import DualCache, RedisCache + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.router import Router + from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter + from litellm.router_strategy.auto_router.auto_router import AutoRouter + from litellm.router_strategy.budget_limiter import RouterBudgetLimiting + from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter + from litellm.router_strategy.quality_router.quality_router import QualityRouter + from litellm.router_utils.cooldown_cache import CooldownCache + from litellm.router_utils.health_state_cache import DeploymentHealthCache + from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter + from litellm.router_utils.ptu_shares import PTUTeamCeiling + from litellm.types.router import ( + AllowedFailsPolicy, + Deployment, + DeploymentModelListingInfo, + DeploymentTypedDict, + FallbackAccessCheck, + FallbackBudgetCheck, + GuardrailTypedDict, + ModelGroupInfo, + PreRoutingHookResponse, + RouterGeneralSettings, + RouterModelGroupAliasItem, + RoutingGroup, + RoutingPlugin, + SearchToolTypedDict, + TaggedPreRoutingStrategy, + ) + from litellm.types.utils import BudgetConfig + from litellm.types.utils import ModelInfo as ModelMapInfo + + # The cores' untyped containers (bare ``list`` / ``dict``, element types Unknown), spelled with Any: + # an overload set with one partially-unknown overload makes every call through it partially unknown. + UntypedList: TypeAlias = list[Any] # pyright: ignore[reportExplicitAny] # see above + UntypedDict: TypeAlias = dict[Any, Any] # pyright: ignore[reportExplicitAny] # see above + + # What registry().plugins(kind) returns, one member per kind. + RegistryPlugins: TypeAlias = ( + dict[str, list[TaggedPreRoutingStrategy[AutoRouter]]] + | dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] + | dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] + | dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] + | list[RoutingPlugin] + | RouterBudgetLimiting + | DualCache + | None + ) + + +DeploymentSource: TypeAlias = Literal[ + "group", + "all", + "config", + "named", + "alias", + "routing_groups", + "routed_group", + "wildcard", + "routable", + "first_in_group", + "for_request", +] +NameKind: TypeAlias = Literal[ + "ids", + "groups", + "group_set", + "deployment_names", + "team_public", + "litellm_models", + "litellm_base_models", + "wildcard", + "route_candidates", +] +ResolveKind: TypeAlias = Literal[ + "group", + "alias", + "served", + "from_id", + "team_model", + "recognized", + "upstream", + "group_of_id", + "wildcard_access_group", + "team_model_alternatives", + "routing_group_alternatives", + "search_tool_support", +] +GroupField: TypeAlias = Literal[ + "info", + "cached_info", + "display_name", + "mode", + "service_tiers", + "token_limits", + "listing_info", +] +CredentialForm: TypeAlias = Literal["params", "with_provider", "deployment"] +IndexName: TypeAlias = Literal["model_name", "team_model"] +PluginKind: TypeAlias = Literal[ + "auto", + "complexity", + "quality", + "adaptive", + "routing", + "budget_limiter", + "prompt_caching_pins", +] +SelectKind: TypeAlias = Literal["pick", "healthy", "pass_through"] +UsageKind: TypeAlias = Literal["current", "remaining"] + + +def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str: + """ + The stable id of a deployment that has no ``model_info.id``: a sha256 over the model group and + every litellm_params key and value, in order. Replaces ``Router.generate_model_id``. + """ + parts: Final = [model_group] + for k, v in litellm_params.items(): + for item in (k, v): + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict): + parts.append(json.dumps(item, default=_json_default_stable_id)) + else: + parts.append(str(item)) + return hashlib.sha256("".join(parts).encode()).hexdigest() + + +def _json_default_stable_id(obj: object) -> str: + from litellm.router import Router + + return Router._json_default_stable_id(obj) + + +class RouterRegistry: + """ + Read-only queries over the configured deployments (``registry()``). + + ``version`` changes whenever the deployments change, so a caller can tell whether + what it read earlier is still current. + """ + + __slots__ = ("_router",) + + def __init__(self, router: "Router") -> None: + self._router = router + + @property + def version(self) -> int: + return self._router._registry_version + + # -- one deployment ------------------------------------------------------------------------- + + def deployment(self, id: str) -> "Deployment | None": + """The deployment with this ``model_info.id`` (was ``get_deployment``).""" + return self._router._core_get_deployment(model_id=id) + + def has_deployment(self, id: str) -> bool: + """Whether a deployment has this id (was ``has_model_id``).""" + return self._router._core_has_model_id(id) + + def deployment_dict( + self, id: str + ) -> "UntypedDict | None": # a mutable list/dict, as before: Router._core_get_model_info's type, the live dict + """The deployment as its model_list dict (was ``get_model_info``).""" + return self._router._core_get_model_info(id=id) + + def deployment_model_info(self, id: str, model_name: str, *, cached: bool = False) -> "ModelMapInfo | None": + """ + Model info for a deployment, merged with the pricing map (was + ``get_deployment_model_info``; ``cached=True`` was ``cached_deployment_model_info``). + """ + if cached: + return self._router._core_cached_deployment_model_info(id, model_name) + return self._router._core_get_deployment_model_info(model_id=id, model_name=model_name) + + def router_model_info( + self, + deployment: "dict[str, object] | Deployment | None", # mutable-ok: the core takes a dict + received_model_name: str, + id: str | None = None, + ) -> "ModelMapInfo": + """Model info used for routing decisions (was ``get_router_model_info``).""" + r: Final = self._router + # The core's overloads split (deployment, id) by which one is None; this forwards either form unchanged. + return r._core_get_router_model_info( # pyright: ignore[reportCallIssue] # see above + deployment=deployment, # pyright: ignore[reportArgumentType] # see above + received_model_name=received_model_name, + id=id, # pyright: ignore[reportArgumentType] # see above + ) + + def discovered_model_info(self, id: str | None) -> Mapping[str, int]: + """Limits discovered from the provider for a deployment (was ``get_discovered_model_info``).""" + return self._router._core_get_discovered_model_info(id) + + def zero_cost(self) -> Mapping[str, bool] | None: + """The per-deployment zero-cost lookup the auth checks read (was ``_zero_cost_cache``).""" + return getattr(self._router, "_zero_cost_cache", None) + + # -- many deployments ----------------------------------------------------------------------- + + @overload + def deployments( + self, + model_name: str | None = None, + *, + source: Literal["group"] = "group", + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> "list[DeploymentTypedDict] | None": ... # mutable-ok: Router._core_get_model_list's type, the live lists + + @overload + def deployments( + self, + model_name: str | None = None, + *, + source: Literal["alias"], + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> "list[DeploymentTypedDict]": ... # mutable-ok: Router._core_get_model_list_from_model_alias's type + + @overload + def deployments( + self, + model_name: str, + *, + source: Literal["all", "routed_group"], + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> "list[DeploymentTypedDict]": ... # mutable-ok: the cores' type (get_all_deployments, ...) + + @overload + def deployments( + self, + model_name: str | None = None, + *, + source: Literal["config"], + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> Iterator[Mapping[str, object]]: ... + + @overload + def deployments( + self, + model_name: str | None = None, + *, + source: Literal["routing_groups"], + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> "Sequence[DeploymentTypedDict]": ... + + @overload + def deployments( + self, + model_name: str, + *, + source: Literal["for_request"], + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> "Sequence[DeploymentTypedDict]": ... + + @overload + def deployments( + self, + model_name: str, + *, + source: Literal["named"], + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> "tuple[DeploymentTypedDict, ...] | None": ... + + @overload + def deployments( + self, + model_name: str, + *, + source: Literal["routable"], + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> "tuple[DeploymentTypedDict, ...]": ... + + @overload + def deployments( + self, + model_name: str, + *, + source: Literal["wildcard"], + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> Sequence[Mapping[str, object]]: ... + + @overload + def deployments( + self, + model_name: str, + *, + source: Literal["first_in_group"], + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> "Deployment | None": ... + + def deployments( + self, + model_name: str | None = None, + *, + source: DeploymentSource = "group", + team_id: str | None = None, + model_alias: str | None = None, + request: Mapping[str, object] | None = None, + call_type: str | None = None, + ) -> "Sequence[DeploymentTypedDict] | Sequence[Mapping[str, object]] | Iterator[Mapping[str, object]] | Deployment | None": + """ + Deployments by name. ``source`` picks which index answers: + + - ``group``: the group, its aliases, routing groups, then wildcards; every deployment when + ``model_name`` is None (was ``get_model_list``) + - ``all``: every deployment of the group, with ``model_alias`` (was ``get_all_deployments``) + - ``config``: deployments that came from config rather than the DB (was ``config_deployments``) + - ``named``: the group index entry, or None (was ``deployments_named``) + - ``alias`` / ``routing_groups`` / ``routed_group``: (was ``get_model_list_from_model_alias``, + ``get_model_list_from_routing_groups``, ``get_model_list_of_routed_group``) + - ``wildcard``: deployments a wildcard route serves (was ``wildcard_route_deployments``) + - ``routable``: deployments that pass the routing checks for ``request`` and, when given, + the endpoint ``call_type`` (was ``get_routable_deployments``) + - ``first_in_group``: the first deployment of the group (was ``get_deployment_by_model_group_name``) + - ``for_request``: the deployments a request may use (was ``deployments_for_request``) + """ + r: Final = self._router + if source == "group": + return r._core_get_model_list(model_name=model_name, team_id=team_id) + if source == "config": + return r._core_config_deployments() + if source == "alias": + return r._core_get_model_list_from_model_alias(model_name=model_name) + if source == "routing_groups": + return r._core_get_model_list_from_routing_groups(model_name=model_name) + if model_name is None: + raise TypeError(f"registry().deployments(source={source!r}) needs a model_name") + if source == "all": + return r._core_get_all_deployments(model_name=model_name, model_alias=model_alias, team_id=team_id) + if source == "named": + return r._core_deployments_named(model_name) + if source == "routed_group": + return r._core_get_model_list_of_routed_group(model_name) + if source == "wildcard": + return r._core_wildcard_route_deployments(model_name, team_id) + if source == "routable": + return r._core_get_routable_deployments( + model_name, MappingProxyType({}) if request is None else request, call_type + ) + if source == "first_in_group": + return r._core_get_deployment_by_model_group_name(model_group_name=model_name) + if source == "for_request": + return r._core_deployments_for_request(model_name, request or {}) + raise ValueError(f"unknown deployment source {source!r}") + + def all(self) -> "UntypedList": # a mutable list/dict, as before: the model_list attribute's type, the live list + """The live model list, in configured order (was the ``model_list`` attribute).""" + return self._router._core_model_list + + def includes( + self, + deployment: "DeploymentTypedDict | dict[str, object]", # mutable-ok: the core takes a dict + *, + model_name: str | None = None, + team_id: str | None = None, + ) -> bool: + """ + Whether a deployment serves ``model_name`` for ``team_id`` (was + ``should_include_deployment``), or, without ``model_name``, whether the team may use it at + all (was ``deployment_usable_by_team``). + """ + from litellm.router import Router + + if model_name is None: + return Router._core_deployment_usable_by_team(deployment, team_id) + return self._router._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id) + + @overload + def index(self, name: Literal["model_name"]) -> dict[str, list[int]]: ... # mutable-ok: the live index + + @overload + def index(self, name: Literal["team_model"]) -> dict[tuple[str, str], list[int]]: ... # mutable-ok: the live index + + def index(self, name: IndexName) -> Mapping[str, Sequence[int]] | Mapping[tuple[str, str], Sequence[int]]: + """ + The raw name indexes: ``model_name`` maps a group name to model_list positions (was + ``model_name_to_deployment_indices``), ``team_model`` maps (team, public name) keys (was + ``team_model_to_deployment_indices``). + """ + if name == "model_name": + return self._router._core_model_name_to_deployment_indices + if name == "team_model": + return self._router._core_team_model_to_deployment_indices + raise ValueError(f"unknown index {name!r}") + + # -- names ---------------------------------------------------------------------------------- + + @overload + def names( + self, + model_name: str | None = None, + *, + kind: Literal["groups", "ids"] = "groups", + team_id: str | None = None, + exclude_team_models: bool = False, + ) -> list[str]: ... # mutable-ok: the cores' type (get_model_names, get_model_ids), a fresh list + + @overload + def names( + self, + model_name: str | None = None, + *, + kind: Literal["group_set"], + team_id: str | None = None, + exclude_team_models: bool = False, + ) -> set[str]: ... # mutable-ok: the model_names attribute's type, the live set + + @overload + def names( + self, + model_name: str | None = None, + *, + kind: Literal["deployment_names"], + team_id: str | None = None, + exclude_team_models: bool = False, + ) -> "UntypedList": ... # a mutable list/dict, as before: the deployment_names attribute's type, the live list + + @overload + def names( + self, + model_name: str | None = None, + *, + kind: Literal["team_public"], + team_id: str | None = None, + exclude_team_models: bool = False, + ) -> frozenset[str]: ... + + @overload + def names( + self, + model_name: str, + *, + kind: Literal["wildcard", "route_candidates"], + team_id: str | None = None, + exclude_team_models: bool = False, + ) -> frozenset[str]: ... + + @overload + def names( + self, + model_name: str, + *, + kind: Literal["litellm_models", "litellm_base_models"], + team_id: str | None = None, + exclude_team_models: bool = False, + ) -> tuple[str, ...]: ... + + def names( + self, + model_name: str | None = None, + *, + kind: NameKind = "groups", + team_id: str | None = None, + exclude_team_models: bool = False, + ) -> Sequence[str] | AbstractSet[str]: + """ + Names known to the router. ``kind``: + + - ``groups``: every routable model name, with team public names for ``team_id`` + (was ``get_model_names``) + - ``group_set``: the set of configured group names (was the ``model_names`` attribute) + - ``ids``: deployment ids, of one group or all (was ``get_model_ids``) + - ``deployment_names``: (was the ``deployment_names`` attribute) + - ``team_public``: team public model names (was ``team_public_model_names``) + - ``litellm_models``: upstream model names a group resolves to (was ``resolved_litellm_models``) + - ``litellm_base_models``: the same, with each deployment's ``base_model`` where it sets one + (was ``resolved_litellm_models(..., use_base_model=True)``) + - ``wildcard``: names a wildcard route serves (was ``wildcard_routed_models``) + - ``route_candidates``: deployment ids a request for the name may reach + (was ``get_candidate_model_ids_for_route``) + """ + r: Final = self._router + if kind == "groups": + return r._core_get_model_names(team_id=team_id) + if kind == "group_set": + return r._core_model_names + if kind == "ids": + return r._core_get_model_ids(model_name=model_name, exclude_team_models=exclude_team_models) + if kind == "deployment_names": + return r._core_deployment_names + if kind == "team_public": + return r._core_team_public_model_names + if model_name is None: + raise TypeError(f"registry().names(kind={kind!r}) needs a model_name") + if kind == "litellm_models": + return r._core_resolved_litellm_models(model_name, team_id=team_id) + if kind == "litellm_base_models": + return r._core_resolved_litellm_models(model_name, team_id=team_id, use_base_model=True) + if kind == "wildcard": + return r._core_wildcard_routed_models(model_name, team_id) + if kind == "route_candidates": + return r._core_get_candidate_model_ids_for_route(model_name, team_id=team_id) + raise ValueError(f"unknown name kind {kind!r}") + + # -- resolving a name ----------------------------------------------------------------------- + + @overload + def resolve( + self, + name: str, + *, + kind: Literal["group", "served"] = "group", + team_id: str | None = None, + custom_llm_provider: str | None = None, + served_model: str | None = None, + ) -> str: ... + + @overload + def resolve( + self, + name: str, + *, + kind: Literal["alias", "upstream"], + team_id: str | None = None, + custom_llm_provider: str | None = None, + served_model: str | None = None, + ) -> str | None: ... + + @overload + def resolve( + self, + name: str, + *, + kind: Literal["search_tool_support"], + team_id: str | None = None, + custom_llm_provider: str | None = None, + served_model: str | None = None, + ) -> tuple[bool | None, ...]: ... + + @overload + def resolve( + self, + name: str | None, + *, + kind: Literal["from_id", "team_model"], + team_id: str | None = None, + custom_llm_provider: str | None = None, + served_model: str | None = None, + ) -> str | None: ... + + @overload + def resolve( + self, + name: str, + *, + kind: Literal["recognized", "wildcard_access_group", "team_model_alternatives"], + team_id: str | None = None, + custom_llm_provider: str | None = None, + served_model: str | None = None, + ) -> bool: ... + + @overload + def resolve( + self, + name: str | None, + *, + kind: Literal["routing_group_alternatives"], + team_id: str | None = None, + custom_llm_provider: str | None = None, + served_model: str | None = None, + ) -> bool: ... + + @overload + def resolve( + self, + name: str, + *, + kind: Literal["group_of_id"], + team_id: str | None = None, + custom_llm_provider: str | None = None, + served_model: str | None = None, + ) -> "UntypedList | None": ... # a mutable list/dict, as before: Router._core_get_model_group's type, unchanged + + def resolve( + self, + name: str | None, + *, + kind: ResolveKind = "group", + team_id: str | None = None, + custom_llm_provider: str | None = None, + served_model: str | None = None, + ) -> str | bool | Sequence[object] | None: + """ + Resolve a requested name. ``kind``: + + - ``group``: the model group a name routes to (was ``routable_model_group``) + - ``alias``: the target of a model_group_alias, or None (was ``get_model_from_alias``) + - ``served``: the name to report for ``served_model`` (was ``resolve_served_model_name``) + - ``from_id``: the model name of a deployment id (was ``resolve_model_name_from_model_id``) + - ``team_model``: a team's public name to its group (was ``map_team_model``; ``team_id`` required) + - ``recognized``: whether the router knows the name (was ``is_recognized_model``) + - ``upstream``: the upstream model a name routes to (was ``get_routable_upstream_model``) + - ``group_of_id``: the group of a deployment id (was ``get_model_group``) + - ``wildcard_access_group``: whether an access group grants a wildcard route + (was ``is_model_access_group_for_wildcard_route``) + - ``team_model_alternatives``: whether a team deployment has siblings to fall back to + (was ``team_model_has_alternatives``) + - ``routing_group_alternatives``: (was ``routing_group_has_alternatives``) + - ``search_tool_support``: per routable deployment, whether it answers Codex's client + ``tool_search`` (was ``get_search_tool_support``) + """ + r: Final = self._router + if kind == "from_id": + return r._core_resolve_model_name_from_model_id(name, custom_llm_provider=custom_llm_provider) + if kind == "team_model": + if team_id is None: + raise TypeError("registry().resolve(kind='team_model') needs a team_id") + return r._core_map_team_model(name, team_id, custom_llm_provider=custom_llm_provider) + if kind == "routing_group_alternatives": + return r._core_routing_group_has_alternatives(name) + if name is None: + raise TypeError(f"registry().resolve(kind={kind!r}) needs a name") + if kind == "group": + return r._core_routable_model_group(name) + if kind == "alias": + return r._core_get_model_from_alias(model=name) + if kind == "served": + # Without a served_model the name is reported as requested, as the core does for a non-match. + return name if served_model is None else r._core_resolve_served_model_name(name, served_model) + if kind == "recognized": + return r._core_is_recognized_model(name) + if kind == "upstream": + return r._core_get_routable_upstream_model(name, team_id=team_id) + if kind == "search_tool_support": + return r._core_get_search_tool_support(name, team_id=team_id) + if kind == "group_of_id": + return r._core_get_model_group(id=name) + if kind == "wildcard_access_group": + return r._core_is_model_access_group_for_wildcard_route(model_access_group=name) + if kind == "team_model_alternatives": + return r._core_team_model_has_alternatives(name) + raise ValueError(f"unknown resolve kind {kind!r}") + + def aliases( + self, + ) -> "dict[str, str | RouterModelGroupAliasItem]": # mutable-ok: the model_group_alias attribute, the live dict + """model_group_alias as configured (was the ``model_group_alias`` attribute).""" + return self._router._core_model_group_alias + + @overload + def patterns(self, *, team: Literal[False] = False) -> "PatternMatchRouter": ... + + @overload + def patterns(self, *, team: Literal[True]) -> "dict[str, PatternMatchRouter]": ... # mutable-ok: the live dict + + @overload + def patterns(self, *, team: bool) -> "PatternMatchRouter | dict[str, PatternMatchRouter]": ... # mutable-ok: same + + def patterns(self, *, team: bool = False) -> "PatternMatchRouter | Mapping[str, PatternMatchRouter]": + """The wildcard pattern router, or the per-team ones (was ``pattern_router`` / ``team_pattern_routers``).""" + return self._router._core_team_pattern_routers if team else self._router._core_pattern_router + + # -- groups --------------------------------------------------------------------------------- + + @overload + def group( + self, + name: str, + *, + field: Literal["info", "cached_info"] = "info", + include_hidden: bool = False, + team_id: str | None = None, + ) -> "ModelGroupInfo | None": ... + + @overload + def group( + self, + name: str, + *, + field: Literal["display_name", "mode"], + include_hidden: bool = False, + team_id: str | None = None, + ) -> str | None: ... + + @overload + def group( + self, + name: str, + *, + field: Literal["service_tiers"], + include_hidden: bool = False, + team_id: str | None = None, + ) -> tuple[object, ...]: ... + + @overload + def group( + self, + name: str, + *, + field: Literal["token_limits"], + include_hidden: bool = False, + team_id: str | None = None, + ) -> tuple[int | None, int | None]: ... + + @overload + def group( + self, + name: str, + *, + field: Literal["listing_info"], + include_hidden: bool = False, + team_id: str | None = None, + ) -> "DeploymentModelListingInfo | None": ... + + def group( + self, + name: str, + *, + field: GroupField = "info", + include_hidden: bool = False, + team_id: str | None = None, + ) -> ( + "ModelGroupInfo | DeploymentModelListingInfo | tuple[int | None, int | None] | tuple[object, ...] | str | None" + ): + """ + What the router knows about a model group. ``field``: + + - ``info``: the group's merged model info (was ``get_model_group_info``) + - ``cached_info``: (was ``cached_model_group_info``) + - ``display_name`` / ``mode`` / ``token_limits`` / ``listing_info``: configured values + (were ``get_configured_display_name``, ``get_configured_mode``, + ``get_configured_token_limits``, ``get_model_listing_info``) + - ``service_tiers``: (was ``get_configured_service_tiers``) + """ + r: Final = self._router + if field == "info": + return r._core_get_model_group_info(name, include_hidden=include_hidden, team_id=team_id) + if field == "cached_info": + return r._core_cached_model_group_info(name) + if field == "display_name": + return r._core_get_configured_display_name(name) + if field == "mode": + return r._core_get_configured_mode(name) + if field == "service_tiers": + return r._core_get_configured_service_tiers(name, team_id=team_id) + if field == "token_limits": + return r._core_get_configured_token_limits(name) + if field == "listing_info": + return r._core_get_model_listing_info(name) + raise ValueError(f"unknown group field {field!r}") + + def routing_groups(self) -> "dict[str, RoutingGroup]": # mutable-ok: the routing_groups attribute, the live dict + """Routing groups by name (was the ``routing_groups`` attribute).""" + return self._router._core_routing_groups + + def access_groups( + self, + model_name: str | None = None, + model_access_group: str | None = None, + team_id: str | None = None, + ) -> dict[str, list[str]]: # mutable-ok: Router._core_get_model_access_groups's type, a fresh dict + """Access group name to model names (was ``get_model_access_groups``).""" + return self._router._core_get_model_access_groups( + model_name=model_name, model_access_group=model_access_group, team_id=team_id + ) + + @overload + def credentials( + self, id: str, *, form: Literal["params"] = "params", team_id: str | None = None + ) -> ( + "UntypedDict | None" + ): ... # a mutable list/dict, as before: Router._core_get_deployment_credentials's type, unchanged + + @overload + def credentials( + self, id: str, *, form: Literal["with_provider"], team_id: str | None = None + ) -> "dict[str, Any] | None": ... # pyright: ignore[reportExplicitAny] # the core's type, unchanged # mutable-ok: same + + @overload + def credentials( + self, id: str, *, form: Literal["deployment"], team_id: str | None = None + ) -> "Deployment | None": ... + + def credentials( + self, id: str, *, form: CredentialForm = "params", team_id: str | None = None + ) -> "Mapping[str, object] | Deployment | None": + """ + A deployment's credentials. ``form``: ``params`` (was ``get_deployment_credentials``), + ``with_provider`` (was ``get_deployment_credentials_with_provider``), ``deployment``, the + deployment that owns the credential (was ``get_credential_deployment``). + """ + r: Final = self._router + if form == "params": + return r._core_get_deployment_credentials(model_id=id) + if form == "with_provider": + return r._core_get_deployment_credentials_with_provider(model_id=id, team_id=team_id) + if form == "deployment": + return r._core_get_credential_deployment(model_id=id, team_id=team_id) + raise ValueError(f"unknown credential form {form!r}") + + @overload + def plugins( + self, kind: Literal["auto"] + ) -> "dict[str, list[TaggedPreRoutingStrategy[AutoRouter]]]": ... # mutable-ok: the live dict + + @overload + def plugins( + self, kind: Literal["complexity"] + ) -> "dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]]": ... # mutable-ok: the live dict + + @overload + def plugins( + self, kind: Literal["quality"] + ) -> "dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]]": ... # mutable-ok: the live dict + + @overload + def plugins( + self, kind: Literal["adaptive"] + ) -> "dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]]": ... # mutable-ok: the live dict + + @overload + def plugins(self, kind: Literal["routing"]) -> "list[RoutingPlugin]": ... # mutable-ok: the live list + + @overload + def plugins(self, kind: Literal["budget_limiter"]) -> "RouterBudgetLimiting | None": ... + + @overload + def plugins(self, kind: Literal["prompt_caching_pins"]) -> "DualCache": ... + + def plugins(self, kind: PluginKind) -> "RegistryPlugins": + """ + Plugins configured by the deployments: ``auto``, ``complexity``, ``quality`` and ``adaptive`` + sub-routers by deployment name, ``routing`` plugins, the deployment ``budget_limiter`` and + the ``prompt_caching_pins`` store (were ``auto_routers``, ``complexity_routers``, + ``quality_routers``, ``adaptive_routers``, ``routing_plugins``, + ``get_router_deployment_budget_limiter`` and ``prompt_caching_pin_cache``). + """ + r: Final = self._router + if kind == "auto": + return r._core_auto_routers + if kind == "complexity": + return r._core_complexity_routers + if kind == "quality": + return r._core_quality_routers + if kind == "adaptive": + return r._core_adaptive_routers + if kind == "routing": + return r._core_routing_plugins + if kind == "budget_limiter": + return r._core_get_router_deployment_budget_limiter() + if kind == "prompt_caching_pins": + return r._core_prompt_caching_pin_cache + raise ValueError(f"unknown plugin kind {kind!r}") + + +# Settings a caller may read. Assigning one from outside the router goes through apply(ChangeSet(assign=...)). +SETTING_FIELDS: Final[tuple[str, ...]] = ( + "routing_strategy", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "default_deployment", + "default_litellm_params", + "default_max_parallel_requests", + "cooldown_time", + "allowed_fails", + "allowed_fails_policy", + "disable_cooldowns", + "enable_pre_call_checks", + "enable_health_check_routing", + "health_check_ignore_transient_errors", + "background_health_check_model_groups", + "enable_tag_filtering", + "tag_filtering_match_any", + "tag_routing_prefix", + "router_general_settings", + "provider_budget_config", + "provider_default_deployment_ids", + "fallback_access_check", + "fallback_budget_check", + "kubernetes_pod_discovery", + "search_tools", + "guardrail_list", + "cache_responses", +) + +_MISSING: Final = object() + + +@dataclass(frozen=True) +class RouterSettings: + """ + The router's configuration as one frozen snapshot (``settings()``). Fields hold the router's + own objects, not copies: mutating a list in here mutates the router's. + A field the router never set reads as ``None``. + + Each field has the type of the Router attribute it reads (``_core_``), the type the old + ``router.`` attribute had; the mutable ones are the router's live objects. ``of`` fills + every field. The ``None`` defaults exist for snapshots built directly (tests), which set only the + fields they need; the fields whose Router attribute cannot be ``None`` say so on their line. + """ + + routing_strategy: str | None = None + fallbacks: "list | list[dict[str, list[str] | list | None]] | None" = None # mutable-ok: the router's live list + context_window_fallbacks: list | None = None # mutable-ok: the router's live list + content_policy_fallbacks: list | None = None # mutable-ok: the router's live list + default_deployment: object = None + default_litellm_params: dict = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) # mutable-ok: the router's live dict + default_max_parallel_requests: int | None = None + cooldown_time: float = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) + allowed_fails: int = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) + allowed_fails_policy: "AllowedFailsPolicy | None" = None + disable_cooldowns: bool | None = None + enable_pre_call_checks: bool = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) + enable_health_check_routing: bool = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) + health_check_ignore_transient_errors: bool = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) + background_health_check_model_groups: frozenset[str] | None = None + enable_tag_filtering: bool = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) + tag_filtering_match_any: bool = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) + tag_routing_prefix: str = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) + router_general_settings: "RouterGeneralSettings" = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) + provider_budget_config: "dict[str, BudgetConfig] | None" = None # mutable-ok: the router's live dict + provider_default_deployment_ids: list[str] = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) # mutable-ok: the router's live list + fallback_access_check: "FallbackAccessCheck | None" = None + fallback_budget_check: "FallbackBudgetCheck | None" = None + kubernetes_pod_discovery: "KubernetesPodDiscovery" = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) + search_tools: "list[SearchToolTypedDict]" = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) # mutable-ok: the router's live list + guardrail_list: "list[GuardrailTypedDict]" = None # pyright: ignore[reportAssignmentType] # unset only when built directly (see above) # mutable-ok: the router's live list + cache_responses: bool | None = None + _router: "Router | None" = field(default=None, repr=False, compare=False) + _unset: frozenset[str] = field(default=frozenset(), repr=False, compare=False) + + @classmethod + def of(cls, router: "Router") -> "RouterSettings": + read: Final = {name: getattr(router, "_core_" + name, _MISSING) for name in SETTING_FIELDS} + values: Final = {name: None if value is _MISSING else value for name, value in read.items()} + unset: Final = frozenset(name for name, value in read.items() if value is _MISSING) + # Each value is the Router's own ``_core_`` attribute, typed as the field (or None when unset). + return cls(**values, _router=router, _unset=unset) # pyright: ignore[reportArgumentType] # see above + + def _owner(self, member: str) -> "Router": + if self._router is None: + raise AttributeError(f"'NoneType' object has no attribute {member!r}") + return self._router + + def is_set(self, name: str) -> bool: + """Whether the router has this setting at all (some exist only in some configurations).""" + return name in SETTING_FIELDS and name not in self._unset + + def as_dict(self) -> Mapping[str, object]: + """The runtime-updatable settings and routing groups as plain values (was ``get_settings``).""" + return self._owner("_core_get_settings")._core_get_settings() + + def allowed_fails_for(self, exception: Exception) -> int | None: + """Allowed fails for this exception under allowed_fails_policy (was ``get_allowed_fails_from_policy``).""" + return self._owner("_core_get_allowed_fails_from_policy")._core_get_allowed_fails_from_policy( + exception=exception + ) + + def guardrail(self, name: str) -> "GuardrailTypedDict": + """The guardrail configured under this name (was ``get_available_guardrail``).""" + return self._owner("_core_get_available_guardrail")._core_get_available_guardrail(name) + + +class RouterHealth: + """Cooldowns, blocked deployments, health checks and usage (``health()``).""" + + __slots__ = ("_router",) + + def __init__(self, router: "Router") -> None: + self._router = router + + @property + def store(self) -> "DualCache": + """ + The cache that holds routing state (cooldowns, fail counts, usage counters, latency): + in-memory plus the router's Redis when configured. Read and write it with the router's own + key names. Replaces the ``cache`` attribute. + """ + return self._router._core_cache + + @property + def cooldowns(self) -> "CooldownCache": + """The cooldown store (was ``cooldown_cache``).""" + return self._router._core_cooldown_cache + + @property + def health_checks(self) -> "DeploymentHealthCache": + """Background health check results (was ``health_state_cache``).""" + return self._router._core_health_state_cache + + @property + def latency( + self, + ) -> "UntypedDict": # a mutable list/dict, as before: the deployment_latency_map attribute's type, the live dict + """Latency samples by deployment (was ``deployment_latency_map``).""" + return self._router._core_deployment_latency_map + + @staticmethod + def is_blocked(deployment: "Deployment") -> bool: + """Whether an admin paused this deployment (was ``is_deployment_blocked``).""" + from litellm.router import Router + + return Router._core_is_deployment_blocked(deployment) + + @staticmethod + def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool: + """Whether every deployment in the list is paused (was ``are_all_deployments_blocked``).""" + from litellm.router import Router + + return Router._core_are_all_deployments_blocked(deployments) + + def fully_blocked_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set + """Groups whose every deployment is paused (was ``get_fully_blocked_model_names``).""" + return self._router._core_get_fully_blocked_model_names() + + async def fully_unhealthy_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set + """Groups whose every deployment failed its health check (was ``async_get_fully_unhealthy_model_names``).""" + return await self._router._core_async_get_fully_unhealthy_model_names() + + @overload + async def usage( + self, model_group: str, *, kind: Literal["current"] = "current" + ) -> tuple[int | None, int | None]: ... + + @overload + async def usage( + self, model_group: str, *, kind: Literal["remaining"] + ) -> dict[str, int]: ... # mutable-ok: Router._core_get_remaining_model_group_usage's type, a fresh dict + + async def usage( + self, model_group: str, *, kind: UsageKind = "current" + ) -> tuple[int | None, int | None] | Mapping[str, int]: + """ + tpm/rpm usage of a group this minute (``current``, was ``get_model_group_usage``) or what is + left of its limits (``remaining``, was ``get_remaining_model_group_usage``). + """ + if kind == "current": + return await self._router._core_get_model_group_usage(model_group=model_group) + if kind == "remaining": + return await self._router._core_get_remaining_model_group_usage(model_group=model_group) + raise ValueError(f"unknown usage kind {kind!r}") + + def team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None": + """A team's PTU share ceiling on a group (was ``team_ptu_ceiling``).""" + return self._router._core_team_ptu_ceiling(team_id, model_group) + + def split_priority(self, model: str, kwargs: Mapping[str, object]) -> tuple[int | None, Mapping[str, object]]: + """The request priority and the kwargs without it (was ``split_request_priority``).""" + return self._router._core_split_request_priority(model, kwargs) + + +@dataclass +class ChangeSet: + """ + Runtime changes for ``apply``. Applied in this order: ``assign``, ``settings``, ``upsert``, + ``delete``, ``redis_cache``. + + - ``assign``: replace settings fields as they are (what ``router. = value`` did) + - ``settings``: validated runtime settings (what ``update_settings(**settings)`` did) + - ``upsert``: add or replace deployments (``upsert_deployment``) + - ``delete``: remove deployments by id (``delete_deployment``) + - ``redis_cache``: point the routing state at a new Redis cache (``update_redis_cache``) + - ``refresh_model_info``: re-read discovered model info (``arefresh_model_info``; ``aapply`` only) + """ + + assign: Mapping[str, object] = field(default_factory=dict) + settings: Mapping[str, object] = field(default_factory=dict) + upsert: Sequence["Deployment"] = () + delete: Sequence[str] = () + redis_cache: "RedisCache | None" = None + refresh_model_info: bool = False + refresh_client: "AsyncHTTPHandler | None" = None + + +@dataclass +class ApplyResult: + """What ``apply`` did: the registry version after it, and per upsert/delete the old deployment.""" + + version: int + upserted: "Sequence[Deployment | None]" = field(default_factory=list) + deleted: "Sequence[Deployment | None]" = field(default_factory=list) + + +@dataclass(frozen=True) +class RoutingObservers: + """The router's routing observers (``observe()``).""" + + slack_alerting_logger: "SlackAlerting | None" = None + + +def apply_changes(router: "Router", changes: ChangeSet) -> ApplyResult: + if changes.refresh_model_info: + raise TypeError("ChangeSet(refresh_model_info=True) needs aapply()") + unknown: Final = sorted(set(changes.assign) - set(SETTING_FIELDS)) + if unknown: + raise ValueError(f"not assignable router settings: {unknown}") + for name, value in changes.assign.items(): + setattr(router, "_core_" + name, value) + if changes.settings: + router._core_update_settings(**changes.settings) + upserted: Final = [router._core_upsert_deployment(deployment=d) for d in changes.upsert] + deleted: Final = [router._core_delete_deployment(id=i) for i in changes.delete] + if changes.redis_cache is not None: + router._core_update_redis_cache(cache=changes.redis_cache) + return ApplyResult(version=router._registry_version, upserted=upserted, deleted=deleted) + + +async def aapply_changes(router: "Router", changes: ChangeSet) -> ApplyResult: + refresh: Final = changes.refresh_model_info + result: Final = apply_changes( + router, + ChangeSet( + assign=changes.assign, + settings=changes.settings, + upsert=changes.upsert, + delete=changes.delete, + redis_cache=changes.redis_cache, + ), + ) + if refresh: + await router._core_arefresh_model_info(client=changes.refresh_client) + return result + + +class RoutingContext: + """ + What a routing plugin (strategy, pre-call check, sub-router) may use of the router. Plugins get + one instead of the Router itself. + """ + + __slots__ = ("_router",) + + def __init__(self, router: "Router") -> None: + self._router = router + + def registry(self) -> RouterRegistry: + return self._router.registry() + + def settings(self) -> RouterSettings: + return self._router.settings() + + def health(self) -> RouterHealth: + return self._router.health() + + def pre_call_checks( + self, + model: str, + healthy_deployments: list[object], # mutable-ok: Router._pre_call_checks takes and filters a list + messages: list[dict[str, str]] | None = None, # mutable-ok: Router._pre_call_checks takes a list + input: str | list[object] | None = None, # mutable-ok: Router._pre_call_checks takes a list + request_kwargs: dict[str, object] | None = None, # mutable-ok: Router._pre_call_checks takes a dict + input_token_count: int | None = None, + skip_inline_token_count: bool = False, + request_deployments: "Sequence[DeploymentTypedDict] | None" = None, + ) -> Sequence[object]: + return self._router._pre_call_checks( + model=model, + healthy_deployments=healthy_deployments, + messages=messages, + input=input, + request_kwargs=request_kwargs, + input_token_count=input_token_count, + skip_inline_token_count=skip_inline_token_count, + request_deployments=request_deployments, + ) + + def count_pre_call_check_tokens( + self, + messages: Sequence[Mapping[str, object]] | None, + input: str | list[object] | None, # mutable-ok: Router._count_pre_call_check_tokens takes a list + request_kwargs: Mapping[str, object] | None = None, + ) -> int: + return self._router._count_pre_call_check_tokens(messages, input, request_kwargs) + + def update_kwargs_before_fallbacks( + self, + model: str, + kwargs: dict[str, object], # mutable-ok: the core updates the request kwargs in place + metadata_variable_name: str | None = "metadata", + ) -> None: + return self._router._update_kwargs_before_fallbacks(model, kwargs, metadata_variable_name) + + def arm_read_prefetch( + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the core takes a dict + ) -> None: + """(was ``arm_routing_read_prefetch``)""" + return self._router._core_arm_routing_read_prefetch(model, request_kwargs) + + async def pre_routing_hook( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: Router._core_async_pre_routing_hook takes a dict + messages: list[dict[str, object]] | None = None, # mutable-ok: the core takes a list + input: str | list[object] | None = None, # mutable-ok: Router._core_async_pre_routing_hook takes a list + specific_deployment: bool | None = False, + ) -> "PreRoutingHookResponse | None": + """(was ``async_pre_routing_hook``)""" + return await self._router._core_async_pre_routing_hook( + model=model, + request_kwargs=request_kwargs, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) + + def log_retry( + self, + kwargs: dict[str, object], # mutable-ok: Router._core_log_retry records the retry into the request kwargs + e: Exception, + ) -> Mapping[str, object]: + return self._router._core_log_retry(kwargs=kwargs, e=e) + + +__all__ = [ + "SETTING_FIELDS", + "ApplyResult", + "ChangeSet", + "RouterHealth", + "RouterRegistry", + "RouterSettings", + "RoutingContext", + "RoutingObservers", + "deployment_id", +] diff --git a/litellm/router_deprecated.py b/litellm/router_deprecated.py new file mode 100644 index 00000000000..046fb8c452a --- /dev/null +++ b/litellm/router_deprecated.py @@ -0,0 +1,852 @@ +""" +DeprecatedRouterMembers: the Router members that outside code used before the entry points. + +``litellm.Router`` mixes this class in so existing callers keep working while they move to the +entry points (``registry()``, ``settings()``, ``health()``, ``select()``, ``apply()``, +``observe()``, ``register_plugin()``, ``retire()``; see litellm/router_api.py). Every member here +returns what it returned before, with the type it had before, and names its replacement in a +``DeprecationWarning``. Reads go to the same Router implementation (``Router._core_``) the +entry point uses; writes go through ``apply`` / ``aapply`` so the registry version moves. + +Access from the router itself and from its own modules (``litellm.router*``) is not counted: +those are the router's internals and move behind the core interface separately. Every other access +is counted in ``DEPRECATED_ROUTER_MEMBER_USES`` so the remaining users show up before this class is +deleted (planned about a month after the core swap). +""" + +# pyright: reportPrivateUsage=false, reportMissingTypeArgument=false, reportUnknownParameterType=false +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# pyright: reportUnknownLambdaType=false +# This shim is part of the router (it reads Router's ``_core_*`` members, as Router's own methods do) +# and repeats each old member's signature exactly, untyped parts included (``dict``, ``list``), so +# callers keep the types they had. Typing those parts is the core's job, not the shim's. + +import contextvars +import functools +import inspect +import sys +import warnings +from collections import Counter +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload + +from litellm.router_api import SETTING_FIELDS, ChangeSet + +if TYPE_CHECKING: + from typing import Any # noqa: TID251 # only to repeat two old signatures that had Any, unchanged + + from opentelemetry.trace import Span + + from litellm.caching.caching import RedisCache + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.router import Router + from litellm.router_strategy.budget_limiter import RouterBudgetLimiting + from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery + from litellm.router_utils.ptu_shares import PTUTeamCeiling + from litellm.types.router import ( + CustomRoutingStrategyBase, + Deployment, + DeploymentModelListingInfo, + DeploymentTypedDict, + GuardrailTypedDict, + ModelGroupInfo, + PreRoutingHookResponse, + SearchToolTypedDict, + ) + from litellm.types.utils import ModelInfo as ModelMapInfo + +_T: Final = TypeVar("_T") + +DEPRECATED_ROUTER_MEMBER_USES: Final[Counter[str]] = Counter() # mutable-ok: the live tally _note increments + +_INTERNAL_PREFIX: Final = "litellm.router" + + +# Router's own attribute hooks: a write or delete through an old name passes through them first. +_ATTRIBUTE_HOOKS: Final = frozenset({"__setattr__", "__delattr__"}) + + +def _note(name: str, replacement: str) -> None: + depth = 2 # rebind-ok: grows by one per attribute hook frame below + caller = sys._getframe(depth) # rebind-ok: follows depth up past those hooks + while caller.f_code.co_name in _ATTRIBUTE_HOOKS and caller.f_globals.get("__name__") == "litellm.router": + depth += 1 + caller = sys._getframe(depth) + module: Final = caller.f_globals.get("__name__") + if isinstance(module, str) and module.startswith(_INTERNAL_PREFIX): + return + DEPRECATED_ROUTER_MEMBER_USES[name] += 1 + warnings.warn( + f"Router.{name} is deprecated; use Router.{replacement}", + DeprecationWarning, + stacklevel=depth + 1, + ) + + +def _as_router(obj: "DeprecatedRouterMembers") -> "Router": + """The Router this mixin is part of (it is mixed into ``litellm.Router`` only).""" + from litellm.router import Router + + if isinstance(obj, Router): + return obj + raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only") + + +def _public_attribute_error(obj: object, core_name: str, public_name: str, error: AttributeError) -> AttributeError: + """ + ``error`` renamed to the public attribute when it is the router's own ``core_name`` that is + missing, so the message is the one reading the old attribute raised; any other error unchanged. + """ + if error.name not in (core_name, public_name) or error.obj is not obj: + return error + return AttributeError(f"'{type(obj).__name__}' object has no attribute '{public_name}'", name=public_name, obj=obj) + + +class _DeprecatedAttribute(Generic[_T]): + """A former Router attribute, read through an entry point. Assignment still replaces it.""" + + def __init__(self, replacement: str, read: Callable[["Router"], _T]) -> None: + self.replacement = replacement + self.read = read + self.name = "" + + def __set_name__(self, owner: type, name: str) -> None: + # An alias (``_routing_groups = routing_groups``) keeps the first name. + self.name = self.name or name + + @overload + def __get__(self, obj: None, owner: type) -> object: ... + + @overload + def __get__(self, obj: "DeprecatedRouterMembers", owner: type) -> _T: ... + + def __get__(self, obj: "DeprecatedRouterMembers | None", owner: type) -> object: + if obj is None: + return getattr(owner, "_core_" + self.name, self) + _note(self.name, self.replacement) + router: Final = _as_router(obj) + try: + return self.read(router) + except AttributeError as e: + raise _public_attribute_error(router, "_core_" + self.name, self.name, e) from None + + def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None: + _note(self.name, self.replacement) + if self.name in SETTING_FIELDS: + _as_router(obj).apply(ChangeSet(assign={self.name: value})) + else: + setattr(obj, "_core_" + self.name, value) + + def __delete__(self, obj: "DeprecatedRouterMembers") -> None: + _note(self.name, self.replacement) + try: + delattr(obj, "_core_" + self.name) + except AttributeError: + # The only attribute this deletes; Python's own message for it, which sets no name/obj. + raise AttributeError(f"'{type(obj).__name__}' object has no attribute '{self.name}'") from None + + +def _setting(name: str, read: Callable[["Router"], _T]) -> _DeprecatedAttribute[_T]: + """A former settings attribute: ``read`` returns the router's own value, typed as before.""" + + def read_setting(router: "Router") -> _T: + if not router.settings().is_set(name): + raise AttributeError(f"'{type(router).__name__}' object has no attribute {name!r}", name=name, obj=router) + return read(router) + + return _DeprecatedAttribute(f"settings().{name}", read_setting) + + +def _attr(replacement: str, read: Callable[["Router"], _T]) -> _DeprecatedAttribute[_T]: + return _DeprecatedAttribute(replacement, read) + + +# (id(router), member name) pairs whose subclass override is running in this context. +_OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] = contextvars.ContextVar( + "router_overrides_running", default=frozenset() +) + + +def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., object]: + """ + A direct call of a subclass override (``router.()``) marks it running, so its + ``super().()`` reaches the parent once instead of re-entering the override through + ``_core_``. + """ + if inspect.iscoroutinefunction(override): + async_override: Final[Callable[..., Awaitable[object]]] = override + + @functools.wraps(override) + async def entered_async( + router: object, + *args: object, + **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged + ) -> object: + token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)}) + try: + return await async_override(router, *args, **kwargs) + finally: + _OVERRIDES_RUNNING.reset(token) + + return entered_async + + @functools.wraps(override) + def entered( + router: object, + *args: object, + **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged + ) -> object: + token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)}) + try: + return override(router, *args, **kwargs) + finally: + _OVERRIDES_RUNNING.reset(token) + + return entered + + +def _private_alias(name: str, member: Callable[..., object]) -> Callable[..., object]: + """ + A private alias of an old member (``_get_all_deployments``). It was bound to Router's own + method, so it never reached a subclass override of ``name``, and still doesn't. + """ + return _enter_override(name, member) + + +def _route_override(cls: type, name: str, override: Callable[..., object]) -> Callable[..., object]: + """ + Make the router's own calls to ``_core_`` reach a subclass override of ````. + + When the override calls ``super().()``, the call comes back to ``_core_``; it is + then sent on to the parent's implementation instead of back into the override. + """ + core: Final = "_core_" + name + + def parent(router: object) -> Callable[..., object]: + return getattr(super(cls, router), core) + + if inspect.iscoroutinefunction(override): + async_override: Final[Callable[..., Awaitable[object]]] = override + + @functools.wraps(override) + async def routed_async( + router: object, + *args: object, + **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged + ) -> object: + key: Final = (id(router), name) + running: Final = _OVERRIDES_RUNNING.get() + if key in running: + parent_async: Final[Callable[..., Awaitable[object]]] = getattr(super(cls, router), core) + return await parent_async(*args, **kwargs) + token: Final = _OVERRIDES_RUNNING.set(running | {key}) + try: + return await async_override(router, *args, **kwargs) + finally: + _OVERRIDES_RUNNING.reset(token) + + return routed_async + + @functools.wraps(override) + def routed( + router: object, + *args: object, + **kwargs: object, # kwargs-ok: passes the override's arguments through unchanged + ) -> object: + key: Final = (id(router), name) + running: Final = _OVERRIDES_RUNNING.get() + if key in running: + return parent(router)(*args, **kwargs) + token: Final = _OVERRIDES_RUNNING.set(running | {key}) + try: + return override(router, *args, **kwargs) + finally: + _OVERRIDES_RUNNING.reset(token) + + return routed + + +class DeprecatedRouterMembers: + """Old Router members, kept for about a month. Each names its replacement.""" + + def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: class keywords, passed on to super() + # A Router subclass that overrides an old member (say async_pre_routing_hook) keeps + # having its override called by the router, which now calls _core_ internally. + super().__init_subclass__(**kwargs) + for name, value in list(vars(cls).items()): + if ( + name.startswith("_core_") + or name not in vars(DeprecatedRouterMembers) + or not hasattr(cls, "_core_" + name) + ): + continue + if isinstance(value, staticmethod): + # Its super() call reaches the old member here, which calls Router's own implementation. + setattr(cls, "_core_" + name, value) + elif inspect.isfunction(value): + setattr(cls, "_core_" + name, _route_override(cls, name, value)) + setattr(cls, name, _enter_override(name, value)) + + # -- registry: attributes -------------------------------------------------------------------- + model_list = _attr("registry().all()", lambda r: r._core_model_list) + model_names = _attr("registry().names(kind='group_set')", lambda r: r._core_model_names) + deployment_names = _attr("registry().names(kind='deployment_names')", lambda r: r._core_deployment_names) + team_public_model_names = _attr("registry().names(kind='team_public')", lambda r: r._core_team_public_model_names) + model_name_to_deployment_indices = _attr( + "registry().index('model_name')", lambda r: r._core_model_name_to_deployment_indices + ) + team_model_to_deployment_indices = _attr( + "registry().index('team_model')", lambda r: r._core_team_model_to_deployment_indices + ) + cached_deployment_model_info = _attr( + "registry().deployment_model_info(id, model_name, cached=True)", + lambda r: r._core_cached_deployment_model_info, + ) + model_group_alias = _attr("registry().aliases()", lambda r: r._core_model_group_alias) + pattern_router = _attr("registry().patterns()", lambda r: r._core_pattern_router) + team_pattern_routers = _attr("registry().patterns(team=True)", lambda r: r._core_team_pattern_routers) + routing_groups = _attr("registry().routing_groups()", lambda r: r._core_routing_groups) + _routing_groups = routing_groups + auto_routers = _attr("registry().plugins('auto')", lambda r: r._core_auto_routers) + complexity_routers = _attr("registry().plugins('complexity')", lambda r: r._core_complexity_routers) + quality_routers = _attr("registry().plugins('quality')", lambda r: r._core_quality_routers) + adaptive_routers = _attr("registry().plugins('adaptive')", lambda r: r._core_adaptive_routers) + routing_plugins = _attr("registry().plugins('routing')", lambda r: r._core_routing_plugins) + prompt_caching_pin_cache = _attr( + "registry().plugins('prompt_caching_pins')", lambda r: r._core_prompt_caching_pin_cache + ) + + # -- health and selection: attributes -------------------------------------------------------- + cache = _attr("health().store", lambda r: r._core_cache) + cooldown_cache = _attr("health().cooldowns", lambda r: r._core_cooldown_cache) + health_state_cache = _attr("health().health_checks", lambda r: r._core_health_state_cache) + deployment_latency_map = _attr("health().latency", lambda r: r._core_deployment_latency_map) + async_get_available_deployment = _attr( + "select(model, request_kwargs)", + lambda r: r._core_async_get_available_deployment, + ) + + # -- observers ------------------------------------------------------------------------------ + slack_alerting_logger = _attr("observe().slack_alerting_logger", lambda r: r._core_slack_alerting_logger) + + # -- settings ------------------------------------------------------------------------------- + routing_strategy = _setting("routing_strategy", lambda r: r._core_routing_strategy) + context_window_fallbacks = _setting("context_window_fallbacks", lambda r: r._core_context_window_fallbacks) + content_policy_fallbacks = _setting("content_policy_fallbacks", lambda r: r._core_content_policy_fallbacks) + default_deployment = _setting("default_deployment", lambda r: r._core_default_deployment) + default_litellm_params = _setting("default_litellm_params", lambda r: r._core_default_litellm_params) + default_max_parallel_requests = _setting( + "default_max_parallel_requests", lambda r: r._core_default_max_parallel_requests + ) + cooldown_time = _setting("cooldown_time", lambda r: r._core_cooldown_time) + allowed_fails = _setting("allowed_fails", lambda r: r._core_allowed_fails) + allowed_fails_policy = _setting("allowed_fails_policy", lambda r: r._core_allowed_fails_policy) + disable_cooldowns = _setting("disable_cooldowns", lambda r: r._core_disable_cooldowns) + enable_pre_call_checks = _setting("enable_pre_call_checks", lambda r: r._core_enable_pre_call_checks) + enable_health_check_routing = _setting("enable_health_check_routing", lambda r: r._core_enable_health_check_routing) + health_check_ignore_transient_errors = _setting( + "health_check_ignore_transient_errors", lambda r: r._core_health_check_ignore_transient_errors + ) + background_health_check_model_groups = _setting( + "background_health_check_model_groups", lambda r: r._core_background_health_check_model_groups + ) + enable_tag_filtering = _setting("enable_tag_filtering", lambda r: r._core_enable_tag_filtering) + tag_filtering_match_any = _setting("tag_filtering_match_any", lambda r: r._core_tag_filtering_match_any) + tag_routing_prefix = _setting("tag_routing_prefix", lambda r: r._core_tag_routing_prefix) + router_general_settings = _setting("router_general_settings", lambda r: r._core_router_general_settings) + provider_budget_config = _setting("provider_budget_config", lambda r: r._core_provider_budget_config) + provider_default_deployment_ids = _setting( + "provider_default_deployment_ids", lambda r: r._core_provider_default_deployment_ids + ) + fallback_access_check = _setting("fallback_access_check", lambda r: r._core_fallback_access_check) + fallback_budget_check = _setting("fallback_budget_check", lambda r: r._core_fallback_budget_check) + # These three are declared with the attribute type Router gave them rather than the descriptor's: + # protocols in router_utils and the proxy match Router on them, and protocol matching does not + # see through a descriptor. Reads and writes still go through _setting at runtime. + fallbacks: list | list[dict[str, list[str] | list | None]] | None = _setting( # pyright: ignore[reportAssignmentType] # see above # mutable-ok: the old type + "fallbacks", lambda r: r._core_fallbacks + ) + kubernetes_pod_discovery: "KubernetesPodDiscovery" = _setting( # pyright: ignore[reportAssignmentType] # see above + "kubernetes_pod_discovery", lambda r: r._core_kubernetes_pod_discovery + ) + search_tools: "list[SearchToolTypedDict]" = _setting( # pyright: ignore[reportAssignmentType] # see above # mutable-ok: the old type + "search_tools", lambda r: r._core_search_tools + ) + guardrail_list = _setting("guardrail_list", lambda r: r._core_guardrail_list) + cache_responses = _setting("cache_responses", lambda r: r._core_cache_responses) + + # -- registry: one deployment --------------------------------------------------------------- + def get_deployment(self, model_id: str) -> "Deployment | None": + _note("get_deployment", "registry().deployment(id)") + return _as_router(self)._core_get_deployment(model_id=model_id) + + def has_model_id(self, candidate_id: str) -> bool: + _note("has_model_id", "registry().has_deployment(id)") + return _as_router(self)._core_has_model_id(candidate_id) + + def get_model_info(self, id: str) -> dict | None: # mutable-ok: the old return type, unchanged + _note("get_model_info", "registry().deployment_dict(id)") + return _as_router(self)._core_get_model_info(id=id) + + def get_deployment_model_info(self, model_id: str, model_name: str) -> "ModelMapInfo | None": + _note("get_deployment_model_info", "registry().deployment_model_info(id, model_name)") + return _as_router(self)._core_get_deployment_model_info(model_id=model_id, model_name=model_name) + + @overload + def get_router_model_info( + self, + deployment: "dict | Deployment", # mutable-ok: the old parameter type, unchanged + received_model_name: str, + id: None = None, + ) -> "ModelMapInfo": ... + + @overload + def get_router_model_info(self, deployment: None, received_model_name: str, id: str) -> "ModelMapInfo": ... + + def get_router_model_info( + self, + deployment: "dict | Deployment | None", # mutable-ok: the old parameter type, unchanged + received_model_name: str, + id: str | None = None, + ) -> "ModelMapInfo": + _note("get_router_model_info", "registry().router_model_info(...)") + router: Final = _as_router(self) + # The core's overloads split (deployment, id) by which one is None; this forwards either form unchanged. + return router._core_get_router_model_info( # pyright: ignore[reportCallIssue] # see above + deployment=deployment, # pyright: ignore[reportArgumentType] # see above + received_model_name=received_model_name, + id=id, # pyright: ignore[reportArgumentType] # see above + ) + + def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: + _note("get_discovered_model_info", "registry().discovered_model_info(id)") + return _as_router(self)._core_get_discovered_model_info(model_id) + + # -- registry: many deployments ------------------------------------------------------------- + def get_model_list( + self, model_name: str | None = None, team_id: str | None = None + ) -> "list[DeploymentTypedDict] | None": # mutable-ok: the old return type, unchanged + _note("get_model_list", "registry().deployments(model_name, team_id=...)") + return _as_router(self)._core_get_model_list(model_name=model_name, team_id=team_id) + + def get_all_deployments( + self, model_name: str, model_alias: str | None = None, team_id: str | None = None + ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged + _note("get_all_deployments", "registry().deployments(model_name, source='all')") + return _as_router(self)._core_get_all_deployments( + model_name=model_name, model_alias=model_alias, team_id=team_id + ) + + _get_all_deployments = _private_alias("get_all_deployments", get_all_deployments) + + def config_deployments(self) -> Iterator[Mapping[str, object]]: + _note("config_deployments", "registry().deployments(source='config')") + return _as_router(self)._core_config_deployments() + + def deployments_named(self, model_name: str) -> "tuple[DeploymentTypedDict, ...] | None": + _note("deployments_named", "registry().deployments(model_name, source='named')") + return _as_router(self)._core_deployments_named(model_name) + + def get_model_list_from_model_alias( + self, model_name: str | None = None + ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged + _note("get_model_list_from_model_alias", "registry().deployments(model_name, source='alias')") + return _as_router(self)._core_get_model_list_from_model_alias(model_name=model_name) + + def get_model_list_from_routing_groups(self, model_name: str | None = None) -> "Sequence[DeploymentTypedDict]": + _note("get_model_list_from_routing_groups", "registry().deployments(model_name, source='routing_groups')") + return _as_router(self)._core_get_model_list_from_routing_groups(model_name=model_name) + + def get_model_list_of_routed_group( + self, model_group: str + ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged + _note("get_model_list_of_routed_group", "registry().deployments(model_group, source='routed_group')") + return _as_router(self)._core_get_model_list_of_routed_group(model_group) + + def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: + _note("wildcard_route_deployments", "registry().deployments(model, source='wildcard')") + return _as_router(self)._core_wildcard_route_deployments(model, team_id) + + def get_routable_deployments( + self, + model: str, + request_kwargs: Mapping[str, object] = MappingProxyType({}), + call_type: str | None = None, + ) -> "tuple[DeploymentTypedDict, ...]": + _note( + "get_routable_deployments", "registry().deployments(model, source='routable', request=..., call_type=...)" + ) + return _as_router(self)._core_get_routable_deployments(model, request_kwargs, call_type) + + def get_deployment_by_model_group_name(self, model_group_name: str) -> "Deployment | None": + _note("get_deployment_by_model_group_name", "registry().deployments(name, source='first_in_group')") + return _as_router(self)._core_get_deployment_by_model_group_name(model_group_name=model_group_name) + + def deployments_for_request( + self, model: str, request_kwargs: Mapping[str, object] + ) -> "Sequence[DeploymentTypedDict]": + _note("deployments_for_request", "registry().deployments(model, source='for_request', request=...)") + return _as_router(self)._core_deployments_for_request(model, request_kwargs) + + def should_include_deployment( + self, + model_name: str, + model: "DeploymentTypedDict | dict", # mutable-ok: the old parameter type, unchanged + team_id: str | None = None, + ) -> bool: + _note("should_include_deployment", "registry().includes(deployment, model_name=..., team_id=...)") + return _as_router(self)._core_should_include_deployment(model_name=model_name, model=model, team_id=team_id) + + @staticmethod + def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool: + _note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)") + from litellm.router import Router + + return Router._core_deployment_usable_by_team(model, team_id) + + _deployment_usable_by_team = deployment_usable_by_team + + # -- registry: names ------------------------------------------------------------------------- + def get_model_names(self, team_id: str | None = None) -> list[str]: # mutable-ok: the old return type, unchanged + _note("get_model_names", "registry().names(team_id=...)") + return _as_router(self)._core_get_model_names(team_id=team_id) + + def get_model_ids( + self, model_name: str | None = None, exclude_team_models: bool = False + ) -> list[str]: # mutable-ok: the old return type, unchanged + _note("get_model_ids", "registry().names(model_name, kind='ids')") + return _as_router(self)._core_get_model_ids(model_name=model_name, exclude_team_models=exclude_team_models) + + def resolved_litellm_models( + self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False + ) -> tuple[str, ...]: + _note("resolved_litellm_models", "registry().names(model_name, kind='litellm_models' or 'litellm_base_models')") + return _as_router(self)._core_resolved_litellm_models( + model_name, team_id=team_id, use_base_model=use_base_model + ) + + def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: + _note("wildcard_routed_models", "registry().names(model, kind='wildcard')") + return _as_router(self)._core_wildcard_routed_models(model, team_id) + + def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: + _note("get_candidate_model_ids_for_route", "registry().names(model, kind='route_candidates')") + return _as_router(self)._core_get_candidate_model_ids_for_route(model, team_id=team_id) + + # -- registry: resolving names --------------------------------------------------------------- + def routable_model_group(self, model_name: str) -> str: + _note("routable_model_group", "registry().resolve(name)") + return _as_router(self)._core_routable_model_group(model_name) + + def get_model_from_alias(self, model: str) -> str | None: + _note("get_model_from_alias", "registry().resolve(name, kind='alias')") + return _as_router(self)._core_get_model_from_alias(model=model) + + _get_model_from_alias = _private_alias("get_model_from_alias", get_model_from_alias) + + def resolve_served_model_name(self, model_name: str, served_model: str) -> str: + _note("resolve_served_model_name", "registry().resolve(name, kind='served', served_model=...)") + return _as_router(self)._core_resolve_served_model_name(model_name, served_model) + + def resolve_model_name_from_model_id( + self, model_id: str | None, custom_llm_provider: str | None = None + ) -> str | None: + _note("resolve_model_name_from_model_id", "registry().resolve(id, kind='from_id')") + return _as_router(self)._core_resolve_model_name_from_model_id( + model_id, custom_llm_provider=custom_llm_provider + ) + + def map_team_model( + self, team_model_name: str | None, team_id: str, custom_llm_provider: str | None = None + ) -> str | None: + _note("map_team_model", "registry().resolve(name, kind='team_model', team_id=...)") + return _as_router(self)._core_map_team_model(team_model_name, team_id, custom_llm_provider=custom_llm_provider) + + def is_recognized_model(self, model: str) -> bool: + _note("is_recognized_model", "registry().resolve(name, kind='recognized')") + return _as_router(self)._core_is_recognized_model(model) + + def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: + _note("get_routable_upstream_model", "registry().resolve(name, kind='upstream')") + return _as_router(self)._core_get_routable_upstream_model(model_name, team_id=team_id) + + def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: + _note("get_search_tool_support", "registry().resolve(name, kind='search_tool_support')") + return _as_router(self)._core_get_search_tool_support(model_name, team_id=team_id) + + def get_model_group(self, id: str) -> list | None: # mutable-ok: the old return type, unchanged + _note("get_model_group", "registry().resolve(id, kind='group_of_id')") + return _as_router(self)._core_get_model_group(id=id) + + def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: + _note("is_model_access_group_for_wildcard_route", "registry().resolve(group, kind='wildcard_access_group')") + return _as_router(self)._core_is_model_access_group_for_wildcard_route(model_access_group=model_access_group) + + _is_model_access_group_for_wildcard_route = _private_alias( + "is_model_access_group_for_wildcard_route", is_model_access_group_for_wildcard_route + ) + + def team_model_has_alternatives(self, deployment_id: str) -> bool: + _note("team_model_has_alternatives", "registry().resolve(id, kind='team_model_alternatives')") + return _as_router(self)._core_team_model_has_alternatives(deployment_id) + + def routing_group_has_alternatives(self, model_group: str | None) -> bool: + _note("routing_group_has_alternatives", "registry().resolve(group, kind='routing_group_alternatives')") + return _as_router(self)._core_routing_group_has_alternatives(model_group) + + # -- registry: groups, access groups, credentials, plugins ----------------------------------- + def get_model_group_info( + self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None + ) -> "ModelGroupInfo | None": + _note("get_model_group_info", "registry().group(name)") + return _as_router(self)._core_get_model_group_info(model_group, include_hidden=include_hidden, team_id=team_id) + + def cached_model_group_info(self, model_group: str) -> "ModelGroupInfo | None": + _note("cached_model_group_info", "registry().group(name, field='cached_info')") + return _as_router(self)._core_cached_model_group_info(model_group) + + def get_configured_display_name(self, model_name: str) -> str | None: + _note("get_configured_display_name", "registry().group(name, field='display_name')") + return _as_router(self)._core_get_configured_display_name(model_name) + + def get_configured_mode(self, model_name: str) -> str | None: + _note("get_configured_mode", "registry().group(name, field='mode')") + return _as_router(self)._core_get_configured_mode(model_name) + + def get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: + _note("get_configured_service_tiers", "registry().group(name, field='service_tiers')") + return _as_router(self)._core_get_configured_service_tiers(model_name, team_id=team_id) + + def get_configured_token_limits(self, model_name: str) -> tuple[int | None, int | None]: + _note("get_configured_token_limits", "registry().group(name, field='token_limits')") + return _as_router(self)._core_get_configured_token_limits(model_name) + + def get_model_listing_info(self, model_name: str) -> "DeploymentModelListingInfo | None": + _note("get_model_listing_info", "registry().group(name, field='listing_info')") + return _as_router(self)._core_get_model_listing_info(model_name) + + def get_model_access_groups( + self, + model_name: str | None = None, + model_access_group: str | None = None, + team_id: str | None = None, + ) -> dict[str, list[str]]: # mutable-ok: the old return type, unchanged + _note("get_model_access_groups", "registry().access_groups(...)") + return _as_router(self)._core_get_model_access_groups( + model_name=model_name, model_access_group=model_access_group, team_id=team_id + ) + + def get_deployment_credentials(self, model_id: str) -> dict | None: # mutable-ok: the old return type, unchanged + _note("get_deployment_credentials", "registry().credentials(id)") + return _as_router(self)._core_get_deployment_credentials(model_id=model_id) + + def get_deployment_credentials_with_provider( + self, model_id: str, team_id: str | None = None + ) -> "dict[str, Any] | None": # pyright: ignore[reportExplicitAny] # the old return type, unchanged # mutable-ok: same + _note("get_deployment_credentials_with_provider", "registry().credentials(id, form='with_provider')") + return _as_router(self)._core_get_deployment_credentials_with_provider(model_id=model_id, team_id=team_id) + + def get_credential_deployment(self, model_id: str, team_id: str | None = None) -> "Deployment | None": + _note("get_credential_deployment", "registry().credentials(id, form='deployment')") + return _as_router(self)._core_get_credential_deployment(model_id=model_id, team_id=team_id) + + def get_router_deployment_budget_limiter(self) -> "RouterBudgetLimiting | None": + _note("get_router_deployment_budget_limiter", "registry().plugins('budget_limiter')") + return _as_router(self)._core_get_router_deployment_budget_limiter() + + _get_router_deployment_budget_limiter = _private_alias( + "get_router_deployment_budget_limiter", get_router_deployment_budget_limiter + ) + + @staticmethod + def generate_model_id(model_group: str, litellm_params: dict) -> str: # mutable-ok: the old parameter type + _note("generate_model_id", "litellm.router_api.deployment_id(model_group, litellm_params)") + from litellm.router_api import deployment_id + + return deployment_id(model_group, litellm_params) + + # -- settings: methods ----------------------------------------------------------------------- + def get_settings(self) -> dict[str, object]: # mutable-ok: Router._core_get_settings's type, a fresh dict + _note("get_settings", "settings().as_dict()") + return _as_router(self)._core_get_settings() + + def get_allowed_fails_from_policy(self, exception: Exception) -> int | None: + _note("get_allowed_fails_from_policy", "settings().allowed_fails_for(exception)") + return _as_router(self)._core_get_allowed_fails_from_policy(exception=exception) + + def get_available_guardrail(self, guardrail_name: str) -> "GuardrailTypedDict": + _note("get_available_guardrail", "settings().guardrail(name)") + return _as_router(self)._core_get_available_guardrail(guardrail_name) + + # -- health ---------------------------------------------------------------------------------- + @staticmethod + def is_deployment_blocked(deployment: "Deployment") -> bool: + _note("is_deployment_blocked", "health().is_blocked(deployment)") + from litellm.router_api import RouterHealth + + return RouterHealth.is_blocked(deployment) + + _is_deployment_blocked = is_deployment_blocked + + @staticmethod + def are_all_deployments_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool: + _note("are_all_deployments_blocked", "health().all_blocked(deployments)") + from litellm.router_api import RouterHealth + + return RouterHealth.all_blocked(deployments) + + _are_all_deployments_blocked = are_all_deployments_blocked + + def get_fully_blocked_model_names(self) -> set[str]: # mutable-ok: the old return type, unchanged + _note("get_fully_blocked_model_names", "health().fully_blocked_model_names()") + return _as_router(self)._core_get_fully_blocked_model_names() + + async def async_get_fully_unhealthy_model_names(self) -> set[str]: # mutable-ok: the old return type, unchanged + _note("async_get_fully_unhealthy_model_names", "health().fully_unhealthy_model_names()") + return await _as_router(self)._core_async_get_fully_unhealthy_model_names() + + async def get_model_group_usage(self, model_group: str) -> tuple[int | None, int | None]: + _note("get_model_group_usage", "health().usage(group)") + return await _as_router(self)._core_get_model_group_usage(model_group=model_group) + + async def get_remaining_model_group_usage( + self, model_group: str + ) -> dict[str, int]: # mutable-ok: the old return type, unchanged + _note("get_remaining_model_group_usage", "health().usage(group, kind='remaining')") + return await _as_router(self)._core_get_remaining_model_group_usage(model_group=model_group) + + def team_ptu_ceiling(self, team_id: str, model_group: str) -> "PTUTeamCeiling | None": + _note("team_ptu_ceiling", "health().team_ptu_ceiling(team_id, group)") + return _as_router(self)._core_team_ptu_ceiling(team_id, model_group) + + def split_request_priority( + self, model: str, kwargs: Mapping[str, object] + ) -> tuple[int | None, Mapping[str, object]]: + _note("split_request_priority", "health().split_priority(model, kwargs)") + return _as_router(self)._core_split_request_priority(model, kwargs) + + # -- selection ------------------------------------------------------------------------------- + async def async_get_healthy_deployments( + self, + model: str, + request_kwargs: dict, # mutable-ok: the old parameter type, unchanged + messages: list[dict[str, str]] | None = None, # mutable-ok: the old parameter type, unchanged + input: str | list | None = None, # mutable-ok: the old parameter type, unchanged + specific_deployment: bool | None = False, + parent_otel_span: "Span | None" = None, + health_check_probe: bool = False, + ) -> list[dict] | dict: # mutable-ok: the old return type, unchanged + _note("async_get_healthy_deployments", "select(model, request_kwargs, kind='healthy')") + return await _as_router(self)._core_async_get_healthy_deployments( + model=model, + request_kwargs=request_kwargs, + messages=messages, + input=input, + specific_deployment=specific_deployment, + parent_otel_span=parent_otel_span, + health_check_probe=health_check_probe, + ) + + def get_available_deployment_for_pass_through( # noqa: ANN201 # the old member had no return annotation; its inferred type is kept by inferring it from the core + self, + model: str, + messages: list[dict[str, str]] | None = None, # mutable-ok: the old parameter type, unchanged + input: str | list | None = None, # mutable-ok: the old parameter type, unchanged + specific_deployment: bool | None = False, + request_kwargs: dict | None = None, # mutable-ok: the old parameter type, unchanged + ): + _note("get_available_deployment_for_pass_through", "select(model, kind='pass_through')") + return _as_router(self)._core_get_available_deployment_for_pass_through( + model=model, + messages=messages, + input=input, + specific_deployment=specific_deployment, + request_kwargs=request_kwargs, + ) + + # -- writes ---------------------------------------------------------------------------------- + def upsert_deployment(self, deployment: "Deployment") -> "Deployment | None": + _note("upsert_deployment", "apply(ChangeSet(upsert=[deployment]))") + return _as_router(self).apply(ChangeSet(upsert=[deployment])).upserted[0] + + def delete_deployment(self, id: str) -> "Deployment | None": + _note("delete_deployment", "apply(ChangeSet(delete=[id]))") + return _as_router(self).apply(ChangeSet(delete=[id])).deleted[0] + + def update_settings(self, **kwargs: object) -> None: # kwargs-ok: the old signature, unchanged + _note("update_settings", "apply(ChangeSet(settings={...}))") + _as_router(self).apply(ChangeSet(settings=kwargs)) + + def update_redis_cache(self, cache: "RedisCache") -> None: + _note("update_redis_cache", "apply(ChangeSet(redis_cache=cache))") + _as_router(self).apply(ChangeSet(redis_cache=cache)) + + _update_redis_cache = _private_alias("update_redis_cache", update_redis_cache) + + async def arefresh_model_info(self, *, client: "AsyncHTTPHandler | None" = None) -> None: + _note("arefresh_model_info", "aapply(ChangeSet(refresh_model_info=True))") + await _as_router(self).aapply(ChangeSet(refresh_model_info=True, refresh_client=client)) + + # -- plugins, observers, lifecycle ----------------------------------------------------------- + def set_custom_routing_strategy(self, CustomRoutingStrategy: "CustomRoutingStrategyBase") -> None: + _note("set_custom_routing_strategy", "register_plugin(strategy)") + _as_router(self).register_plugin(CustomRoutingStrategy) + + def serve_through_request_logging(self) -> None: + _note("serve_through_request_logging", "observe(request_scoped=True)") + _as_router(self).observe(request_scoped=True) + + def bind_to_request_logging(self, logging_obj: "LiteLLMLogging") -> None: + _note("bind_to_request_logging", "observe(logging_obj)") + # Straight to the core, as before: observe() skips binding when logging_obj is None, the old member did not. + _as_router(self)._core_bind_to_request_logging(logging_obj) + + def discard(self) -> None: + _note("discard", "retire()") + _as_router(self).retire() + + # -- plugin context (router strategies) ------------------------------------------------------ + def arm_routing_read_prefetch( + self, + model: str, + request_kwargs: dict[str, object] | None = None, # mutable-ok: the old parameter type, unchanged + ) -> None: + _note("arm_routing_read_prefetch", "RoutingContext.arm_read_prefetch") + _as_router(self)._core_arm_routing_read_prefetch(model, request_kwargs) + + async def async_pre_routing_hook( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: the old parameter type, unchanged + messages: "list[dict[str, Any]] | None" = None, # pyright: ignore[reportExplicitAny] # the old type # mutable-ok: same + input: str | list | None = None, # mutable-ok: the old parameter type, unchanged + specific_deployment: bool | None = False, + ) -> "PreRoutingHookResponse | None": + _note("async_pre_routing_hook", "RoutingContext.pre_routing_hook") + return await _as_router(self)._core_async_pre_routing_hook( + model=model, + request_kwargs=request_kwargs, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) + + def log_retry(self, kwargs: dict, e: Exception) -> dict: # mutable-ok: the old signature, unchanged + _note("log_retry", "RoutingContext.log_retry") + return _as_router(self)._core_log_retry(kwargs=kwargs, e=e) + + +def _name_members_after_router() -> None: + # The old members were Router methods, so errors that name the method (a wrong-arity TypeError, + # or "litellm.router.Router.() argument after ** must be a mapping") keep naming it the + # way they did before the members moved here. + for member in vars(DeprecatedRouterMembers).values(): + function = member.__func__ if isinstance(member, staticmethod) else member + if inspect.isfunction(function): + function.__qualname__ = f"Router.{function.__name__}" + function.__module__ = "litellm.router" + + +_name_members_after_router() diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index c7781819ae4..7e05e98d917 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -186,6 +186,14 @@ ignored_function_names = [ ] +def public_name(function_name): + """ + The deprecated public name a ``_core_`` router member is reached through. Tests that call the + old name run the core member, since ``DeprecatedRouterMembers`` forwards it there. + """ + return function_name.removeprefix("_core_") + + def main(): router_file = [ "./litellm/router.py", @@ -205,12 +213,16 @@ def main(): router_functions.extend(get_functions_from_router(file)) print("router_functions: ", router_functions) called_functions_in_tests = get_all_functions_called_in_tests(tests_dir) - untested_functions = [fn for fn in router_functions if fn not in called_functions_in_tests] + untested_functions = [ + fn + for fn in router_functions + if fn not in called_functions_in_tests and public_name(fn) not in called_functions_in_tests + ] if untested_functions: all_untested_functions = [] for func in untested_functions: - if func not in ignored_function_names: + if func not in ignored_function_names and public_name(func) not in ignored_function_names: all_untested_functions.append(func) untested_perc = (len(all_untested_functions)) / len(router_functions) print("untested_perc: ", untested_perc) diff --git a/tests/code_coverage_tests/router_surface_check.py b/tests/code_coverage_tests/router_surface_check.py index 0b35bac5240..40046391a14 100644 --- a/tests/code_coverage_tests/router_surface_check.py +++ b/tests/code_coverage_tests/router_surface_check.py @@ -37,9 +37,11 @@ ALLOWLIST = os.path.join(os.path.dirname(__file__), "router_surface_allowlist.js ROUTER_FILES = { "litellm/router.py", + "litellm/router_api.py", + "litellm/router_deprecated.py", } -ROUTER_CLASS_FILES = ["litellm/router.py"] -ROUTER_CLASS_NAMES = {"Router"} +ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_deprecated.py"] +ROUTER_CLASS_NAMES = {"Router", "DeprecatedRouterMembers"} SCAN_ROOTS = ["litellm", "enterprise"] SKIP_DIRS = { diff --git a/tests/unit/router_utils/test_router_health_check_routing.py b/tests/unit/router_utils/test_router_health_check_routing.py index 46ed679f746..b1498ea3687 100644 --- a/tests/unit/router_utils/test_router_health_check_routing.py +++ b/tests/unit/router_utils/test_router_health_check_routing.py @@ -53,10 +53,10 @@ class TestFilterHealthCheckUnhealthyDeployments: class FakeRouter: def __init__(self): - self.enable_health_check_routing = enable - self.health_state_cache = health_cache - self.allowed_fails_policy = None - self.background_health_check_model_groups = model_groups + self._core_enable_health_check_routing = enable + self._core_health_state_cache = health_cache + self._core_allowed_fails_policy = None + self._core_background_health_check_model_groups = model_groups # Import the actual method and bind it from litellm.router import Router @@ -169,10 +169,10 @@ class TestAsyncFilterHealthCheckUnhealthyDeployments: class FakeRouter: def __init__(self): - self.enable_health_check_routing = enable - self.health_state_cache = health_cache - self.allowed_fails_policy = None - self.background_health_check_model_groups = model_groups + self._core_enable_health_check_routing = enable + self._core_health_state_cache = health_cache + self._core_allowed_fails_policy = None + self._core_background_health_check_model_groups = model_groups fake = FakeRouter() fake._async_filter_health_check_unhealthy_deployments = ( diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index d0c5c8247d3..f2195512355 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -5242,10 +5242,11 @@ def test_public_aliases( resolved_owner: Final = _get_owner(module_path, owner_name) alias_owner: Final = _get_instance(resolved_owner, new_name) if use_instance else resolved_owner old_value, new_value = getattr(alias_owner, old_name), getattr(alias_owner, new_name) + # Router's private aliases wrap the public method so they skip a subclass override of it. if isinstance(old_value, MethodType) and isinstance(new_value, MethodType): - assert old_value.__func__ is new_value.__func__ + assert old_value.__func__ is new_value.__func__ or inspect.unwrap(old_value.__func__) is new_value.__func__ else: - assert old_value is new_value + assert old_value is new_value or inspect.unwrap(old_value) is new_value @pytest.mark.parametrize( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 62510dac26d..a0a7dd78193 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2321,7 +2321,7 @@ def test_with_router_resolved_session_model(session, expected): def test_router_get_model_access_groups_team_only_models(): """ - Test that Router.get_model_access_groups returns the correct response for team-only models + Test that Router._core_get_model_access_groups returns the correct response for team-only models """ router = litellm.Router( model_list=[ @@ -2440,7 +2440,7 @@ def test_model_group_info_cost_from_db_model_info(): ) with patch.object( - router, "get_deployment_model_info", side_effect=Exception("not found") + router, "_core_get_deployment_model_info", side_effect=Exception("not found") ): result = router._cached_get_model_group_info("my-custom-model") assert result is not None @@ -2470,7 +2470,7 @@ def test_model_group_info_cost_none_when_db_model_info_has_no_cost(): ) with patch.object( - router, "get_deployment_model_info", side_effect=Exception("not found") + router, "_core_get_deployment_model_info", side_effect=Exception("not found") ): result = router._cached_get_model_group_info("my-custom-model-no-cost") assert result is not None @@ -2854,7 +2854,7 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto }, ) assert router.routing_strategy == "lar1" - assert "async_get_available_deployment" in router.__dict__ + assert "_core_async_get_available_deployment" in router.__dict__ router._switch_routing_strategy("usage-based-routing-v2", {}) assert router.lowesttpm_logger_v2 is not None @@ -2925,7 +2925,7 @@ def test_model_group_info_with_stringified_cost_values(): return None with patch.object( - router, "get_deployment_model_info", side_effect=_model_info_with_str_costs + router, "_core_get_deployment_model_info", side_effect=_model_info_with_str_costs ): result = router._set_model_group_info( model_group="my-custom-model", @@ -2973,7 +2973,7 @@ def test_model_group_info_db_fallback_with_stringified_cost_values(): ) with patch.object( - router, "get_deployment_model_info", side_effect=Exception("not found") + router, "_core_get_deployment_model_info", side_effect=Exception("not found") ): result = router._set_model_group_info( model_group="my-custom-model", @@ -7828,7 +7828,7 @@ async def test_async_function_with_fallbacks_common_utils(): def test_should_include_deployment(): - """Test that Router.should_include_deployment returns the correct response""" + """Test that Router._core_should_include_deployment returns the correct response""" router = litellm.Router( model_list=[ { @@ -7883,7 +7883,7 @@ def test_pre_call_checks_skips_token_count_without_max_input_tokens(monkeypatch) ], enable_pre_call_checks=True, ) - monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: {}) + monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: {}) calls = [] monkeypatch.setattr( @@ -7917,7 +7917,7 @@ def test_pre_call_checks_counts_once_and_filters_on_max_input_tokens(monkeypatch enable_pre_call_checks=True, ) monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} ) calls = [] @@ -8031,7 +8031,7 @@ def test_pre_call_checks_uses_precounted_tokens(monkeypatch): enable_pre_call_checks=True, ) monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} ) calls = [] @@ -8065,7 +8065,7 @@ async def test_async_get_healthy_deployments_counts_tokens_off_the_event_loop(mo enable_pre_call_checks=True, ) monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 1_000_000} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 1_000_000} ) counting_threads = [] @@ -8115,7 +8115,7 @@ def test_pre_call_checks_need_token_count(monkeypatch, model_info, expected): ], enable_pre_call_checks=True, ) - monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: model_info) + monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: model_info) deployments = [ {"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "d1"}}, @@ -8141,7 +8141,7 @@ def test_deployment_max_input_tokens_survives_an_unmappable_deployment(monkeypat raise ValueError("This model isn't mapped yet.") return {"max_input_tokens": 100} - monkeypatch.setattr(router, "get_router_model_info", flaky_model_info) + monkeypatch.setattr(router, "_core_get_router_model_info", flaky_model_info) unmapped = {"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "unmapped"}} mapped = {"litellm_params": {"model": "gpt-3.5-turbo"}, "model_info": {"id": "mapped"}} @@ -8163,7 +8163,7 @@ def test_pre_call_checks_does_not_recount_inline_after_an_off_loop_failure(monke enable_pre_call_checks=True, ) monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} ) calls = [] @@ -8198,7 +8198,7 @@ async def test_async_get_healthy_deployments_never_recounts_on_the_loop(monkeypa enable_pre_call_checks=True, ) monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} ) counting_threads = [] @@ -8235,7 +8235,7 @@ async def test_acount_pre_call_check_tokens_leaves_the_event_loop_free(monkeypat enable_pre_call_checks=True, ) monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} ) deployments = [ @@ -8269,7 +8269,7 @@ async def test_acount_pre_call_check_tokens_skips_without_max_input_tokens(monke ], enable_pre_call_checks=True, ) - monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: {}) + monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: {}) calls = [] monkeypatch.setattr( @@ -8303,7 +8303,7 @@ def test_pre_call_checks_counts_tokens_from_responses_input_string(monkeypatch): enable_pre_call_checks=True, ) monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 1} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 1} ) deployments = [ @@ -8330,7 +8330,7 @@ def test_pre_call_checks_counts_tokens_from_responses_input_list(monkeypatch): enable_pre_call_checks=True, ) monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 1} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 1} ) deployments = [ @@ -8374,7 +8374,7 @@ def test_pre_call_checks_counts_responses_instructions_tokens(monkeypatch): assert with_instructions_tokens > input_only_tokens monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": input_only_tokens} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": input_only_tokens} ) with pytest.raises(litellm.ContextWindowExceededError): router._pre_call_checks( @@ -8445,7 +8445,7 @@ def test_pre_call_checks_counts_tool_definition_tokens(monkeypatch, prompt_kwarg messages=prompt_kwargs.get("messages"), input=prompt_kwargs.get("input") ) monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": prompt_only_tokens} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": prompt_only_tokens} ) assert len(router._pre_call_checks(model="m", healthy_deployments=deployments, **prompt_kwargs)) == 1 @@ -8486,7 +8486,7 @@ def test_pre_call_checks_counts_anthropic_system_tokens(monkeypatch, system): messages = [{"role": "user", "content": "hi"}] messages_only_tokens = router._count_pre_call_check_tokens(messages=messages, input=None) - monkeypatch.setattr(router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": messages_only_tokens}) + monkeypatch.setattr(router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": messages_only_tokens}) assert len(router._pre_call_checks(model="m", healthy_deployments=deployments, messages=messages)) == 1 with pytest.raises(litellm.ContextWindowExceededError): @@ -8605,7 +8605,7 @@ def test_pre_call_checks_no_messages_or_input_does_not_crash(monkeypatch): enable_pre_call_checks=True, ) monkeypatch.setattr( - router, "get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} + router, "_core_get_router_model_info", lambda **kwargs: {"max_input_tokens": 5} ) counted: list[dict] = [] @@ -9952,7 +9952,7 @@ def test_deployment_usable_by_team_uses_dynamic_model_info_get(): def get(self, key: str) -> str | None: return "team-b" if key == "team_id" else None - assert litellm.Router.deployment_usable_by_team({"model_info": ModelInfo()}, "team-a") is False + assert litellm.Router._core_deployment_usable_by_team({"model_info": ModelInfo()}, "team-a") is False def test_get_deployment_credentials_with_provider_skips_other_team_wildcard(): @@ -11599,7 +11599,7 @@ def test_access_group_filter_empty_does_not_bypass_via_litellm_model_fallback( team_id=team_id, ) - monkeypatch.setattr(router, "get_model_access_groups", fake_get_model_access_groups) + monkeypatch.setattr(router, "_core_get_model_access_groups", fake_get_model_access_groups) scoped_key = UserAPIKeyAuth( api_key="hashed-key", @@ -11676,7 +11676,7 @@ def test_access_group_block_does_not_silently_use_default_fallback_model( team_id=team_id, ) - monkeypatch.setattr(router, "get_model_access_groups", fake_get_model_access_groups) + monkeypatch.setattr(router, "_core_get_model_access_groups", fake_get_model_access_groups) scoped_key = UserAPIKeyAuth( api_key="hashed-key", @@ -11745,7 +11745,7 @@ def test_access_group_block_via_litellm_model_branch_does_not_use_default_fallba team_id=team_id, ) - monkeypatch.setattr(router, "get_model_access_groups", fake_get_model_access_groups) + monkeypatch.setattr(router, "_core_get_model_access_groups", fake_get_model_access_groups) scoped_key = UserAPIKeyAuth( api_key="hashed-key", @@ -12246,7 +12246,7 @@ def test_get_deployment_credentials_with_provider_returns_none_for_blocked_deplo def test_is_deployment_blocked_static_helper_reflects_blocked_flag(): """ - Exercises Router.is_deployment_blocked so router_code_coverage.py (AST call graph) + Exercises Router._core_is_deployment_blocked so router_code_coverage.py (AST call graph) marks the helper as covered by router-named tests. """ import types @@ -12257,15 +12257,15 @@ def test_is_deployment_blocked_static_helper_reflects_blocked_flag(): blocked_dep = router.get_deployment("dep-0") unblocked_dep = router.get_deployment("dep-1") assert blocked_dep is not None and unblocked_dep is not None - assert litellm.Router.is_deployment_blocked(blocked_dep) is True - assert litellm.Router.is_deployment_blocked(unblocked_dep) is False + assert litellm.Router._core_is_deployment_blocked(blocked_dep) is True + assert litellm.Router._core_is_deployment_blocked(unblocked_dep) is False # No model_info on deployment object → treated as not blocked - assert litellm.Router.is_deployment_blocked(object()) is False + assert litellm.Router._core_is_deployment_blocked(object()) is False missing_blocked = types.SimpleNamespace() - assert litellm.Router.is_deployment_blocked(types.SimpleNamespace(model_info=missing_blocked)) is False + assert litellm.Router._core_is_deployment_blocked(types.SimpleNamespace(model_info=missing_blocked)) is False assert ( - litellm.Router.is_deployment_blocked(types.SimpleNamespace(model_info=types.SimpleNamespace(blocked=True))) + litellm.Router._core_is_deployment_blocked(types.SimpleNamespace(model_info=types.SimpleNamespace(blocked=True))) is True ) @@ -14991,7 +14991,7 @@ def test_pre_call_checks_uses_deployment_model_when_model_info_lookup_raises(mon def _raise_unmapped(**kwargs): raise ValueError("This model isn't mapped yet") - monkeypatch.setattr(router, "get_router_model_info", _raise_unmapped) + monkeypatch.setattr(router, "_core_get_router_model_info", _raise_unmapped) seen: list[tuple] = [] original_get_supported_openai_params = litellm.get_supported_openai_params @@ -17182,7 +17182,7 @@ def test_model_group_info_intersects_supported_reasoning_efforts(): "supports_xhigh_reasoning_effort": False, } - with patch.object(router, "get_deployment_model_info", side_effect=_model_info): + with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info): result = router._set_model_group_info( model_group="smart-group", user_facing_model_group_name="smart-group", @@ -17226,7 +17226,7 @@ def test_model_group_info_reasoning_efforts_are_unknown_when_any_deployment_is_o } return {"key": model_name, "litellm_provider": "openai", "mode": None, "supports_reasoning": None} - with patch.object(router, "get_deployment_model_info", side_effect=_model_info): + with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info): result = router._set_model_group_info( model_group="smart-group", user_facing_model_group_name="smart-group", @@ -17361,7 +17361,7 @@ def test_model_group_info_reasoning_efforts_empty_on_a_mapped_non_reasoning_depl } return {"key": model_name, "litellm_provider": "openai", "mode": "chat", "supports_reasoning": None} - with patch.object(router, "get_deployment_model_info", side_effect=_model_info): + with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info): result = router._set_model_group_info( model_group="mixed-group", user_facing_model_group_name="mixed-group", @@ -17402,7 +17402,7 @@ def test_model_group_info_reasoning_efforts_ignore_a_value_declared_in_model_inf info["supported_reasoning_efforts"] = ("high",) return info - with patch.object(router, "get_deployment_model_info", side_effect=_model_info): + with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info): result = router._set_model_group_info( model_group="declared-group", user_facing_model_group_name="declared-group", @@ -17436,7 +17436,7 @@ def test_model_group_info_survives_a_junk_typed_operator_effort_value(): "supported_reasoning_efforts": "high", } - with patch.object(router, "get_deployment_model_info", side_effect=_model_info): + with patch.object(router, "_core_get_deployment_model_info", side_effect=_model_info): result = router._set_model_group_info( model_group="junk-declared-group", user_facing_model_group_name="junk-declared-group", @@ -23993,7 +23993,7 @@ def test_a_failed_routing_read_prefetch_logs_the_request_model_without_its_line_ def fail_lookup(model_name: str | None = None, team_id: str | None = None) -> None: raise RuntimeError(f"no deployments for {model_name}") - monkeypatch.setattr(router, "get_model_list", fail_lookup) + monkeypatch.setattr(router, "_core_get_model_list", fail_lookup) caplog.clear() with caplog.at_level(logging.DEBUG, logger="LiteLLM Router"): @@ -26003,7 +26003,7 @@ async def test_router_acancel_batch_removes_unresolved_credential_name(): mock_response.status = "cancelled" with ( - patch.object(router, "get_deployment_credentials_with_provider", return_value=None), + patch.object(router, "_core_get_deployment_credentials_with_provider", return_value=None), patch.object(litellm, "acancel_batch", new_callable=AsyncMock) as mock_cancel, ): mock_cancel.return_value = mock_response @@ -29090,7 +29090,7 @@ def test_reset_custom_routing_strategy(): router.set_custom_routing_strategy(SpecialEndpointRoutingStrategy(router)) assert "get_available_deployment" in router.__dict__ - assert "async_get_available_deployment" in router.__dict__ + assert "_core_async_get_available_deployment" in router.__dict__ router._reset_custom_routing_strategy() @@ -31179,7 +31179,7 @@ def test_team_model_has_alternatives_when_a_delete_starts_after_its_deployment_l has_alternatives: Final = _read_during_held_write( lambda: router.team_model_has_alternatives("deployment-3"), lambda: router.delete_deployment("deployment-0"), - write_after=Router.get_deployment.__code__, + write_after=Router._core_get_deployment.__code__, ) assert has_alternatives is True diff --git a/tests/unit/test_router/test_router_entry_points.py b/tests/unit/test_router/test_router_entry_points.py new file mode 100644 index 00000000000..2df5cbc9457 --- /dev/null +++ b/tests/unit/test_router/test_router_entry_points.py @@ -0,0 +1,829 @@ +""" +Router entry points and DeprecatedRouterMembers. + +Every member that moved into DeprecatedRouterMembers must return what its entry point returns, +warn once per access from outside the router, and count the access. Internal access (the router's +own modules) must stay silent. +""" + +import asyncio +import copy +import dataclasses +import warnings + +import httpx +import pytest + +import litellm +from litellm import Router +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.router_api import SETTING_FIELDS, ChangeSet, RouterSettings +from litellm.router_deprecated import DEPRECATED_ROUTER_MEMBER_USES, DeprecatedRouterMembers +from litellm.types.router import Deployment + + +def _router() -> Router: + return Router( + model_list=[ + { + "model_name": "gpt", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"}, + "model_info": {"id": "dep-a", "access_groups": ["grp"]}, + }, + { + "model_name": "gpt", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-b"}, + "model_info": {"id": "dep-b"}, + }, + { + "model_name": "team-internal", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-t"}, + "model_info": {"id": "dep-team", "team_id": "team-1", "team_public_model_name": "team-gpt"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "sk-w"}, + "model_info": {"id": "dep-wild"}, + }, + ], + model_group_alias={"alias-gpt": "gpt"}, + fallbacks=[{"gpt": ["team-internal"]}], + cooldown_time=7, + ) + + +@pytest.fixture +def router(): + r = _router() + yield r + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + r.retire() + + +# (old member, how to read it through the old name, how to read it through the entry point) +READS = [ + ("model_list", lambda r: r.model_list, lambda r: r.registry().all()), + ("model_names", lambda r: r.model_names, lambda r: r.registry().names(kind="group_set")), + ("deployment_names", lambda r: r.deployment_names, lambda r: r.registry().names(kind="deployment_names")), + ("team_public_model_names", lambda r: r.team_public_model_names, lambda r: r.registry().names(kind="team_public")), + ( + "model_name_to_deployment_indices", + lambda r: r.model_name_to_deployment_indices, + lambda r: r.registry().index("model_name"), + ), + ( + "team_model_to_deployment_indices", + lambda r: r.team_model_to_deployment_indices, + lambda r: r.registry().index("team_model"), + ), + ("model_group_alias", lambda r: r.model_group_alias, lambda r: r.registry().aliases()), + ("pattern_router", lambda r: r.pattern_router, lambda r: r.registry().patterns()), + ("team_pattern_routers", lambda r: r.team_pattern_routers, lambda r: r.registry().patterns(team=True)), + ("routing_groups", lambda r: r.routing_groups, lambda r: r.registry().routing_groups()), + ("auto_routers", lambda r: r.auto_routers, lambda r: r.registry().plugins("auto")), + ("complexity_routers", lambda r: r.complexity_routers, lambda r: r.registry().plugins("complexity")), + ("quality_routers", lambda r: r.quality_routers, lambda r: r.registry().plugins("quality")), + ("adaptive_routers", lambda r: r.adaptive_routers, lambda r: r.registry().plugins("adaptive")), + ("routing_plugins", lambda r: r.routing_plugins, lambda r: r.registry().plugins("routing")), + ( + "prompt_caching_pin_cache", + lambda r: r.prompt_caching_pin_cache, + lambda r: r.registry().plugins("prompt_caching_pins"), + ), + ("cache", lambda r: r.cache, lambda r: r.health().store), + ("cooldown_cache", lambda r: r.cooldown_cache, lambda r: r.health().cooldowns), + ("health_state_cache", lambda r: r.health_state_cache, lambda r: r.health().health_checks), + ("deployment_latency_map", lambda r: r.deployment_latency_map, lambda r: r.health().latency), + ("get_deployment", lambda r: r.get_deployment("dep-a"), lambda r: r.registry().deployment("dep-a")), + ("has_model_id", lambda r: r.has_model_id("dep-b"), lambda r: r.registry().has_deployment("dep-b")), + ("get_model_info", lambda r: r.get_model_info("dep-a"), lambda r: r.registry().deployment_dict("dep-a")), + ("get_model_list", lambda r: r.get_model_list("gpt"), lambda r: r.registry().deployments("gpt")), + ("get_model_list_all", lambda r: r.get_model_list(), lambda r: r.registry().deployments()), + ("get_model_list_alias", lambda r: r.get_model_list("alias-gpt"), lambda r: r.registry().deployments("alias-gpt")), + ( + "get_all_deployments", + lambda r: r.get_all_deployments("gpt"), + lambda r: r.registry().deployments("gpt", source="all"), + ), + ( + "config_deployments", + lambda r: list(r.config_deployments()), + lambda r: list(r.registry().deployments(source="config")), + ), + ( + "deployments_named", + lambda r: r.deployments_named("gpt"), + lambda r: r.registry().deployments("gpt", source="named"), + ), + ( + "get_model_list_from_model_alias", + lambda r: r.get_model_list_from_model_alias("alias-gpt"), + lambda r: r.registry().deployments("alias-gpt", source="alias"), + ), + ( + "get_deployment_by_model_group_name", + lambda r: r.get_deployment_by_model_group_name("gpt"), + lambda r: r.registry().deployments("gpt", source="first_in_group"), + ), + ("get_model_names", lambda r: r.get_model_names(), lambda r: r.registry().names()), + ( + "get_model_names_team", + lambda r: r.get_model_names(team_id="team-1"), + lambda r: r.registry().names(team_id="team-1"), + ), + ("get_model_ids", lambda r: r.get_model_ids("gpt"), lambda r: r.registry().names("gpt", kind="ids")), + ( + "get_model_ids_all", + lambda r: r.get_model_ids(exclude_team_models=True), + lambda r: r.registry().names(kind="ids", exclude_team_models=True), + ), + ( + "get_model_from_alias", + lambda r: r.get_model_from_alias("alias-gpt"), + lambda r: r.registry().resolve("alias-gpt", kind="alias"), + ), + ( + "is_recognized_model", + lambda r: r.is_recognized_model("gpt"), + lambda r: r.registry().resolve("gpt", kind="recognized"), + ), + ( + "routable_model_group", + lambda r: r.routable_model_group("alias-gpt"), + lambda r: r.registry().resolve("alias-gpt"), + ), + ( + "map_team_model", + lambda r: r.map_team_model("team-gpt", "team-1"), + lambda r: r.registry().resolve("team-gpt", kind="team_model", team_id="team-1"), + ), + ( + "get_model_group", + lambda r: r.get_model_group("dep-a"), + lambda r: r.registry().resolve("dep-a", kind="group_of_id"), + ), + ( + "get_search_tool_support", + lambda r: r.get_search_tool_support("gpt"), + lambda r: r.registry().resolve("gpt", kind="search_tool_support"), + ), + ("get_model_access_groups", lambda r: r.get_model_access_groups(), lambda r: r.registry().access_groups()), + ( + "get_deployment_credentials", + lambda r: r.get_deployment_credentials("dep-a"), + lambda r: r.registry().credentials("dep-a"), + ), + ( + "get_deployment_credentials_with_provider", + lambda r: r.get_deployment_credentials_with_provider("dep-a"), + lambda r: r.registry().credentials("dep-a", form="with_provider"), + ), + ( + "get_credential_deployment", + lambda r: r.get_credential_deployment("dep-a"), + lambda r: r.registry().credentials("dep-a", form="deployment"), + ), + ("get_configured_mode", lambda r: r.get_configured_mode("gpt"), lambda r: r.registry().group("gpt", field="mode")), + ( + "get_configured_token_limits", + lambda r: r.get_configured_token_limits("gpt"), + lambda r: r.registry().group("gpt", field="token_limits"), + ), + ( + "get_fully_blocked_model_names", + lambda r: r.get_fully_blocked_model_names(), + lambda r: r.health().fully_blocked_model_names(), + ), + ("get_settings", lambda r: r.get_settings(), lambda r: r.settings().as_dict()), + ("fallbacks", lambda r: r.fallbacks, lambda r: r.settings().fallbacks), + ("cooldown_time", lambda r: r.cooldown_time, lambda r: r.settings().cooldown_time), + ("routing_strategy", lambda r: r.routing_strategy, lambda r: r.settings().routing_strategy), +] + + +@pytest.mark.parametrize("name, old, new", READS, ids=[r[0] for r in READS]) +def test_deprecated_read_matches_entry_point(router, name, old, new): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + old_value = old(router) + new_value = new(router) + if isinstance(old_value, (list, dict, set)) or hasattr(old_value, "__dict__"): + assert old_value == new_value + else: + assert old_value == new_value + assert any(issubclass(w.category, DeprecationWarning) for w in caught), name + + +def test_live_objects_not_copies(router): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert router.model_list is router.registry().all() + assert router.fallbacks is router.settings().fallbacks + assert router.cache is router.health().store + assert router.model_group_alias is router.registry().aliases() + + +def test_assignment_goes_through_apply(router): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + router.fallbacks = [{"gpt": ["openai/gpt-4.1"]}] + assert router.settings().fallbacks == [{"gpt": ["openai/gpt-4.1"]}] + router.search_tools = [{"search_tool_name": "s", "litellm_params": {"search_provider": "tavily"}}] + assert router.settings().search_tools[0]["search_tool_name"] == "s" + router.apply(ChangeSet(assign={"cooldown_time": 3})) + assert router.settings().cooldown_time == 3 + with pytest.raises(ValueError, match="not assignable router settings"): + router.apply(ChangeSet(assign={"not_a_setting": 1})) + + +def test_apply_upsert_and_delete_bump_version(router): + v0 = router.registry().version + added = Deployment( + model_name="new-group", + litellm_params={"model": "openai/gpt-4o-mini", "api_key": "sk-n"}, + model_info={"id": "dep-new"}, + ) + result = router.apply(ChangeSet(upsert=[added])) + assert result.version == v0 + 1 + assert router.registry().has_deployment("dep-new") + assert router.registry().deployments("new-group")[0]["model_info"]["id"] == "dep-new" + result = router.apply(ChangeSet(delete=["dep-new"])) + assert result.version == v0 + 2 + assert result.deleted[0] is not None + assert not router.registry().has_deployment("dep-new") + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert router.upsert_deployment(added) is added + assert router.upsert_deployment(added) is None # unchanged + assert router.delete_deployment("dep-new") is not None + + +def test_old_deployment_writes_also_advance_the_version(router): + v0 = router.registry().version + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + router.add_deployment( + Deployment( + model_name="added", + litellm_params={"model": "openai/gpt-4o-mini", "api_key": "sk-x"}, + model_info={"id": "dep-added"}, + ) + ) + v1 = router.registry().version + router.set_model_list([{"model_name": "only", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-o"}}]) + assert v0 < v1 < router.registry().version + assert router.registry().names() == ["only"] + v2 = router.registry().version + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + router.model_list = [] + assert router.registry().version > v2 + + +def test_update_settings_through_apply(router): + router.apply(ChangeSet(settings={"cooldown_time": "11", "not_allowed": 1})) + assert router.settings().cooldown_time == 11 + + +def test_aapply_applies_changes_then_refreshes_model_info(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + router = Router( + model_list=[ + { + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://capacity.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "dep-local"}, + } + ] + ) + with pytest.raises(TypeError, match="needs aapply"): + router.apply(ChangeSet(refresh_model_info=True)) + + async def refresh() -> None: + handler = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}) + ) + ) as client: + handler.client = client + await router.aapply(ChangeSet(assign={"cooldown_time": 4}, refresh_model_info=True, refresh_client=handler)) + + asyncio.run(refresh()) + assert router.settings().cooldown_time == 4 + assert router.registry().group("local", field="token_limits") == (4096, 4096) + + +def test_settings_snapshot_is_reused_until_a_setting_is_written(router): + first = router.settings() + assert router.settings() is first + router.apply(ChangeSet(assign={"cooldown_time": 7})) + assert router.settings() is not first + assert router.settings().cooldown_time == 7 + router.apply(ChangeSet(settings={"cooldown_time": "8"})) + assert router.settings().cooldown_time == 8 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + router.fallbacks = [{"gpt": ["team-gpt"]}] + assert router.settings().fallbacks == [{"gpt": ["team-gpt"]}] + router._core_fallbacks = [{"gpt": ["other"]}] # the router's own writes invalidate it too + assert router.settings().fallbacks == [{"gpt": ["other"]}] + router.settings().fallbacks.append({"x": ["y"]}) # the snapshot holds the router's own objects + assert router._core_fallbacks[-1] == {"x": ["y"]} + unrelated = router.settings() + router._core_model_list = list(router._core_model_list) + assert router.settings() is unrelated + + +def test_deleting_a_setting_shows_as_unset(router): + router.apply(ChangeSet(assign={"cooldown_time": 3})) + assert router.settings().is_set("cooldown_time") + del router._core_cooldown_time + assert router.settings().is_set("cooldown_time") is (hasattr(router, "_core_cooldown_time")) + + +def test_settings_snapshot_is_frozen(router): + snapshot = router.settings() + assert isinstance(snapshot, RouterSettings) + with pytest.raises(dataclasses.FrozenInstanceError): + snapshot.cooldown_time = 1 # type: ignore[misc] + assert set(SETTING_FIELDS) <= set(RouterSettings.__dataclass_fields__) + + +def test_unset_setting_raises_attribute_error_through_old_name(): + r = _router() + try: + del r.__dict__["_core_kubernetes_pod_discovery"] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert getattr(r, "kubernetes_pod_discovery", "missing") == "missing" + assert r.settings().is_set("kubernetes_pod_discovery") is False + finally: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + r.retire() + + +def test_unset_attribute_error_names_the_public_attribute(router): + # Reading (or deleting) an old attribute the router never set raises what reading it raised before. + router.__dict__.pop("_core_slack_alerting_logger", None) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with pytest.raises(AttributeError) as read: + _ = router.slack_alerting_logger + with pytest.raises(AttributeError) as deleted: + del router.slack_alerting_logger + del router.__dict__["_core_kubernetes_pod_discovery"] + with pytest.raises(AttributeError) as setting: + _ = router.kubernetes_pod_discovery + for caught, name in ((read, "slack_alerting_logger"), (setting, "kubernetes_pod_discovery")): + assert str(caught.value) == f"'Router' object has no attribute '{name}'" + assert caught.value.name == name + assert caught.value.obj is router + # A failed del sets no name/obj in Python either. + assert str(deleted.value) == "'Router' object has no attribute 'slack_alerting_logger'" + + +def test_bind_to_request_logging_none_raises_as_before(router): + # The old member bound unconditionally, so None fails on the first callback registration. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with pytest.raises(AttributeError, match="add_dynamic_function_callbacks"): + router.bind_to_request_logging(None) + + +def test_select_picks_like_async_get_available_deployment(router): + picked = asyncio.run(router.select("gpt", {})) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + old = asyncio.run(router.async_get_available_deployment(model="gpt", request_kwargs={})) + assert picked["model_name"] == old["model_name"] == "gpt" + healthy = asyncio.run(router.select("gpt", {}, kind="healthy")) + assert {d["model_info"]["id"] for d in healthy} == {"dep-a", "dep-b"} + + +def test_counter_and_internal_silence(router): + before = DEPRECATED_ROUTER_MEMBER_USES["get_model_ids"] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + router.get_model_ids() + # Router internals call the core names; a completion must not warn. + asyncio.run(router.acompletion(model="gpt", messages=[{"role": "user", "content": "hi"}], mock_response="ok")) + assert DEPRECATED_ROUTER_MEMBER_USES["get_model_ids"] == before + 1 + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning) and "Router." in str(w.message)] + assert [str(w.message).split(" ")[0] for w in deprecations] == ["Router.get_model_ids"] + assert deprecations[0].filename == __file__ + + +def test_every_deprecated_member_names_a_replacement(): + import inspect + + from litellm.router_deprecated import _DeprecatedAttribute + + for name, member in vars(DeprecatedRouterMembers).items(): + if name.startswith("__"): + continue + if isinstance(member, _DeprecatedAttribute): + assert member.replacement + continue + func = member.__func__ if isinstance(member, staticmethod) else member + if callable(func): + assert "_note(" in inspect.getsource(func), name + + +def test_generate_model_id_is_the_module_function(): + from litellm.router_api import deployment_id + + params = {"model": "openai/gpt-4o", "api_key": "k", "extra": {"a": 1}} + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert Router.generate_model_id("g", params) == deployment_id("g", params) + + +def test_subclass_override_of_an_old_member_is_still_called(): + calls = [] + + class Sub(litellm.Router): + async def async_pre_routing_hook( + self, model, request_kwargs, messages=None, input=None, specific_deployment=False + ): + calls.append(model) + return await super().async_pre_routing_hook(model, request_kwargs, messages, input, specific_deployment) + + r = Sub( + model_list=[ + { + "model_name": "m", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x", "mock_response": "ok"}, + } + ] + ) + response = asyncio.run(r.acompletion(model="m", messages=[{"role": "user", "content": "hi"}])) + assert response.choices[0].message.content == "ok" + assert calls == ["m"] + + +def test_direct_call_of_a_subclass_override_runs_it_once(): + calls = [] + + class Sub(litellm.Router): + def get_model_access_groups(self, model_name=None, model_access_group=None, team_id=None): + calls.append(model_name) + return super().get_model_access_groups(model_name, model_access_group, team_id) + + r = Sub(model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}}]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert r.get_model_access_groups("m") == {} + assert calls == ["m"] + assert asyncio.run(asyncio.to_thread(r.registry().access_groups, model_name="m")) == {} + assert calls == ["m", "m"] + + +def test_subclass_staticmethod_override_is_still_consulted(): + calls = [] + + class Sub(litellm.Router): + @staticmethod + def is_deployment_blocked(deployment): + calls.append(deployment.model_info.id) + return True + + r = Sub( + model_list=[ + { + "model_name": "m", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}, + "model_info": {"id": "d1"}, + } + ] + ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert r.get_deployment_credentials(model_id="d1") is None + assert calls == ["d1"] + + +def test_private_alias_skips_a_subclass_override_of_the_public_name(): + calls = [] + + class Sub(litellm.Router): + def get_all_deployments(self, *args, **kwargs): + calls.append("get_all_deployments") + return super().get_all_deployments(*args, **kwargs) + + r = Sub(model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}}]) + calls.clear() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert len(r._get_all_deployments(model_name="m")) == 1 + assert calls == [] + + +def test_old_name_writes_and_deletes_warn(router): + with pytest.warns(DeprecationWarning, match=r"^Router\.fallbacks is deprecated"): + router.fallbacks = [] + with pytest.warns(DeprecationWarning, match=r"^Router\.fallbacks is deprecated"): + del router.fallbacks + + +def test_alias_attribute_writes_the_same_storage(router): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + replacement = {} + router._routing_groups = replacement + assert router.routing_groups is replacement + assert router.registry().routing_groups() is replacement + + +def test_core_members_keep_their_old_names_in_errors(): + from litellm.router import _name_core_members_after_old_names + + _name_core_members_after_old_names() + + assert Router._core_get_deployment_model_info.__qualname__ == "Router.get_deployment_model_info" + assert Router._core_get_deployment.__qualname__ == "Router.get_deployment" + + +def test_wrong_arity_errors_still_name_router(router): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with pytest.raises(TypeError, match=r"^Router\.get_deployment\(\) takes 2 positional arguments"): + router.get_deployment("dep-a", "extra") # pyright: ignore[reportCallIssue] + with pytest.raises(TypeError, match=r"^Router\.get_available_guardrail\(\) missing 1 required"): + router.get_available_guardrail() # pyright: ignore[reportCallIssue] + with pytest.raises(TypeError, match=r"^litellm\.router\.Router\.has_model_id\(\) argument after \*\*"): + router.has_model_id(**None) # pyright: ignore[reportCallIssue, reportArgumentType] + with pytest.raises(TypeError, match=r"^Router\.is_deployment_blocked\(\) missing 1 required"): + router.is_deployment_blocked() # pyright: ignore[reportCallIssue] + with pytest.raises(TypeError, match=r"^Router\.get_deployment_model_info\(\) missing 2 required"): + router.cached_deployment_model_info() # pyright: ignore[reportCallIssue] + + +_SENTINEL = object() + +# (entry point call, Router core member it forwards to, the arguments the core member must get) +FORWARDS = [ + ( + lambda r: r.registry().deployment_model_info("dep-a", "gpt", cached=True), + "_core_cached_deployment_model_info", + (("dep-a", "gpt"), {}), + ), + ( + lambda r: r.registry().deployment_model_info("dep-a", "gpt"), + "_core_get_deployment_model_info", + ((), {"model_id": "dep-a", "model_name": "gpt"}), + ), + ( + lambda r: r.registry().router_model_info(None, "gpt", id="dep-a"), + "_core_get_router_model_info", + ((), {"deployment": None, "received_model_name": "gpt", "id": "dep-a"}), + ), + (lambda r: r.registry().discovered_model_info("dep-a"), "_core_get_discovered_model_info", (("dep-a",), {})), + ( + lambda r: r.registry().deployments("gpt", source="routing_groups"), + "_core_get_model_list_from_routing_groups", + ((), {"model_name": "gpt"}), + ), + ( + lambda r: r.registry().deployments("gpt", source="routed_group"), + "_core_get_model_list_of_routed_group", + (("gpt",), {}), + ), + ( + lambda r: r.registry().deployments("gpt", source="wildcard", team_id="t"), + "_core_wildcard_route_deployments", + (("gpt", "t"), {}), + ), + ( + lambda r: r.registry().deployments("gpt", source="routable", request={"a": 1}, call_type="aembedding"), + "_core_get_routable_deployments", + (("gpt", {"a": 1}, "aembedding"), {}), + ), + ( + lambda r: r.registry().deployments("gpt", source="for_request"), + "_core_deployments_for_request", + (("gpt", {}), {}), + ), + ( + lambda r: r.registry().includes({"model_name": "gpt"}, model_name="gpt", team_id="t"), + "_core_should_include_deployment", + ((), {"model_name": "gpt", "model": {"model_name": "gpt"}, "team_id": "t"}), + ), + ( + lambda r: r.registry().names("gpt", kind="litellm_models", team_id="t"), + "_core_resolved_litellm_models", + (("gpt",), {"team_id": "t"}), + ), + ( + lambda r: r.registry().names("gpt", kind="litellm_base_models"), + "_core_resolved_litellm_models", + (("gpt",), {"team_id": None, "use_base_model": True}), + ), + ( + lambda r: r.registry().names("gpt", kind="wildcard", team_id="t"), + "_core_wildcard_routed_models", + (("gpt", "t"), {}), + ), + ( + lambda r: r.registry().names("gpt", kind="route_candidates"), + "_core_get_candidate_model_ids_for_route", + (("gpt",), {"team_id": None}), + ), + ( + lambda r: r.registry().resolve("dep-a", kind="from_id", custom_llm_provider="openai"), + "_core_resolve_model_name_from_model_id", + (("dep-a",), {"custom_llm_provider": "openai"}), + ), + ( + lambda r: r.registry().resolve("rg", kind="routing_group_alternatives"), + "_core_routing_group_has_alternatives", + (("rg",), {}), + ), + ( + lambda r: r.registry().resolve("gpt", kind="served", served_model="gpt-4o"), + "_core_resolve_served_model_name", + (("gpt", "gpt-4o"), {}), + ), + ( + lambda r: r.registry().resolve("gpt", kind="upstream", team_id="t"), + "_core_get_routable_upstream_model", + (("gpt",), {"team_id": "t"}), + ), + ( + lambda r: r.registry().resolve("gpt", kind="search_tool_support", team_id="t"), + "_core_get_search_tool_support", + (("gpt",), {"team_id": "t"}), + ), + ( + lambda r: r.registry().resolve("grp", kind="wildcard_access_group"), + "_core_is_model_access_group_for_wildcard_route", + ((), {"model_access_group": "grp"}), + ), + ( + lambda r: r.registry().resolve("dep-a", kind="team_model_alternatives"), + "_core_team_model_has_alternatives", + (("dep-a",), {}), + ), + ( + lambda r: r.registry().group("gpt", include_hidden=True, team_id="t"), + "_core_get_model_group_info", + (("gpt",), {"include_hidden": True, "team_id": "t"}), + ), + (lambda r: r.registry().group("gpt", field="cached_info"), "_core_cached_model_group_info", (("gpt",), {})), + (lambda r: r.registry().group("gpt", field="display_name"), "_core_get_configured_display_name", (("gpt",), {})), + ( + lambda r: r.registry().group("gpt", field="service_tiers", team_id="t"), + "_core_get_configured_service_tiers", + (("gpt",), {"team_id": "t"}), + ), + (lambda r: r.registry().group("gpt", field="listing_info"), "_core_get_model_listing_info", (("gpt",), {})), + (lambda r: r.registry().plugins("budget_limiter"), "_core_get_router_deployment_budget_limiter", ((), {})), + (lambda r: r.settings().allowed_fails_for(ValueError("x")), "_core_get_allowed_fails_from_policy", None), + (lambda r: r.settings().guardrail("g"), "_core_get_available_guardrail", (("g",), {})), + (lambda r: r.health().team_ptu_ceiling("t", "gpt"), "_core_team_ptu_ceiling", (("t", "gpt"), {})), + ( + lambda r: r.health().split_priority("gpt", {"priority": 1}), + "_core_split_request_priority", + (("gpt", {"priority": 1}), {}), + ), +] + + +@pytest.mark.parametrize("call, core, expected", FORWARDS, ids=[f[1] for f in FORWARDS]) +def test_entry_point_forwards_to_its_core_member(router, monkeypatch, call, core, expected): + from unittest.mock import MagicMock + + mock = MagicMock(return_value=_SENTINEL) + monkeypatch.setattr(router, core, mock) + assert call(router) is _SENTINEL + if expected is not None: + assert mock.call_args == expected + else: + mock.assert_called_once() + + +ASYNC_FORWARDS = [ + (lambda r: r.health().fully_unhealthy_model_names(), "_core_async_get_fully_unhealthy_model_names", ((), {})), + (lambda r: r.health().usage("gpt"), "_core_get_model_group_usage", ((), {"model_group": "gpt"})), + ( + lambda r: r.health().usage("gpt", kind="remaining"), + "_core_get_remaining_model_group_usage", + ((), {"model_group": "gpt"}), + ), +] + + +@pytest.mark.parametrize("call, core, expected", ASYNC_FORWARDS, ids=[f[1] for f in ASYNC_FORWARDS]) +def test_async_entry_point_forwards_to_its_core_member(router, monkeypatch, call, core, expected): + from unittest.mock import AsyncMock + + mock = AsyncMock(return_value=_SENTINEL) + monkeypatch.setattr(router, core, mock) + assert asyncio.run(call(router)) is _SENTINEL + assert mock.call_args == expected + + +def test_registry_reads_without_a_core_forward(router): + registry = router.registry() + assert registry.zero_cost() is getattr(router, "_zero_cost_cache", None) + assert registry.includes(router.registry().deployment("dep-a").model_dump(), team_id=None) is True + + +@pytest.mark.parametrize( + "call, error", + [ + (lambda r: r.registry().deployments(source="named"), TypeError), + (lambda r: r.registry().deployments("gpt", source="nope"), ValueError), + (lambda r: r.registry().index("nope"), ValueError), + (lambda r: r.registry().names(kind="litellm_models"), TypeError), + (lambda r: r.registry().names("gpt", kind="nope"), ValueError), + (lambda r: r.registry().resolve("team-gpt", kind="team_model"), TypeError), + (lambda r: r.registry().resolve(None, kind="alias"), TypeError), + (lambda r: r.registry().resolve("gpt", kind="nope"), ValueError), + (lambda r: r.registry().group("gpt", field="nope"), ValueError), + (lambda r: r.registry().credentials("dep-a", form="nope"), ValueError), + (lambda r: r.registry().plugins("nope"), ValueError), + (lambda r: asyncio.run(r.health().usage("gpt", kind="nope")), ValueError), + (lambda r: RouterSettings().as_dict(), AttributeError), + ], +) +def test_entry_points_reject_unknown_kinds_and_missing_names(router, call, error): + with pytest.raises(error): + call(router) + + +def test_health_is_blocked_and_the_old_static_name_agree(): + paused = Deployment( + model_name="m", + litellm_params={"model": "openai/gpt-4o-mini"}, # pyright: ignore[reportArgumentType] + model_info={"id": "d1", "blocked": True}, # pyright: ignore[reportArgumentType] + ) + from litellm.router_api import RouterHealth + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + assert Router.is_deployment_blocked(paused) is RouterHealth.is_blocked(paused) + + +def test_old_members_refuse_objects_that_are_not_routers(): + from litellm.router_deprecated import _as_router + + with pytest.raises(TypeError, match=r"is not a litellm\.Router"): + _as_router(object()) # pyright: ignore[reportArgumentType] + + +def test_direct_call_of_an_async_subclass_override_runs_it_once(): + calls = [] + + class Sub(litellm.Router): + async def async_pre_routing_hook( + self, model, request_kwargs, messages=None, input=None, specific_deployment=False + ): + calls.append(model) + return await super().async_pre_routing_hook(model, request_kwargs, messages, input, specific_deployment) + + r = Sub(model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-x"}}]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + asyncio.run(r.async_pre_routing_hook("m", {})) + assert calls == ["m"] + + +def test_routing_context_forwards_to_the_router(router, monkeypatch): + from unittest.mock import AsyncMock, MagicMock + + from litellm.router_api import RoutingContext + + ctx = RoutingContext(router) + assert ctx.registry().version == router.registry().version + assert ctx.settings() is router.settings() + assert isinstance(ctx.health(), type(router.health())) + for name in ( + "_pre_call_checks", + "_count_pre_call_check_tokens", + "_update_kwargs_before_fallbacks", + "_core_arm_routing_read_prefetch", + "_core_log_retry", + ): + monkeypatch.setattr(router, name, MagicMock(return_value=_SENTINEL)) + monkeypatch.setattr(router, "_core_async_pre_routing_hook", AsyncMock(return_value=_SENTINEL)) + assert ctx.pre_call_checks("gpt", []) is _SENTINEL + assert ctx.count_pre_call_check_tokens(None, "hi") is _SENTINEL + assert ctx.update_kwargs_before_fallbacks("gpt", {}) is _SENTINEL + assert ctx.arm_read_prefetch("gpt") is _SENTINEL + assert asyncio.run(ctx.pre_routing_hook("gpt", {})) is _SENTINEL + assert ctx.log_retry({}, ValueError("x")) is _SENTINEL + + +def test_attribute_errors_about_other_objects_pass_through_unchanged(router): + from litellm.router_deprecated import _public_attribute_error + + other = AttributeError("'dict' object has no attribute 'x'", name="x", obj={}) + assert _public_attribute_error(router, "_core_x", "x", other) is other diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 513136ae150..5f1b6032ca8 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -1158,7 +1158,7 @@ async def test_set_response_headers_passes_through_post_increment_counters(model _hidden_params: dict = {} router = Router(model_list=model_list) - router.get_remaining_model_group_usage = AsyncMock( + router._core_get_remaining_model_group_usage = AsyncMock( return_value={ "x-ratelimit-remaining-tokens": 958, "x-ratelimit-limit-tokens": 1000, @@ -1539,7 +1539,7 @@ async def test_get_remaining_model_group_usage_merges_io_and_tpm_headers(model_l ) ) router.get_model_group_io_token_usage = AsyncMock(return_value=(100, 40)) - router.get_model_group_usage = AsyncMock(return_value=(500, 5)) + router._core_get_model_group_usage = AsyncMock(return_value=(500, 5)) headers = await router.get_remaining_model_group_usage("gpt-3.5-turbo") @@ -1566,7 +1566,7 @@ async def test_set_response_headers_native_input_token_header_does_not_suppress_ _hidden_params: dict = {} router = Router(model_list=model_list) - router.get_remaining_model_group_usage = AsyncMock( + router._core_get_remaining_model_group_usage = AsyncMock( return_value={ "x-ratelimit-remaining-tokens": 1000, "x-ratelimit-remaining-requests": 100, @@ -1596,7 +1596,7 @@ async def test_set_response_headers_native_token_header_does_not_suppress_io_hea _hidden_params: dict = {} router = Router(model_list=model_list) - router.get_remaining_model_group_usage = AsyncMock( + router._core_get_remaining_model_group_usage = AsyncMock( return_value={ "x-ratelimit-remaining-tokens": 1000, "x-ratelimit-remaining-requests": 100, @@ -1628,7 +1628,7 @@ async def test_set_response_headers_handles_missing_usage(model_list): _hidden_params: dict = {} router = Router(model_list=model_list) - router.get_remaining_model_group_usage = AsyncMock( + router._core_get_remaining_model_group_usage = AsyncMock( return_value={ "x-ratelimit-remaining-tokens": 1000, "x-ratelimit-remaining-requests": 100, @@ -1648,7 +1648,7 @@ async def test_set_response_headers_handles_missing_usage(model_list): async def test_set_response_headers_dict_anthropic_messages_response(model_list): """Anthropic /v1/messages returns a dict; IO rate-limit headers must attach.""" router = Router(model_list=model_list) - router.get_remaining_model_group_usage = AsyncMock( + router._core_get_remaining_model_group_usage = AsyncMock( return_value={ "x-ratelimit-limit-input-tokens": 25, "x-ratelimit-remaining-input-tokens": 20, @@ -1688,7 +1688,7 @@ async def test_set_response_headers_wraps_bare_async_generator(model_list): yield {"type": "message_stop"} router = Router(model_list=model_list) - router.get_remaining_model_group_usage = AsyncMock( + router._core_get_remaining_model_group_usage = AsyncMock( return_value={ "x-ratelimit-limit-input-tokens": 25, "x-ratelimit-remaining-input-tokens": 20,