From 1e08643bd367ba54858baef0e3acbab95cc3a861 Mon Sep 17 00:00:00 2001 From: narmstrong-berry Date: Mon, 2 Nov 2026 04:59:06 +0000 Subject: [PATCH] refactor(router): route every router read of litellm settings and functions through the host Both router cores and the Python helpers they share (cooldown handlers and callbacks, budget limiter, lowest-cost strategy, complexity router, savings baseline, continuation prefill, io-token and model rate-limit checks, request priority, fallback event handlers) now read process settings through host.dynamic() and call litellm functions through host.invoke(). RustCore's allowed_fails default comes from the host too. A guard test fails if any router module reads litellm. again. --- litellm/router_api.py | 11 ++++ litellm/router_config.py | 10 ++++ litellm/router_python_core.py | 20 +++++-- litellm/router_rust_callbacks.py | 4 +- litellm/router_rust_core.py | 27 +++++---- litellm/router_rust_endpoints.py | 1 + litellm/router_rust_selection.py | 15 +++-- litellm/router_rust_streaming.py | 4 +- litellm/router_rust_walk.py | 10 ++-- litellm/router_strategy/budget_limiter.py | 29 +++++++--- .../complexity_router/complexity_router.py | 18 ++++-- .../complexity_router/jev_classifier.py | 26 ++++++--- litellm/router_strategy/lowest_cost.py | 32 ++++++----- litellm/router_strategy/savings_baseline.py | 15 +++-- litellm/router_utils/cooldown_callbacks.py | 22 ++++--- litellm/router_utils/cooldown_handlers.py | 8 ++- .../router_utils/fallback_event_handlers.py | 57 +++++++++++++------ .../continuation_prefill_check.py | 14 +++-- .../io_token_rate_limit_check.py | 33 ++++++++--- .../pre_call_checks/model_rate_limit_check.py | 13 ++++- litellm/router_utils/request_priority.py | 13 ++++- .../complexity_router/test_jev_classifier.py | 4 +- .../test_budget_limiter_hotpath.py | 2 +- ...est_fallbacks_characterization_outcomes.py | 8 +-- .../core/test_gaps_router_core_crate_picks.py | 4 +- .../core/test_retries_cooldowns.py | 17 ++++++ .../core/test_router_host_boundary.py | 54 ++++++++++++++++++ 27 files changed, 349 insertions(+), 122 deletions(-) create mode 100644 tests/unit/test_router/core/test_router_host_boundary.py diff --git a/litellm/router_api.py b/litellm/router_api.py index da25ef2eb26..97b492488a5 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -59,6 +59,7 @@ if TYPE_CHECKING: from litellm.messages.dispatch import MessagesResult from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.router import Router + from litellm.router_python_host import PythonHost from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter from litellm.router_strategy.auto_router.auto_router import AutoRouter from litellm.router_strategy.budget_limiter import RouterBudgetLimiting @@ -1474,6 +1475,16 @@ class RoutingContext: def belongs_to(self, router: "Router") -> bool: return self._router is router or router.__dict__.get("_core") is self._router + def host(self) -> "PythonHost": + """The router core's host: the process settings (``dynamic()``) and litellm functions + (``invoke``) a plugin reads go through it, never ``litellm.*`` directly.""" + from litellm.router_python_host import PythonHost, python_host + + core: Final[object] = self._router.__dict__.get("_core", self._router) # pyright: ignore[reportAny] # __dict__ is untyped + host: Final[object] = getattr(core, "_host", None) + # Every core keeps its host as _host; a stand-in router without one reads the process host. + return host if isinstance(host, PythonHost) else python_host + @property def explicit_allowed_fails(self) -> int | None: """``allowed_fails`` as the caller passed it, ``None`` when it came from the global default.""" diff --git a/litellm/router_config.py b/litellm/router_config.py index 2227ab3bacb..a83ac459f85 100644 --- a/litellm/router_config.py +++ b/litellm/router_config.py @@ -18,6 +18,7 @@ import litellm if TYPE_CHECKING: from litellm.caching.caching import Cache from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager + from litellm.types.utils import BudgetConfig @dataclass(frozen=True, slots=True) @@ -59,6 +60,9 @@ class RouterDynamicConfig: callback_manager: "LoggingCallbackManager" cache: "Cache | None" request_timeout_explicitly_set: bool = False + tag_budget_config: "Mapping[str, BudgetConfig] | None" = None + disable_token_counter: bool = False + post_call_rules: Sequence[object] = () _versions: Final = itertools.count(1) @@ -112,6 +116,9 @@ def current_dynamic_config() -> RouterDynamicConfig: callback_manager=litellm.logging_callback_manager, cache=litellm.cache, request_timeout_explicitly_set=litellm.request_timeout_explicitly_set, + tag_budget_config=litellm.tag_budget_config, + disable_token_counter=litellm.disable_token_counter, + post_call_rules=litellm.post_call_rules, ) @@ -141,6 +148,9 @@ _LIVE_SOURCES: Final = MappingProxyType( "input_callback": "input_callback", "callback_manager": "logging_callback_manager", "cache": "cache", + "tag_budget_config": "tag_budget_config", + "disable_token_counter": "disable_token_counter", + "post_call_rules": "post_call_rules", } ) diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 7b94fa5a244..119d4731076 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -2185,7 +2185,7 @@ class PythonCore(DeprecatedRouterMembers): self._core_provider_budget_config = provider_budget_config self.router_budget_logger: RouterBudgetLimiting | None = None if RouterBudgetLimiting.should_init_router_budget_limiter( - model_list=model_list, provider_budget_config=self._core_provider_budget_config + model_list=model_list, provider_budget_config=self._core_provider_budget_config, host=self._host ): if optional_pre_call_checks is not None: optional_pre_call_checks.append("router_budget_limiting") @@ -2473,6 +2473,7 @@ class PythonCore(DeprecatedRouterMembers): selector = LowestCostLoggingHandler( router_cache=self._core_cache, routing_args={}, + host=self._host, ) case _: pass @@ -3204,10 +3205,11 @@ class PythonCore(DeprecatedRouterMembers): dual_cache=self._core_cache, provider_budget_config=self._core_provider_budget_config, model_list=self._core_model_list, + host=self._host, ) self.router_budget_logger = _callback elif pre_call_check == "enforce_model_rate_limits": - _callback = ModelRateLimitingCheck(dual_cache=self._core_cache) + _callback = ModelRateLimitingCheck(dual_cache=self._core_cache, host=self._host) if _callback is None: continue @@ -3898,6 +3900,7 @@ class PythonCore(DeprecatedRouterMembers): complete_response_object, checks_post_call_rules=initial_kwargs.get(MID_STREAM_CONTINUATION_KWARG) is not MID_STREAM_CONTINUATION_MARKER, + host=self._host, ) if continuation_messages is not None else None @@ -4177,7 +4180,7 @@ class PythonCore(DeprecatedRouterMembers): request_kwargs, requested_group if isinstance(requested_group, str) else None ) targets: Final = continuation_fallback_targets( - request_kwargs.get("fallbacks", self._core_fallbacks), lookup_groups + request_kwargs.get("fallbacks", self._core_fallbacks), lookup_groups, self._host.dynamic().provider_list ) if any(fallback_target_overrides_request_params(target) for target in targets): return False @@ -5740,7 +5743,10 @@ class PythonCore(DeprecatedRouterMembers): def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool: return request_drops_params( - kwargs, self._core_default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs) + kwargs, + self._core_default_litellm_params, + self._request_deployment_params(model=model, kwargs=kwargs), + self._host, ) async def _schedule_factory( @@ -8852,6 +8858,7 @@ class PythonCore(DeprecatedRouterMembers): external_fallback_group, generic_idx = get_fallback_model_group_for_lookup_groups( fallbacks=fallbacks, lookup_groups=lookup_groups, + provider_list=self._host.dynamic().provider_list, ) if external_fallback_group is None and generic_idx is not None: external_fallback_group = fallbacks[generic_idx]["*"] @@ -8981,6 +8988,7 @@ class PythonCore(DeprecatedRouterMembers): ) = get_fallback_model_group_for_lookup_groups( fallbacks=fallbacks, # if fallbacks = [{"gpt-3.5-turbo": ["claude-3-haiku"]}] lookup_groups=lookup_groups, + provider_list=self._host.dynamic().provider_list, ) ## if none, check for generic fallback if fallback_model_group is None and generic_fallback_idx is not None: @@ -9537,7 +9545,7 @@ class PythonCore(DeprecatedRouterMembers): lookup_groups: tuple[str, ...], ) -> list[str] | None: # mutable-ok: mirrors the shared resolver's contract fallback_model_group, _ = get_fallback_model_group_for_lookup_groups( - fallbacks=fallbacks, lookup_groups=lookup_groups + fallbacks=fallbacks, lookup_groups=lookup_groups, provider_list=self._host.dynamic().provider_list ) return fallback_model_group @@ -10143,6 +10151,7 @@ class PythonCore(DeprecatedRouterMembers): resolved, _ = get_fallback_model_group_for_lookup_groups( fallbacks=fallbacks, lookup_groups=fallback_lookup_groups(kwargs, model_group), + provider_list=self._host.dynamic().provider_list, ) return has_unattempted_fallback_target(resolved, kwargs) @@ -10193,6 +10202,7 @@ class PythonCore(DeprecatedRouterMembers): resolved, _ = get_fallback_model_group_for_lookup_groups( fallbacks=fallbacks, lookup_groups=fallback_lookup_groups(kwargs, model_group), + provider_list=self._host.dynamic().provider_list, ) return has_unattempted_fallback_target(resolved, kwargs) diff --git a/litellm/router_rust_callbacks.py b/litellm/router_rust_callbacks.py index 3fd94d89c3a..15bc22420f3 100644 --- a/litellm/router_rust_callbacks.py +++ b/litellm/router_rust_callbacks.py @@ -143,6 +143,7 @@ class OptionalChecks: if RouterBudgetLimiting.should_init_router_budget_limiter( # pyright: ignore[reportUnknownMemberType] # untyped model_list model_list=model_list if isinstance(model_list, list) else None, # pyright: ignore[reportUnknownArgumentType] # the argument as passed provider_budget_config=self.provider_budget_config, # pyright: ignore[reportArgumentType] # the argument as passed + host=self._host, ): if listed is not None: listed.append("router_budget_limiting") @@ -255,10 +256,11 @@ class OptionalChecks: dual_cache=self._cache, provider_budget_config=self.provider_budget_config, # pyright: ignore[reportArgumentType] # the argument as passed model_list=self._hosted._core_model_list, # pyright: ignore[reportPrivateUsage, reportArgumentType] # the live model_list + host=self._host, ) self.router_budget_logger = callback elif name == "enforce_model_rate_limits": - callback = ModelRateLimitingCheck(dual_cache=self._cache) + callback = ModelRateLimitingCheck(dual_cache=self._cache, host=self._host) if callback is None: continue if self.optional_callbacks is not None and any( diff --git a/litellm/router_rust_core.py b/litellm/router_rust_core.py index 4d252cbbcba..17f727a3cb7 100644 --- a/litellm/router_rust_core.py +++ b/litellm/router_rust_core.py @@ -721,7 +721,10 @@ class Resolved: # An explicit `routing_strategy=None` reaches the Rust core as "None": no strategy picks. return kept if self.routing_strategy is not None else {**kept, "routing_strategy": "None"} - def dynamic_document(self, version: int, model_names: Sequence[str]) -> Mapping[str, object]: + def dynamic_document( + self, version: int, model_names: Sequence[str], process_allowed_fails: object + ) -> Mapping[str, object]: + """``process_allowed_fails`` is the process default, read through the host.""" values: Final = { "version": version, "model_group_alias": _plain(dict(self.model_group_alias)), @@ -730,21 +733,19 @@ class Resolved: "content_policy_fallbacks": _plain(self.content_policy_fallbacks), "num_retries": _crate_int(self.num_retries), "max_fallbacks": _crate_int(self.max_fallbacks), - "allowed_fails": _crate_int(self._router_allowed_fails()), + "allowed_fails": _crate_int(self._router_allowed_fails(process_allowed_fails)), "fallback_keys": self._fallback_keys(model_names), "default_deployment": _plain(served_default_deployment(self.default_deployment)[0]), "default_deployment_raises": True if served_default_deployment(self.default_deployment)[1] else None, } return {name: value for name, value in values.items() if value is not None} - def _router_allowed_fails(self) -> object: + def _router_allowed_fails(self, process_allowed_fails: object) -> object: """``allowed_fails`` when it counts as set on the router (``_is_allowed_fails_set_on_router``): - passed explicitly, or different from the process default.""" - import litellm - + passed explicitly, or different from the process default ``process_allowed_fails``.""" if self.explicit_allowed_fails is not None: return self.explicit_allowed_fails - if self.allowed_fails != litellm.allowed_fails: + if self.allowed_fails != process_allowed_fails: return self.allowed_fails return None @@ -1218,7 +1219,9 @@ class RustCore( **{name: value for name, value in failover_document(self._plain).items() if value is not None}, **({"provider_default_deployment_ids": list(default_ids)} if default_ids else {}), } - return RouterCoreHandle(static, self._resolved.dynamic_document(self._version, names)) + return RouterCoreHandle( + static, self._resolved.dynamic_document(self._version, names, self._host.dynamic().allowed_fails) + ) def _provider_default_ids(self) -> tuple[str, ...]: """``provider_default_deployment_ids`` as the cooldown check reads it (``deployment in @@ -1609,7 +1612,9 @@ class RustCore( self._rebuild() else: names: Final = [str(d.get("model_name")) for d in self._handle.deployments()] - self._handle.apply({"settings": resolved.dynamic_document(self._version, names)}) + self._handle.apply( + {"settings": resolved.dynamic_document(self._version, names, self._host.dynamic().allowed_fails)} + ) def _changes_strategy(self, updated: Mapping[str, object]) -> bool: """``update_settings`` names a ``routing_strategy`` other than the router's.""" @@ -3016,7 +3021,7 @@ class _Driver: self._served: tuple[str | None, Mapping[str, object]] = (None, MappingProxyType({})) self._attempts: int = 0 self._walk: Final = WalkMetadata(request, plan.metadata_field, self._params_base_model) - self.outcomes: Final = FallbackOutcomes(request) + self.outcomes: Final = FallbackOutcomes(request, core._host) # pyright: ignore[reportPrivateUsage] # the driver serves its own core # The fallback level the walk is at, from its fallback events. self._level: int = 0 @@ -3581,7 +3586,7 @@ class _Driver: source: Final = own if own is not None else fallbacks entries: Final = [dict(m) for entry in _as_list(source) if (m := as_str_mapping(entry))] found, generic = ( - get_fallback_model_group_for_lookup_groups(entries, lookup) # pyright: ignore[reportArgumentType] # the fallbacks config shape + get_fallback_model_group_for_lookup_groups(entries, lookup, self._core._host.dynamic().provider_list) # pyright: ignore[reportArgumentType, reportPrivateUsage] # the fallbacks config shape; the driver's own core's host if entries else (None, None) ) diff --git a/litellm/router_rust_endpoints.py b/litellm/router_rust_endpoints.py index 07f091d58e1..73d9a486e55 100644 --- a/litellm/router_rust_endpoints.py +++ b/litellm/router_rust_endpoints.py @@ -1306,6 +1306,7 @@ class RustEndpoints: kwargs, self._core._resolved.default_litellm_params, # pyright: ignore[reportPrivateUsage] # see above deployment_params(), + self._core._host, # pyright: ignore[reportPrivateUsage] # see above ) def _split_priority( diff --git a/litellm/router_rust_selection.py b/litellm/router_rust_selection.py index bb04b48403a..208a391169d 100644 --- a/litellm/router_rust_selection.py +++ b/litellm/router_rust_selection.py @@ -58,7 +58,9 @@ def normalize_strategy(strategy: object) -> str | None: return strategy if isinstance(strategy, str) else None -def build_selector(strategy: str | None, args: Mapping[str, object], cache: "DualCache") -> Selector | None: +def build_selector( + strategy: str | None, args: Mapping[str, object], cache: "DualCache", host: "PythonHost" +) -> Selector | None: """``_build_strategy_selector`` without registering: ``None`` for simple-shuffle and every strategy without a selector.""" routing_args: Final = dict(args) @@ -72,7 +74,7 @@ def build_selector(strategy: str | None, args: Mapping[str, object], cache: "Dua case "latency-based-routing": return LowestLatencyLoggingHandler(router_cache=cache, routing_args=routing_args) case "cost-based-routing": - return LowestCostLoggingHandler(router_cache=cache, routing_args={}) + return LowestCostLoggingHandler(router_cache=cache, routing_args={}, host=host) case _: return None @@ -122,7 +124,7 @@ class StrategySelectors: self._unregister([self._default, *self._overrides.values()]) self._overrides = {} self._strategy = normalize_strategy(strategy) - self._default = build_selector(self._strategy, args, self._cache) + self._default = build_selector(self._strategy, args, self._cache, self._host) if self._default is not None: self._register(self._default) @@ -136,7 +138,7 @@ class StrategySelectors: if self._default is None: return try: - rebuilt: Final = build_selector(self._strategy, args, self._cache) + rebuilt: Final = build_selector(self._strategy, args, self._cache, self._host) except (TypeError, ValidationError): verbose_router_logger.exception( "Invalid routing_strategy_args %s for '%s'; keeping the previous ones", dict(args), self._strategy @@ -153,7 +155,8 @@ class StrategySelectors: built: Final = { name: selector for name, strategy, args in groups - if (selector := build_selector(None if strategy == "priority" else strategy, args, self._cache)) is not None + if (selector := build_selector(None if strategy == "priority" else strategy, args, self._cache, self._host)) + is not None } for selector in built.values(): self._register(selector) @@ -211,7 +214,7 @@ class StrategySelectors: return self._default with self._lock: if strategy not in self._overrides: - self._overrides[strategy] = build_selector(strategy, {}, self._cache) + self._overrides[strategy] = build_selector(strategy, {}, self._cache, self._host) selector: Final = self._overrides[strategy] if selector is not None and request is not None and strategy not in self._registered_strategies(): from litellm.litellm_core_utils.litellm_logging import Logging diff --git a/litellm/router_rust_streaming.py b/litellm/router_rust_streaming.py index 02203cf80d1..5553147b3d7 100644 --- a/litellm/router_rust_streaming.py +++ b/litellm/router_rust_streaming.py @@ -423,7 +423,8 @@ def _mid_stream_continuation_eligible( lookup_groups: Final = fallback_lookup_groups( request, requested_group if isinstance(requested_group, str) else None ) - targets: Final = continuation_fallback_targets(core.settings().fallbacks, lookup_groups) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # fallbacks as Python spells them + providers: Final = core._host.dynamic().provider_list # pyright: ignore[reportPrivateUsage] # the core's own host + targets: Final = continuation_fallback_targets(core.settings().fallbacks, lookup_groups, providers) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # fallbacks as Python spells them if any(fallback_target_overrides_request_params(target) for target in targets): return False if any(_names_deployment_that_cannot_continue(core, target) for target in targets): @@ -561,6 +562,7 @@ async def _chat_continuation( str(request.get("model") or ""), complete, checks_post_call_rules=request.get(MID_STREAM_CONTINUATION_KWARG) is not MID_STREAM_CONTINUATION_MARKER, + host=attempt.core._host, # pyright: ignore[reportPrivateUsage] # the attempt's own core's host ) if continuation_messages is not None else None diff --git a/litellm/router_rust_walk.py b/litellm/router_rust_walk.py index 0051b0faa79..fdfd613b792 100644 --- a/litellm/router_rust_walk.py +++ b/litellm/router_rust_walk.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Final from litellm.litellm_core_utils.core_helpers import as_str_mapping if TYPE_CHECKING: + from litellm.router_python_host import PythonHost from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets # `RETRY_BREADCRUMB_LIMIT`: how many failures `previous_models` keeps. @@ -225,9 +226,10 @@ class FallbackOutcomes: custom loggers and, under ``include_fallback_errors``, lists the level's errors (its own first, then each hop that failed) in the response's ``x-litellm-fallback-errors``.""" - __slots__ = ("_failed", "_header_errors", "_include", "_pending") + __slots__ = ("_failed", "_header_errors", "_host", "_include", "_pending") - def __init__(self, request: Mapping[str, object]) -> None: + def __init__(self, request: Mapping[str, object], host: "PythonHost") -> None: + self._host: Final = host self._include: Final = request.get("include_fallback_errors", False) is True self._failed: Final[dict[object, list[object]]] = {} # mutable-ok: each level's failed hops, in order self._header_errors: Final[list[list[object]]] = [] # mutable-ok: each successful hop's errors, innermost first @@ -253,7 +255,7 @@ class FallbackOutcomes: self._header_errors.append([get_fallback_error_info(level_error), *failed]) self._pending.append( handlers.log_success_fallback_event( # pyright: ignore[reportUnknownMemberType] # takes the kwargs as an untyped dict - original_model_group=original, kwargs=dict(kwargs), original_exception=level_error + original_model_group=original, kwargs=dict(kwargs), original_exception=level_error, host=self._host ) ) return @@ -261,7 +263,7 @@ class FallbackOutcomes: failed.append(get_fallback_error_info(hop_error)) self._pending.append( handlers.log_failure_fallback_event( # pyright: ignore[reportUnknownMemberType] # see above - original_model_group=original, kwargs=dict(kwargs), original_exception=level_error + original_model_group=original, kwargs=dict(kwargs), original_exception=level_error, host=self._host ) ) diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 7da1657fa21..5adab5c68cc 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -26,11 +26,10 @@ from collections.abc import Callable, Mapping, Sequence from datetime import datetime, timedelta, timezone from itertools import groupby from types import MappingProxyType -from typing import Any, Final +from typing import TYPE_CHECKING, Any, Final from pydantic import TypeAdapter -import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache @@ -50,6 +49,9 @@ from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, RouterErro from litellm.types.utils import BudgetConfig, GenericBudgetConfigType, StandardLoggingPayload from litellm.types.utils import BudgetConfig as GenericBudgetInfo +if TYPE_CHECKING: + from litellm.router_python_host import PythonHost + DEFAULT_REDIS_SYNC_INTERVAL: Final = 1 _CALLBACK_FIELDS: Final = TypeAdapter(dict[str, object]) _BATCH_CHARGE_VALUES: Final = TypeAdapter(tuple[object, float]) @@ -121,7 +123,12 @@ class RouterBudgetLimiting(CustomLogger): provider_budget_config: dict | None, model_list: list[DeploymentTypedDict | dict[str, Any]] | None = None, clock: Callable[[], float] = time.time, + host: "PythonHost | None" = None, ): + from litellm.router_python_host import python_host + + # The router core's host: the process settings and litellm functions are read through it. + self._host: Final = python_host if host is None else host self.dual_cache = dual_cache self._clock = clock self.redis_increment_operation_queue: list[RedisPipelineIncrementOperation] = [] @@ -141,8 +148,8 @@ class RouterBudgetLimiting(CustomLogger): self._start_background_work() # Add self to litellm callbacks if it's a list - if isinstance(litellm.callbacks, list): - litellm.logging_callback_manager.add_litellm_callback(self) + if isinstance(self._host.dynamic().callbacks, list): + self._host.callbacks.add_litellm_callback(self) def _start_background_work(self) -> None: """Start the periodic Redis sync and the provider budget cache seeding once a loop runs. @@ -810,7 +817,7 @@ class RouterBudgetLimiting(CustomLogger): model = "" provider_resolution_params = _LiteLLMParamsDictView({}) - _, custom_llm_provider, _, _ = litellm.get_llm_provider( + _, custom_llm_provider, _, _ = self._host.invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # the host op table model=str(model), litellm_params=provider_resolution_params, ) @@ -826,7 +833,7 @@ class RouterBudgetLimiting(CustomLogger): This is helpful for debugging and monitoring provider budget limits. """ - prometheus_logger: Final = get_prometheus_logger_from_callbacks() + prometheus_logger: Final = get_prometheus_logger_from_callbacks(self._host) if prometheus_logger: prometheus_logger.track_provider_remaining_budget( provider=provider, @@ -906,6 +913,7 @@ class RouterBudgetLimiting(CustomLogger): def should_init_router_budget_limiter( provider_budget_config: dict | None, model_list: list[DeploymentTypedDict] | list[dict[str, Any]] | None = None, + host: "PythonHost | None" = None, ): """ Returns `True` if the router budget routing settings are set and RouterBudgetLimiting should be initialized @@ -918,7 +926,9 @@ class RouterBudgetLimiting(CustomLogger): if provider_budget_config is not None: return True - if litellm.tag_budget_config is not None: + from litellm.router_python_host import python_host + + if (python_host if host is None else host).dynamic().tag_budget_config is not None: return True if model_list is None: @@ -995,7 +1005,8 @@ class RouterBudgetLimiting(CustomLogger): self.deployment_budget_config = None def _init_tag_budgets(self): - if litellm.tag_budget_config is None: + tag_budget_config: Final = self._host.dynamic().tag_budget_config + if tag_budget_config is None: return from litellm.proxy.proxy_server import CommonProxyErrors, premium_user @@ -1005,7 +1016,7 @@ class RouterBudgetLimiting(CustomLogger): if self.tag_budget_config is None: self.tag_budget_config = {} - for _tag, _tag_budget_config in litellm.tag_budget_config.items(): + for _tag, _tag_budget_config in tag_budget_config.items(): if isinstance(_tag_budget_config, dict): _tag_budget_config = BudgetConfig(**_tag_budget_config) _generic_budget_config = GenericBudgetInfo( diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 4f4ed76c170..f3bd63131c2 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2410,7 +2410,12 @@ class ComplexityRouter(CustomLogger): ) budgeted: Final = await offload_token_count(fit_jev_request)( - build_request, prompt, system_prompt, jev_input_targets(self.ctx, config), _truncate + build_request, + prompt, + system_prompt, + jev_input_targets(self.ctx, config), + _truncate, + self.ctx.host().dynamic().disable_token_counter, ) request: Final = budgeted.request if request is None: @@ -2457,7 +2462,10 @@ class ComplexityRouter(CustomLogger): cost=response.classifier_cost if config.deployment_name else jev_classifier_cost( - response, config.model, "typesafe" if config.provider == "jev" else config.provider + response, + config.model, + "typesafe" if config.provider == "jev" else config.provider, + cost_map=self.ctx.host().dynamic().model_cost, ), provider=accounting_provider, ) @@ -3597,15 +3605,15 @@ class ComplexityRouter(CustomLogger): ) -> int | None: """Real-tokenizer count of the resolved messages plus the out-of-band carriers, off the event loop; None when counting fails, and the gate then leaves the placement alone.""" - import litellm from litellm.litellm_core_utils.token_counter import offload_token_count + token_counter: Final = self.ctx.host().invoke("token_counter") # pyright: ignore[reportUnknownMemberType] # the host op table out_of_band: Final = self._out_of_band_request_text(request_kwargs) try: - counted: Final = await offload_token_count(litellm.token_counter)( + counted: Final = await offload_token_count(token_counter)( messages=cast(list, resolved_messages) # cast-ok: token_counter only iterates the sequence ) - return counted + (await offload_token_count(litellm.token_counter)(text=out_of_band) if out_of_band else 0) + return counted + (await offload_token_count(token_counter)(text=out_of_band) if out_of_band else 0) except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request verbose_router_logger.debug("ComplexityRouter: context-window token count failed. Got - %s", e) return None diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index bb7d2e91cd3..140ff39a579 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -8,7 +8,6 @@ from uuid import uuid4 import httpx from pydantic import ConfigDict, Field, TypeAdapter, ValidationError -import litellm from litellm._logging import verbose_router_logger from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.litellm_core_utils.classifier_logging import masked_originating_request @@ -115,12 +114,14 @@ def _jev_input_target(params: Mapping[str, object], explicit_limit: object) -> J return JevInputTarget(qualified_model, None) -def _jev_request_count(request: JevSystemOneRequest, models: Sequence[str], budget: int) -> tuple[int, str]: +def _jev_request_count( + request: JevSystemOneRequest, models: Sequence[str], budget: int, disable_token_counter: bool +) -> tuple[int, str]: from litellm._lazy_imports import get_token_counter_new payload: Final = request.model_dump_json(exclude={"model"}) byte_count: Final = len(payload.encode("utf-8")) - if byte_count <= budget or litellm.disable_token_counter: + if byte_count <= budget or disable_token_counter: return byte_count, "utf8_bytes" try: counts: Final = tuple(get_token_counter_new()(model=model, text=payload) for model in models) @@ -137,7 +138,9 @@ def fit_jev_request( system_prompt: str | None, targets: Sequence[JevInputTarget], truncate: Callable[[str, int], str], + disable_token_counter: bool, ) -> JevBudgetedRequest: + """``disable_token_counter`` is the process setting, read through the router core's host.""" known: Final = tuple(target for target in targets if target.max_tokens is not None) if not known: return JevBudgetedRequest( @@ -158,6 +161,7 @@ def fit_jev_request( truncate, len(known) != len(targets), bounded_prompt != prompt or bounded_system != system_prompt, + disable_token_counter, ) @@ -170,6 +174,7 @@ def _fit_jev_components( truncate: Callable[[str, int], str], unknown_limits: bool, truncated: bool, + disable_token_counter: bool, ) -> JevBudgetedRequest: attempts: Final = len(prompt).bit_length() + len(system_prompt or "").bit_length() + 1 components: tuple[str, str | None, bool, int] = ( # rebind-ok: bounded iterative fitting state @@ -181,7 +186,7 @@ def _fit_jev_components( while True: current_prompt, current_system, was_truncated, remaining = components request = build_request(current_prompt, current_system) - count, method = _jev_request_count(request, models, budget) + count, method = _jev_request_count(request, models, budget, disable_token_counter) telemetry: StandardLoggingRoutingDecision = { "classifier_input_budget_tokens": budget, "classifier_input_tokens": count, @@ -452,19 +457,22 @@ def build_jev_request( def jev_classifier_cost( - response: JevSystemOneResponse, configured_model: str, provider: ClassifierProvider = "typesafe" + response: JevSystemOneResponse, + configured_model: str, + provider: ClassifierProvider = "typesafe", + *, + cost_map: Mapping[str, object], ) -> float | None: + """The call's cost from ``cost_map`` (the process cost map, read through the router core's host).""" usage: Final = response.usage if usage is None: return None model: Final = response.model or configured_model model_key: Final = f"{provider}/{model}" - if model_key not in litellm.model_cost: # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed + if model_key not in cost_map: return None try: - pricing: Final = _REGISTRY_PRICING_ADAPTER.validate_python( - litellm.model_cost[model_key] # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed - ) + pricing: Final = _REGISTRY_PRICING_ADAPTER.validate_python(cost_map[model_key]) except ValidationError: return None return usage.input_tokens * pricing.input_cost_per_token + usage.output_tokens * pricing.output_cost_per_token diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index 035d18e0317..7bb48fdbab0 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -2,10 +2,9 @@ # picks based on response time (for streaming, this is time to first token) from collections.abc import Mapping from datetime import datetime -from typing import Final +from typing import TYPE_CHECKING, Final -import litellm -from litellm import ModelResponse, token_counter, verbose_logger +from litellm import ModelResponse, Usage, token_counter, verbose_logger from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache @@ -13,22 +12,21 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed +if TYPE_CHECKING: + from litellm.router_python_host import PythonHost -def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]: - """Input and output per-token prices from the first cost-map row among the deployment's + +def _deployment_cost_map_rates(deployment: object, cost_map: Mapping[str, object]) -> tuple[float | None, float | None]: + """Input and output per-token prices from the first ``cost_map`` row among the deployment's pricing models that carries one; (None, None) when no row does, so the caller falls back to its unmapped default.""" - rows: Final = (str_keyed(_cost_map().get(model)) for model in deployment_pricing_models(deployment)) + rows: Final = (str_keyed(cost_map.get(model)) for model in deployment_pricing_models(deployment)) row: Final = next((row for row in rows if _declares_token_price(row)), None) if row is None: return None, None return _as_rate(row.get("input_cost_per_token")), _as_rate(row.get("output_cost_per_token")) -def _cost_map() -> Mapping[str, object]: - return litellm.model_cost - - def _declares_token_price(row: Mapping[str, object]) -> bool: return row.get("input_cost_per_token") is not None or row.get("output_cost_per_token") is not None @@ -42,8 +40,12 @@ class LowestCostLoggingHandler(CustomLogger): logged_success: int = 0 logged_failure: int = 0 - def __init__(self, router_cache: DualCache, routing_args: dict = {}): + def __init__(self, router_cache: DualCache, routing_args: dict = {}, host: "PythonHost | None" = None): + from litellm.router_python_host import python_host + self.router_cache = router_cache + # The router core's host: the cost map is read through it. + self._host: Final = python_host if host is None else host @with_service_target("router_usage") def log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -86,7 +88,7 @@ class LowestCostLoggingHandler(CustomLogger): if isinstance(response_obj, ModelResponse): _usage: Final = getattr(response_obj, "usage", None) - if _usage is not None and isinstance(_usage, litellm.Usage): + if _usage is not None and isinstance(_usage, Usage): total_tokens = _usage.total_tokens # ------------ @@ -163,7 +165,7 @@ class LowestCostLoggingHandler(CustomLogger): if isinstance(response_obj, ModelResponse): _usage: Final = getattr(response_obj, "usage", None) - if _usage is not None and isinstance(_usage, litellm.Usage): + if _usage is not None and isinstance(_usage, Usage): total_tokens = _usage.total_tokens # ------------ @@ -286,7 +288,9 @@ class LowestCostLoggingHandler(CustomLogger): or float("inf") ) deployment_view: object = _deployment - map_input_cost, map_output_cost = _deployment_cost_map_rates(deployment_view) + map_input_cost, map_output_cost = _deployment_cost_map_rates( + deployment_view, self._host.dynamic().model_cost + ) # check if user provided input_cost_per_token and output_cost_per_token in litellm_params item_input_cost = None diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index 645ae92d8e6..eae79d0ed0d 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -23,6 +23,7 @@ from litellm.types.utils import PromptTokensDetailsWrapper, Usage if TYPE_CHECKING: from litellm.router_api import RoutingContext + from litellm.router_python_host import PythonHost _REFERENCE_REQUEST: Final = Usage( @@ -46,7 +47,7 @@ class Baseline(NamedTuple): deployment_id: str | None = None -def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None: +def canonical_model(model: str, custom_llm_provider: str | None = None, host: "PythonHost | None" = None) -> str | None: """``provider/model``, or ``None`` when the pair names no known provider. A deployment may name its vendor in the model prefix or in a separate @@ -56,15 +57,19 @@ def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | A github_copilot or chatgpt candidate is qualified by string alone: resolving either provider runs its OAuth device flow, and for a declared pair the resolver's answer is the declaration itself, so asking it buys nothing but the block. + + The provider is resolved through ``host`` (the router core's host; the process host when none is given). """ - import litellm from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider + from litellm.router_python_host import python_host declared: Final = declared_authenticating_provider(model, custom_llm_provider) if declared is not None: return f"{declared}/{model.removeprefix(f'{declared}/')}" try: - resolved, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + resolved, provider, _, _ = (python_host if host is None else host).invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # the host op table + model=model, custom_llm_provider=custom_llm_provider + ) except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline verbose_router_logger.debug("savings baseline: cannot resolve candidate %s (%s)", model, e) return None @@ -83,7 +88,7 @@ def _models_in(ctx: "RoutingContext", group_name: str) -> tuple[Baseline, ...]: """ deployments: Final = ctx.registry().deployments(group_name, source="named") if not deployments: - return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else () + return (Baseline(qualified),) if (qualified := canonical_model(group_name, host=ctx.host())) else () def candidate(deployment: Mapping[str, object]) -> Baseline | None: params: Final = deployment.get("litellm_params") @@ -92,7 +97,7 @@ def _models_in(ctx: "RoutingContext", group_name: str) -> tuple[Baseline, ...]: info: Final = deployment.get("model_info") base: Final = info.get("base_model") if isinstance(info, dict) else None model: Final = base or params.get("base_model") or params.get("model") - qualified: Final = canonical_model(model, params.get("custom_llm_provider")) if model else None + qualified: Final = canonical_model(model, params.get("custom_llm_provider"), ctx.host()) if model else None if qualified is None: return None deployment_id: Final = info.get("id") if isinstance(info, dict) else None diff --git a/litellm/router_utils/cooldown_callbacks.py b/litellm/router_utils/cooldown_callbacks.py index e80ef95e2f8..e3e69ba3bb9 100644 --- a/litellm/router_utils/cooldown_callbacks.py +++ b/litellm/router_utils/cooldown_callbacks.py @@ -5,13 +5,13 @@ Callbacks triggered on cooling down deployments import copy from typing import TYPE_CHECKING, Any, Final -import litellm from litellm._logging import verbose_logger from litellm.router_deprecated import renamed_router_param if TYPE_CHECKING: from litellm.integrations.prometheus import PrometheusLogger from litellm.router_api import RoutingContext + from litellm.router_python_host import PythonHost else: PrometheusLogger = Any @@ -54,14 +54,17 @@ def emit_router_cooldown_event( temp_litellm_params = copy.deepcopy(_litellm_params) temp_litellm_params = dict(temp_litellm_params) _model_name: Final = _deployment.get("model_name", None) or "" - _api_base: Final = litellm.get_api_base(model=_model_name, optional_params=temp_litellm_params) or "" + host: Final = ctx.host() + _api_base: Final = str( + host.invoke("get_api_base")(model=_model_name, optional_params=temp_litellm_params) or "" # pyright: ignore[reportUnknownMemberType] # the host op table + ) model_info: Final = _deployment["model_info"] model_id: Final = model_info.id litellm_model_name: Final = temp_litellm_params.get("model") or "" llm_provider = "" try: - _, llm_provider, _, _ = litellm.get_llm_provider( + _, llm_provider, _, _ = host.invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # the host op table model=litellm_model_name, custom_llm_provider=temp_litellm_params.get("custom_llm_provider"), ) @@ -69,7 +72,7 @@ def emit_router_cooldown_event( pass # get the prometheus logger from in memory loggers - prometheusLogger: Final[PrometheusLogger | None] = get_prometheus_logger_from_callbacks() + prometheusLogger: Final[PrometheusLogger | None] = get_prometheus_logger_from_callbacks(host) if prometheusLogger is not None: prometheusLogger.set_deployment_complete_outage( @@ -88,19 +91,22 @@ def emit_router_cooldown_event( ) -def get_prometheus_logger_from_callbacks() -> PrometheusLogger | None: +def get_prometheus_logger_from_callbacks(host: "PythonHost | None" = None) -> PrometheusLogger | None: """ - Checks if prometheus is a initalized callback, if yes returns it + Checks if prometheus is a initalized callback, if yes returns it. The callback lists are read + through ``host`` (the router core's host; the process host when none is given). """ from litellm.integrations.prometheus import PrometheusLogger + from litellm.router_python_host import python_host if PrometheusLogger is None: return None - for _callback in litellm._async_success_callback: + source: Final = python_host if host is None else host + for _callback in source.callback_list("_async_success_callback"): if isinstance(_callback, PrometheusLogger): return _callback - for global_callback in litellm.callbacks: + for global_callback in source.dynamic().callbacks: if isinstance(global_callback, PrometheusLogger): return global_callback diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index c40c04f6405..f9bedbb3476 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -453,7 +453,9 @@ def _has_fallback_group(ctx: "RoutingContext", requested_model_group: str | None chains: Final = tuple( chain for chain in map(_fallback_chain_names, _fallback_entries(configured)) if chain is not None ) - fallback_group, _ = get_fallback_model_group(fallbacks=chains, model_group=requested_model_group) + fallback_group, _ = get_fallback_model_group( + fallbacks=chains, model_group=requested_model_group, provider_list=ctx.host().dynamic().provider_list + ) return bool(fallback_group) @@ -544,7 +546,7 @@ def _should_cooldown_deployment( # Only apply error rate cooldown when we have enough requests to make the percentage meaningful return True - elif litellm.should_retry(status_code=exception_status_int) is False: + elif ctx.host().invoke("should_retry")(status_code=exception_status_int) is False: # pyright: ignore[reportUnknownMemberType] # the host op table return True return False @@ -819,7 +821,7 @@ def _is_allowed_fails_set_on_router( return False if ctx.explicit_allowed_fails is not None: return True - if ctx.settings().allowed_fails != litellm.allowed_fails: + if ctx.settings().allowed_fails != ctx.host().dynamic().allowed_fails: return True return False diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 86d824a3f78..57677fa37f2 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -45,6 +45,7 @@ from litellm.types.router import ( if TYPE_CHECKING: from litellm.router_api import RoutingContext + from litellm.router_python_host import PythonHost # Status codes a generic API call's caller-supplied resource id can trigger on its own # (e.g. a nonexistent file/batch/thread id), independent of the selected deployment's health. @@ -266,7 +267,7 @@ def is_bad_request_without_fallback(exception: Exception, fallback_on_unsupporte return exception.status_code == 400 and bool(getattr(exception, "llm_provider", "")) -def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool: +def _check_stripped_model_group(model_group: str, fallback_key: str, provider_list: Sequence[object]) -> bool: """ Handles wildcard routing scenario @@ -279,11 +280,8 @@ def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool: Returns: - True if the stripped model group == fallback_key """ - for provider in litellm.provider_list: - if isinstance(provider, Enum): - _provider = provider.value - else: - _provider = provider + for provider in provider_list: + _provider: str = str(provider.value) if isinstance(provider, Enum) else str(provider) # pyright: ignore[reportAny] # an Enum member's value if model_group.startswith(f"{_provider}/"): stripped_model_group = model_group.replace(f"{_provider}/", "") if stripped_model_group == fallback_key: @@ -588,12 +586,16 @@ def _resolved_a_specific_chain( def get_fallback_model_group_for_lookup_groups( fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: mirrors the fallbacks config shape lookup_groups: tuple[str, ...], + provider_list: Sequence[object] | None = None, ) -> tuple[list[str] | None, int | None]: # mutable-ok: mirrors get_fallback_model_group's contract """ First lookup group with a specifically-keyed chain wins; the generic "*" chain applies only after every group missed, so a catch-all cannot shadow a later group's own chain. """ - results: Final = tuple(get_fallback_model_group(fallbacks=fallbacks, model_group=group) for group in lookup_groups) + results: Final = tuple( + get_fallback_model_group(fallbacks=fallbacks, model_group=group, provider_list=provider_list) + for group in lookup_groups + ) specific: Final = next((result for result in results if _resolved_a_specific_chain(fallbacks, result)), None) if specific is not None: return specific @@ -603,8 +605,12 @@ def get_fallback_model_group_for_lookup_groups( def get_fallback_model_group( fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: mirrors the fallbacks config shape model_group: str, + provider_list: Sequence[object] | None = None, ) -> tuple[list[str] | None, int | None]: # mutable-ok: mirrors get_fallback_model_group's contract """ + ``provider_list`` is the process provider list, read through the router core's host (the + process host's when none is given). + Returns: - fallback_model_group: List[str] of fallback model groups. example: ["gpt-4", "gpt-3.5-turbo"] - generic_fallback_idx: int of the index of the generic fallback in the fallbacks list. @@ -615,6 +621,9 @@ def get_fallback_model_group( - provider-prefixed model group match - generic fallback """ + from litellm.router_python_host import python_host + + providers: Final = python_host.dynamic().provider_list if provider_list is None else provider_list generic_fallback_idx: int | None = None stripped_model_fallback: list[str] | None = None fallback_model_group: list[str] | None = None @@ -628,7 +637,7 @@ def get_fallback_model_group( fallback_model_group = item[model_group] break elif fallback_key == prefixed_model_group or _check_stripped_model_group( - model_group=model_group, fallback_key=fallback_key + model_group=model_group, fallback_key=fallback_key, provider_list=providers ): # check generic fallback stripped_model_fallback = item[fallback_key] elif fallback_key == "*": # check generic fallback @@ -645,20 +654,26 @@ def get_fallback_model_group( return fallback_model_group, generic_fallback_idx -def _resolved_fallback_chain(entries: Sequence[object], lookup_groups: tuple[str, ...]) -> Sequence[object]: +def _resolved_fallback_chain( + entries: Sequence[object], lookup_groups: tuple[str, ...], provider_list: Sequence[object] | None +) -> Sequence[object]: group_entries: Final[Sequence[Mapping[str, list[str]]]] = tuple( # mutable-ok: mirrors the fallbacks config shape entry for entry in entries if isinstance(entry, Mapping) and entry ) - chain, _ = get_fallback_model_group_for_lookup_groups(fallbacks=group_entries, lookup_groups=lookup_groups) + chain, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=group_entries, lookup_groups=lookup_groups, provider_list=provider_list + ) return chain if isinstance(chain, list) else () -def continuation_fallback_targets(fallbacks: object, lookup_groups: tuple[str, ...]) -> Sequence[object]: +def continuation_fallback_targets( + fallbacks: object, lookup_groups: tuple[str, ...], provider_list: Sequence[object] | None = None +) -> Sequence[object]: entries: Final[Sequence[object]] = fallbacks if isinstance(fallbacks, list) else () return ( entries if check_non_standard_fallback_format(fallbacks=entries) - else _resolved_fallback_chain(entries, lookup_groups) + else _resolved_fallback_chain(entries, lookup_groups, provider_list) ) @@ -901,6 +916,7 @@ async def run_async_fallback( original_model_group=original_model_group, kwargs=kwargs, original_exception=original_exception, + host=litellm_routing_ctx.host(), ) return response except Exception as e: @@ -910,6 +926,7 @@ async def run_async_fallback( original_model_group=original_model_group, kwargs=kwargs, original_exception=original_exception, + host=litellm_routing_ctx.host(), ) if ( failure_callbacks_blocked_logging_obj := _failure_callbacks_blocked_logging_object( @@ -925,7 +942,9 @@ async def run_async_fallback( raise error_from_fallbacks -async def log_success_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception): +async def log_success_fallback_event( + original_model_group: str, kwargs: dict, original_exception: Exception, host: "PythonHost | None" = None +): """ Log a successful fallback event to all registered callbacks. @@ -940,7 +959,9 @@ async def log_success_fallback_event(original_model_group: str, kwargs: dict, or Errors during logging are caught and reported but do not interrupt the process. """ # Get deduplicated CustomLogger instances from all callback lists - custom_loggers: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger) + from litellm.router_python_host import python_host + + custom_loggers: Final = (python_host if host is None else host).callbacks.get_custom_loggers_for_type(CustomLogger) for _callback_custom_logger in custom_loggers: try: @@ -953,7 +974,9 @@ async def log_success_fallback_event(original_model_group: str, kwargs: dict, or verbose_router_logger.error("Error in log_success_fallback_event: %s", e) -async def log_failure_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception): +async def log_failure_fallback_event( + original_model_group: str, kwargs: dict, original_exception: Exception, host: "PythonHost | None" = None +): """ Log a failed fallback event to all registered callbacks. @@ -968,7 +991,9 @@ async def log_failure_fallback_event(original_model_group: str, kwargs: dict, or Errors during logging are caught and reported but do not interrupt the process. """ # Get deduplicated CustomLogger instances from all callback lists - custom_loggers: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger) + from litellm.router_python_host import python_host + + custom_loggers: Final = (python_host if host is None else host).callbacks.get_custom_loggers_for_type(CustomLogger) for _callback_custom_logger in custom_loggers: try: diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py index 5c97e26866f..8a5db61cc73 100644 --- a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -7,11 +7,10 @@ from dataclasses import dataclass from itertools import accumulate from os.path import commonprefix from types import MappingProxyType -from typing import Final, TypeVar +from typing import TYPE_CHECKING, Final, TypeVar from pydantic import TypeAdapter, ValidationError -import litellm from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.routing_common_utils import MID_STREAM_CONTINUATION_KWARG from litellm.litellm_core_utils.rules import Rules @@ -19,6 +18,9 @@ from litellm.types.llms.openai import AllMessageValues, OutputTextDeltaEvent from litellm.types.utils import ModelResponse, ModelResponseStream, Usage from litellm.utils import supports_assistant_prefill +if TYPE_CHECKING: + from litellm.router_python_host import PythonHost + class _ContinuationMarker: """Only the router constructs one, so a JSON request body cannot forge the continuation flag.""" @@ -219,7 +221,7 @@ class ContinuationOutput: """Splices a fallback's continuation onto the stream the client already received. Drops a leading echo of the prefill, since some providers (Mistral) return the prefix inside the completion, and the prefill's trailing whitespace sent again by a provider that strips it (Anthropic), keeps the primary's chunk id, created time and single role delta, and - runs litellm.post_call_rules over the joined text, since each stream wrapper only sees its own fragment. A + runs the process post_call_rules (read through the router core's host) over the joined text, since each stream wrapper only sees its own fragment. A continuation nested in another leaves the rules to the outermost one, the only one holding the whole text.""" def __init__( @@ -229,7 +231,11 @@ class ContinuationOutput: model: str, primary_response: object, checks_post_call_rules: bool = True, + host: "PythonHost | None" = None, ) -> None: + from litellm.router_python_host import python_host + + self._host: Final = python_host if host is None else host self._text = emitted_text self._checks_post_call_rules: Final = checks_post_call_rules self._prefill_text: Final = _prefill_text(prefill_message) @@ -283,7 +289,7 @@ class ContinuationOutput: return _without_leading_text(chunk, matched) def _observed(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]: - if not self._checks_post_call_rules or not litellm.post_call_rules: + if not self._checks_post_call_rules or not self._host.dynamic().post_call_rules: return chunks for chunk in chunks: self._text += _delta_text(chunk) diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 80e1f3fe29c..b37f59bb063 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -44,6 +44,8 @@ from litellm.utils import get_utc_datetime if TYPE_CHECKING: from opentelemetry.trace import Span as _Span + from litellm.router_python_host import PythonHost + Span = _Span else: Span = Any @@ -179,11 +181,11 @@ def _estimate_input_tokens(request_kwargs: dict[str, Any] | None, model: str = " return 0 -def _model_max_output_tokens(model_name: str) -> int | None: - # litellm.get_model_info raises a bare Exception for an unrecognized model; - # this lookup is a fallback default and must never fail the request. +def _model_max_output_tokens(model_name: str, host: PythonHost) -> int | None: + # get_model_info (through the router core's host) raises a bare Exception for an + # unrecognized model; this lookup is a fallback default and must never fail the request. with contextlib.suppress(Exception): - info: Final = litellm.get_model_info(model=model_name) + info: Final = host.invoke("get_model_info")(model=model_name) # pyright: ignore[reportUnknownMemberType] # the host op table model_max: Final = info.get("max_output_tokens") if model_max is not None: return max(0, int(model_max)) @@ -227,8 +229,8 @@ def _deployment_output_cap(request_kwargs: Mapping[str, object] | None, deployme ) -def _default_otpm_reservation(model_name: str, otpm_limit: int) -> int: - model_max: Final = _model_max_output_tokens(model_name) if model_name else None +def _default_otpm_reservation(model_name: str, otpm_limit: int, host: PythonHost) -> int: + model_max: Final = _model_max_output_tokens(model_name, host) if model_name else None share_of_limit: Final = max(1, int(otpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION)) return min(4096 if model_max is None else model_max, share_of_limit) @@ -238,11 +240,12 @@ def _otpm_reservation( deployment: Mapping[str, object], model_name: str, otpm_limit: int, + host: PythonHost, ) -> int: from_request: Final = _explicit_max_tokens(request_kwargs) if from_request is not None: return from_request - default: Final = _default_otpm_reservation(model_name, otpm_limit) + default: Final = _default_otpm_reservation(model_name, otpm_limit, host) from_deployment: Final = _deployment_output_cap(request_kwargs, deployment) return default if from_deployment is None else min(from_deployment, default) @@ -515,7 +518,11 @@ async def _increment_with_rollback( def io_token_pre_call_check( dual_cache: DualCache, deployment: dict, + host: PythonHost | None = None, ) -> dict | None: + """``host`` is the router core's host (the process host when none is given).""" + from litellm.router_python_host import python_host + itpm_limit, otpm_limit = get_deployment_io_token_limits(deployment) if itpm_limit is None and otpm_limit is None: return deployment @@ -525,7 +532,9 @@ def io_token_pre_call_check( _model, served_model = _tokenizer_and_served_models(deployment) estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model) otpm_to_reserve: Final = ( - _otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0 + _otpm_reservation(request_kwargs, deployment, served_model, otpm_limit, python_host if host is None else host) + if otpm_limit is not None + else 0 ) dt: Final = get_utc_datetime() @@ -584,7 +593,11 @@ async def async_io_token_pre_call_check( dual_cache: DualCache, deployment: dict, parent_otel_span: Span | None = None, + host: PythonHost | None = None, ) -> dict | None: + """``host`` is the router core's host (the process host when none is given).""" + from litellm.router_python_host import python_host + itpm_limit, otpm_limit = get_deployment_io_token_limits(deployment) if itpm_limit is None and otpm_limit is None: return deployment @@ -600,7 +613,9 @@ async def async_io_token_pre_call_check( _model, served_model = _tokenizer_and_served_models(deployment) estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model) otpm_to_reserve: Final = ( - _otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0 + _otpm_reservation(request_kwargs, deployment, served_model, otpm_limit, python_host if host is None else host) + if otpm_limit is not None + else 0 ) dt: Final = get_utc_datetime() diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index f41d19cd62c..07551884057 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -44,6 +44,8 @@ from litellm.utils import get_utc_datetime if TYPE_CHECKING: from opentelemetry.trace import Span as _Span + from litellm.router_python_host import PythonHost + Span = _Span else: Span = Any @@ -134,7 +136,14 @@ class ModelRateLimitingCheck(CustomLogger): this check actively enforces those limits across ALL routing strategies. """ - def __init__(self, dual_cache: DualCache, clock: Callable[[], datetime] = get_utc_datetime): + def __init__( + self, + dual_cache: DualCache, + clock: Callable[[], datetime] = get_utc_datetime, + host: "PythonHost | None" = None, + ): + # The router core's host: the io-token checks read model info through it. + self._host: PythonHost | None = host self.dual_cache = dual_cache self.clock = clock # model_ids already warned about itpm/otpm + tpm/rpm on the same deployment, @@ -312,6 +321,7 @@ class ModelRateLimitingCheck(CustomLogger): io_token_pre_call_check( self.dual_cache, deployment, + host=self._host, ) io_reservation_made = True @@ -397,6 +407,7 @@ class ModelRateLimitingCheck(CustomLogger): self.dual_cache, deployment, parent_otel_span=parent_otel_span, + host=self._host, ) io_reservation_made = True diff --git a/litellm/router_utils/request_priority.py b/litellm/router_utils/request_priority.py index 61627d4b0cf..6a004114b21 100644 --- a/litellm/router_utils/request_priority.py +++ b/litellm/router_utils/request_priority.py @@ -1,12 +1,15 @@ from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass -from typing import Final +from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.core_helpers import normalize_drop_params +if TYPE_CHECKING: + from litellm.router_python_host import PythonHost + @dataclass(frozen=True, slots=True) class InvalidPriority: @@ -84,9 +87,15 @@ def request_drops_params( kwargs: Mapping[str, object], router_defaults: Mapping[str, object], deployment_params: Iterable[Mapping[str, object]], + host: "PythonHost | None" = None, ) -> bool: + """``host`` is the router core's host, whose process ``drop_params`` counts (the process host when none is given).""" + from litellm.router_python_host import python_host + deployments: Final = tuple(deployment_params) - if litellm.drop_params is True or any(_drops_priority_by_name(s) for s in (kwargs, router_defaults, *deployments)): + if (python_host if host is None else host).dynamic().drop_params is True or any( + _drops_priority_by_name(s) for s in (kwargs, router_defaults, *deployments) + ): return True request_flag: Final = _explicit_drop_params(kwargs) if request_flag is not None: diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index bcbf69e14a6..bc041842db0 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -874,7 +874,7 @@ def test_jev_classifier_cost_uses_registry_pricing(monkeypatch: pytest.MonkeyPat answers={"tier": _answer()}, usage=JevUsage(input_tokens=3, output_tokens=4), ) - assert jev_classifier_cost(response, "jev-latest") == pytest.approx(0.0011) + assert jev_classifier_cost(response, "jev-latest", cost_map=litellm.model_cost) == pytest.approx(0.0011) def test_jev_classifier_cost_is_none_without_registry_pricing() -> None: @@ -883,7 +883,7 @@ def test_jev_classifier_cost_is_none_without_registry_pricing() -> None: answers={"tier": _answer()}, usage=JevUsage(input_tokens=3, output_tokens=4), ) - assert jev_classifier_cost(response, "jev-unpriced") is None + assert jev_classifier_cost(response, "jev-unpriced", cost_map=litellm.model_cost) is None @pytest.mark.asyncio diff --git a/tests/unit/router_strategy/test_budget_limiter_hotpath.py b/tests/unit/router_strategy/test_budget_limiter_hotpath.py index c79ff6325d2..7cee57cb193 100644 --- a/tests/unit/router_strategy/test_budget_limiter_hotpath.py +++ b/tests/unit/router_strategy/test_budget_limiter_hotpath.py @@ -75,7 +75,7 @@ async def test_get_llm_provider_for_deployment_dict_view_supports_mapping_and_at return model, "openai", None, None monkeypatch.setattr( - "litellm.router_strategy.budget_limiter.litellm.get_llm_provider", + "litellm.get_llm_provider", _future_style_get_llm_provider, ) diff --git a/tests/unit/test_router/core/test_fallbacks_characterization_outcomes.py b/tests/unit/test_router/core/test_fallbacks_characterization_outcomes.py index 631b9dcf3c2..3328d3dd3f4 100644 --- a/tests/unit/test_router/core/test_fallbacks_characterization_outcomes.py +++ b/tests/unit/test_router/core/test_fallbacks_characterization_outcomes.py @@ -425,10 +425,10 @@ class FallbackEvents(CustomLogger): @pytest.mark.asyncio async def test_fallback_events_reach_the_process_custom_loggers_innermost_hop_first( - make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch + make_core: MakeCore, host: ScriptedHost ) -> None: events = FallbackEvents() - monkeypatch.setattr(litellm, "callbacks", [events]) + host.callbacks.add_litellm_callback(events) core = build(make_core, fallbacks=[{"a": ["b", "c"]}]) host.answer("openai/a", rate_limit()) host.answer("openai/b", server_error()) @@ -443,10 +443,10 @@ async def test_fallback_events_reach_the_process_custom_loggers_innermost_hop_fi @pytest.mark.asyncio async def test_a_fallback_that_fails_everywhere_logs_failure_events( - make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch + make_core: MakeCore, host: ScriptedHost ) -> None: events = FallbackEvents() - monkeypatch.setattr(litellm, "callbacks", [events]) + host.callbacks.add_litellm_callback(events) core = build(make_core, fallbacks=[{"a": ["b"]}]) host.answer("*", rate_limit()) diff --git a/tests/unit/test_router/core/test_gaps_router_core_crate_picks.py b/tests/unit/test_router/core/test_gaps_router_core_crate_picks.py index 255d731142d..99a5a88d42e 100644 --- a/tests/unit/test_router/core/test_gaps_router_core_crate_picks.py +++ b/tests/unit/test_router/core/test_gaps_router_core_crate_picks.py @@ -267,10 +267,10 @@ def _stale(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok @pytest.mark.asyncio async def test_a_stale_routing_group_at_a_fallback_level_is_that_levels_error( - make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch + make_core: MakeCore, host: ScriptedHost ) -> None: events: Final = FallbackEvents() - monkeypatch.setattr(litellm, "callbacks", [events]) + host.callbacks.add_litellm_callback(events) core: Final = _stale(make_core) host.answer("*", rate_limit()) diff --git a/tests/unit/test_router/core/test_retries_cooldowns.py b/tests/unit/test_router/core/test_retries_cooldowns.py index 004b1762240..8eef3033cea 100644 --- a/tests/unit/test_router/core/test_retries_cooldowns.py +++ b/tests/unit/test_router/core/test_retries_cooldowns.py @@ -531,6 +531,9 @@ async def test_a_deployment_cools_down_once_its_failures_pass_allowed_fails( @pytest.mark.asyncio async def test_process_allowed_fails_is_the_default(make_core: MakeCore, host: ScriptedHost) -> None: + """The host's process ``allowed_fails`` is the router's default, and a router left on the default + does not count it as set on the router: one failure cools nothing down, as in a process whose + ``litellm.allowed_fails`` is 0.""" host.set_dynamic(allowed_fails=0) core: Final = make_core(model_list=two_deployments(), num_retries=0) host.answer("*", fresh(server_error)) @@ -538,6 +541,20 @@ async def test_process_allowed_fails_is_the_default(make_core: MakeCore, host: S await attempt(core) assert core.settings().allowed_fails == 0 + assert list(cooled_down(core, ["a", "b"])) == [] + + +@pytest.mark.asyncio +async def test_an_allowed_fails_set_on_the_router_cools_down_after_that_many_failures( + make_core: MakeCore, host: ScriptedHost +) -> None: + """An ``allowed_fails`` passed to the router counts as set on it, even when it equals the host's default.""" + host.set_dynamic(allowed_fails=0) + core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0) + host.answer("*", fresh(server_error)) + + await attempt(core) + assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion") diff --git a/tests/unit/test_router/core/test_router_host_boundary.py b/tests/unit/test_router/core/test_router_host_boundary.py new file mode 100644 index 00000000000..d4d0ae0dd4c --- /dev/null +++ b/tests/unit/test_router/core/test_router_host_boundary.py @@ -0,0 +1,54 @@ +""" +Both router cores and the helpers they share reach the process only through their host: no router +module reads a ``litellm.`` or calls a ``litellm.`` directly. Exception classes, +types and modules (``litellm.RateLimitError``, ``litellm.Usage``, ``litellm.utils``) are code, not +process state, and stay importable. +""" + +import ast +import inspect +from pathlib import Path +from typing import Final + +import litellm + +_PACKAGE: Final = Path(litellm.__file__).parent +# router_config.py is the host's own view of the process settings; router_python_host.py is the host. +_HOST_MODULES: Final = frozenset({"router_config.py", "router_python_host.py"}) + + +def _router_modules() -> tuple[Path, ...]: + return tuple( + sorted( + path + for path in ( + *_PACKAGE.glob("router_*.py"), + *(_PACKAGE / "router_utils").rglob("*.py"), + *(_PACKAGE / "router_strategy").rglob("*.py"), + ) + if path.name not in _HOST_MODULES + ) + ) + + +def _is_code(name: str) -> bool: + value: Final[object] = getattr(litellm, name, None) + return inspect.isclass(value) or inspect.ismodule(value) + + +def _process_reads(path: Path) -> tuple[str, ...]: + tree: Final = ast.parse(path.read_text(encoding="utf-8")) + return tuple( + f"{path.relative_to(_PACKAGE.parent)}:{node.lineno} litellm.{node.attr}" + for node in ast.walk(tree) + if isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "litellm" + and not _is_code(node.attr) + ) + + +def test_router_code_reaches_litellm_settings_and_functions_only_through_the_host() -> None: + modules: Final = _router_modules() + assert len(modules) > 50 + assert [read for path in modules for read in _process_reads(path)] == []