mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
refactor(router): route every router read of litellm settings and functions through the host
Both router cores and the Python helpers they share (cooldown handlers and callbacks, budget limiter, lowest-cost strategy, complexity router, savings baseline, continuation prefill, io-token and model rate-limit checks, request priority, fallback event handlers) now read process settings through host.dynamic() and call litellm functions through host.invoke(). RustCore's allowed_fails default comes from the host too. A guard test fails if any router module reads litellm.<setting> again.
This commit is contained in:
parent
1a7deafe09
commit
1e08643bd3
27 changed files with 349 additions and 122 deletions
|
|
@ -59,6 +59,7 @@ if TYPE_CHECKING:
|
|||
from litellm.messages.dispatch import MessagesResult
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
from litellm.router import Router
|
||||
from litellm.router_python_host import PythonHost
|
||||
from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter
|
||||
from litellm.router_strategy.auto_router.auto_router import AutoRouter
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
|
|
@ -1474,6 +1475,16 @@ class RoutingContext:
|
|||
def belongs_to(self, router: "Router") -> bool:
|
||||
return self._router is router or router.__dict__.get("_core") is self._router
|
||||
|
||||
def host(self) -> "PythonHost":
|
||||
"""The router core's host: the process settings (``dynamic()``) and litellm functions
|
||||
(``invoke``) a plugin reads go through it, never ``litellm.*`` directly."""
|
||||
from litellm.router_python_host import PythonHost, python_host
|
||||
|
||||
core: Final[object] = self._router.__dict__.get("_core", self._router) # pyright: ignore[reportAny] # __dict__ is untyped
|
||||
host: Final[object] = getattr(core, "_host", None)
|
||||
# Every core keeps its host as _host; a stand-in router without one reads the process host.
|
||||
return host if isinstance(host, PythonHost) else python_host
|
||||
|
||||
@property
|
||||
def explicit_allowed_fails(self) -> int | None:
|
||||
"""``allowed_fails`` as the caller passed it, ``None`` when it came from the global default."""
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import litellm
|
|||
if TYPE_CHECKING:
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
|
||||
from litellm.types.utils import BudgetConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -59,6 +60,9 @@ class RouterDynamicConfig:
|
|||
callback_manager: "LoggingCallbackManager"
|
||||
cache: "Cache | None"
|
||||
request_timeout_explicitly_set: bool = False
|
||||
tag_budget_config: "Mapping[str, BudgetConfig] | None" = None
|
||||
disable_token_counter: bool = False
|
||||
post_call_rules: Sequence[object] = ()
|
||||
|
||||
|
||||
_versions: Final = itertools.count(1)
|
||||
|
|
@ -112,6 +116,9 @@ def current_dynamic_config() -> RouterDynamicConfig:
|
|||
callback_manager=litellm.logging_callback_manager,
|
||||
cache=litellm.cache,
|
||||
request_timeout_explicitly_set=litellm.request_timeout_explicitly_set,
|
||||
tag_budget_config=litellm.tag_budget_config,
|
||||
disable_token_counter=litellm.disable_token_counter,
|
||||
post_call_rules=litellm.post_call_rules,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -141,6 +148,9 @@ _LIVE_SOURCES: Final = MappingProxyType(
|
|||
"input_callback": "input_callback",
|
||||
"callback_manager": "logging_callback_manager",
|
||||
"cache": "cache",
|
||||
"tag_budget_config": "tag_budget_config",
|
||||
"disable_token_counter": "disable_token_counter",
|
||||
"post_call_rules": "post_call_rules",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2185,7 +2185,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self._core_provider_budget_config = provider_budget_config
|
||||
self.router_budget_logger: RouterBudgetLimiting | None = None
|
||||
if RouterBudgetLimiting.should_init_router_budget_limiter(
|
||||
model_list=model_list, provider_budget_config=self._core_provider_budget_config
|
||||
model_list=model_list, provider_budget_config=self._core_provider_budget_config, host=self._host
|
||||
):
|
||||
if optional_pre_call_checks is not None:
|
||||
optional_pre_call_checks.append("router_budget_limiting")
|
||||
|
|
@ -2473,6 +2473,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
selector = LowestCostLoggingHandler(
|
||||
router_cache=self._core_cache,
|
||||
routing_args={},
|
||||
host=self._host,
|
||||
)
|
||||
case _:
|
||||
pass
|
||||
|
|
@ -3204,10 +3205,11 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
dual_cache=self._core_cache,
|
||||
provider_budget_config=self._core_provider_budget_config,
|
||||
model_list=self._core_model_list,
|
||||
host=self._host,
|
||||
)
|
||||
self.router_budget_logger = _callback
|
||||
elif pre_call_check == "enforce_model_rate_limits":
|
||||
_callback = ModelRateLimitingCheck(dual_cache=self._core_cache)
|
||||
_callback = ModelRateLimitingCheck(dual_cache=self._core_cache, host=self._host)
|
||||
|
||||
if _callback is None:
|
||||
continue
|
||||
|
|
@ -3898,6 +3900,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
complete_response_object,
|
||||
checks_post_call_rules=initial_kwargs.get(MID_STREAM_CONTINUATION_KWARG)
|
||||
is not MID_STREAM_CONTINUATION_MARKER,
|
||||
host=self._host,
|
||||
)
|
||||
if continuation_messages is not None
|
||||
else None
|
||||
|
|
@ -4177,7 +4180,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
request_kwargs, requested_group if isinstance(requested_group, str) else None
|
||||
)
|
||||
targets: Final = continuation_fallback_targets(
|
||||
request_kwargs.get("fallbacks", self._core_fallbacks), lookup_groups
|
||||
request_kwargs.get("fallbacks", self._core_fallbacks), lookup_groups, self._host.dynamic().provider_list
|
||||
)
|
||||
if any(fallback_target_overrides_request_params(target) for target in targets):
|
||||
return False
|
||||
|
|
@ -5740,7 +5743,10 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
|
||||
def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool:
|
||||
return request_drops_params(
|
||||
kwargs, self._core_default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs)
|
||||
kwargs,
|
||||
self._core_default_litellm_params,
|
||||
self._request_deployment_params(model=model, kwargs=kwargs),
|
||||
self._host,
|
||||
)
|
||||
|
||||
async def _schedule_factory(
|
||||
|
|
@ -8852,6 +8858,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
external_fallback_group, generic_idx = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=fallbacks,
|
||||
lookup_groups=lookup_groups,
|
||||
provider_list=self._host.dynamic().provider_list,
|
||||
)
|
||||
if external_fallback_group is None and generic_idx is not None:
|
||||
external_fallback_group = fallbacks[generic_idx]["*"]
|
||||
|
|
@ -8981,6 +8988,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
) = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=fallbacks, # if fallbacks = [{"gpt-3.5-turbo": ["claude-3-haiku"]}]
|
||||
lookup_groups=lookup_groups,
|
||||
provider_list=self._host.dynamic().provider_list,
|
||||
)
|
||||
## if none, check for generic fallback
|
||||
if fallback_model_group is None and generic_fallback_idx is not None:
|
||||
|
|
@ -9537,7 +9545,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
lookup_groups: tuple[str, ...],
|
||||
) -> list[str] | None: # mutable-ok: mirrors the shared resolver's contract
|
||||
fallback_model_group, _ = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=fallbacks, lookup_groups=lookup_groups
|
||||
fallbacks=fallbacks, lookup_groups=lookup_groups, provider_list=self._host.dynamic().provider_list
|
||||
)
|
||||
return fallback_model_group
|
||||
|
||||
|
|
@ -10143,6 +10151,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
resolved, _ = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=fallbacks,
|
||||
lookup_groups=fallback_lookup_groups(kwargs, model_group),
|
||||
provider_list=self._host.dynamic().provider_list,
|
||||
)
|
||||
return has_unattempted_fallback_target(resolved, kwargs)
|
||||
|
||||
|
|
@ -10193,6 +10202,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
resolved, _ = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=fallbacks,
|
||||
lookup_groups=fallback_lookup_groups(kwargs, model_group),
|
||||
provider_list=self._host.dynamic().provider_list,
|
||||
)
|
||||
return has_unattempted_fallback_target(resolved, kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -143,6 +143,7 @@ class OptionalChecks:
|
|||
if RouterBudgetLimiting.should_init_router_budget_limiter( # pyright: ignore[reportUnknownMemberType] # untyped model_list
|
||||
model_list=model_list if isinstance(model_list, list) else None, # pyright: ignore[reportUnknownArgumentType] # the argument as passed
|
||||
provider_budget_config=self.provider_budget_config, # pyright: ignore[reportArgumentType] # the argument as passed
|
||||
host=self._host,
|
||||
):
|
||||
if listed is not None:
|
||||
listed.append("router_budget_limiting")
|
||||
|
|
@ -255,10 +256,11 @@ class OptionalChecks:
|
|||
dual_cache=self._cache,
|
||||
provider_budget_config=self.provider_budget_config, # pyright: ignore[reportArgumentType] # the argument as passed
|
||||
model_list=self._hosted._core_model_list, # pyright: ignore[reportPrivateUsage, reportArgumentType] # the live model_list
|
||||
host=self._host,
|
||||
)
|
||||
self.router_budget_logger = callback
|
||||
elif name == "enforce_model_rate_limits":
|
||||
callback = ModelRateLimitingCheck(dual_cache=self._cache)
|
||||
callback = ModelRateLimitingCheck(dual_cache=self._cache, host=self._host)
|
||||
if callback is None:
|
||||
continue
|
||||
if self.optional_callbacks is not None and any(
|
||||
|
|
|
|||
|
|
@ -721,7 +721,10 @@ class Resolved:
|
|||
# An explicit `routing_strategy=None` reaches the Rust core as "None": no strategy picks.
|
||||
return kept if self.routing_strategy is not None else {**kept, "routing_strategy": "None"}
|
||||
|
||||
def dynamic_document(self, version: int, model_names: Sequence[str]) -> Mapping[str, object]:
|
||||
def dynamic_document(
|
||||
self, version: int, model_names: Sequence[str], process_allowed_fails: object
|
||||
) -> Mapping[str, object]:
|
||||
"""``process_allowed_fails`` is the process default, read through the host."""
|
||||
values: Final = {
|
||||
"version": version,
|
||||
"model_group_alias": _plain(dict(self.model_group_alias)),
|
||||
|
|
@ -730,21 +733,19 @@ class Resolved:
|
|||
"content_policy_fallbacks": _plain(self.content_policy_fallbacks),
|
||||
"num_retries": _crate_int(self.num_retries),
|
||||
"max_fallbacks": _crate_int(self.max_fallbacks),
|
||||
"allowed_fails": _crate_int(self._router_allowed_fails()),
|
||||
"allowed_fails": _crate_int(self._router_allowed_fails(process_allowed_fails)),
|
||||
"fallback_keys": self._fallback_keys(model_names),
|
||||
"default_deployment": _plain(served_default_deployment(self.default_deployment)[0]),
|
||||
"default_deployment_raises": True if served_default_deployment(self.default_deployment)[1] else None,
|
||||
}
|
||||
return {name: value for name, value in values.items() if value is not None}
|
||||
|
||||
def _router_allowed_fails(self) -> object:
|
||||
def _router_allowed_fails(self, process_allowed_fails: object) -> object:
|
||||
"""``allowed_fails`` when it counts as set on the router (``_is_allowed_fails_set_on_router``):
|
||||
passed explicitly, or different from the process default."""
|
||||
import litellm
|
||||
|
||||
passed explicitly, or different from the process default ``process_allowed_fails``."""
|
||||
if self.explicit_allowed_fails is not None:
|
||||
return self.explicit_allowed_fails
|
||||
if self.allowed_fails != litellm.allowed_fails:
|
||||
if self.allowed_fails != process_allowed_fails:
|
||||
return self.allowed_fails
|
||||
return None
|
||||
|
||||
|
|
@ -1218,7 +1219,9 @@ class RustCore(
|
|||
**{name: value for name, value in failover_document(self._plain).items() if value is not None},
|
||||
**({"provider_default_deployment_ids": list(default_ids)} if default_ids else {}),
|
||||
}
|
||||
return RouterCoreHandle(static, self._resolved.dynamic_document(self._version, names))
|
||||
return RouterCoreHandle(
|
||||
static, self._resolved.dynamic_document(self._version, names, self._host.dynamic().allowed_fails)
|
||||
)
|
||||
|
||||
def _provider_default_ids(self) -> tuple[str, ...]:
|
||||
"""``provider_default_deployment_ids`` as the cooldown check reads it (``deployment in
|
||||
|
|
@ -1609,7 +1612,9 @@ class RustCore(
|
|||
self._rebuild()
|
||||
else:
|
||||
names: Final = [str(d.get("model_name")) for d in self._handle.deployments()]
|
||||
self._handle.apply({"settings": resolved.dynamic_document(self._version, names)})
|
||||
self._handle.apply(
|
||||
{"settings": resolved.dynamic_document(self._version, names, self._host.dynamic().allowed_fails)}
|
||||
)
|
||||
|
||||
def _changes_strategy(self, updated: Mapping[str, object]) -> bool:
|
||||
"""``update_settings`` names a ``routing_strategy`` other than the router's."""
|
||||
|
|
@ -3016,7 +3021,7 @@ class _Driver:
|
|||
self._served: tuple[str | None, Mapping[str, object]] = (None, MappingProxyType({}))
|
||||
self._attempts: int = 0
|
||||
self._walk: Final = WalkMetadata(request, plan.metadata_field, self._params_base_model)
|
||||
self.outcomes: Final = FallbackOutcomes(request)
|
||||
self.outcomes: Final = FallbackOutcomes(request, core._host) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
|
||||
# The fallback level the walk is at, from its fallback events.
|
||||
self._level: int = 0
|
||||
|
||||
|
|
@ -3581,7 +3586,7 @@ class _Driver:
|
|||
source: Final = own if own is not None else fallbacks
|
||||
entries: Final = [dict(m) for entry in _as_list(source) if (m := as_str_mapping(entry))]
|
||||
found, generic = (
|
||||
get_fallback_model_group_for_lookup_groups(entries, lookup) # pyright: ignore[reportArgumentType] # the fallbacks config shape
|
||||
get_fallback_model_group_for_lookup_groups(entries, lookup, self._core._host.dynamic().provider_list) # pyright: ignore[reportArgumentType, reportPrivateUsage] # the fallbacks config shape; the driver's own core's host
|
||||
if entries
|
||||
else (None, None)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1306,6 +1306,7 @@ class RustEndpoints:
|
|||
kwargs,
|
||||
self._core._resolved.default_litellm_params, # pyright: ignore[reportPrivateUsage] # see above
|
||||
deployment_params(),
|
||||
self._core._host, # pyright: ignore[reportPrivateUsage] # see above
|
||||
)
|
||||
|
||||
def _split_priority(
|
||||
|
|
|
|||
|
|
@ -58,7 +58,9 @@ def normalize_strategy(strategy: object) -> str | None:
|
|||
return strategy if isinstance(strategy, str) else None
|
||||
|
||||
|
||||
def build_selector(strategy: str | None, args: Mapping[str, object], cache: "DualCache") -> Selector | None:
|
||||
def build_selector(
|
||||
strategy: str | None, args: Mapping[str, object], cache: "DualCache", host: "PythonHost"
|
||||
) -> Selector | None:
|
||||
"""``_build_strategy_selector`` without registering: ``None`` for simple-shuffle and every
|
||||
strategy without a selector."""
|
||||
routing_args: Final = dict(args)
|
||||
|
|
@ -72,7 +74,7 @@ def build_selector(strategy: str | None, args: Mapping[str, object], cache: "Dua
|
|||
case "latency-based-routing":
|
||||
return LowestLatencyLoggingHandler(router_cache=cache, routing_args=routing_args)
|
||||
case "cost-based-routing":
|
||||
return LowestCostLoggingHandler(router_cache=cache, routing_args={})
|
||||
return LowestCostLoggingHandler(router_cache=cache, routing_args={}, host=host)
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
|
@ -122,7 +124,7 @@ class StrategySelectors:
|
|||
self._unregister([self._default, *self._overrides.values()])
|
||||
self._overrides = {}
|
||||
self._strategy = normalize_strategy(strategy)
|
||||
self._default = build_selector(self._strategy, args, self._cache)
|
||||
self._default = build_selector(self._strategy, args, self._cache, self._host)
|
||||
if self._default is not None:
|
||||
self._register(self._default)
|
||||
|
||||
|
|
@ -136,7 +138,7 @@ class StrategySelectors:
|
|||
if self._default is None:
|
||||
return
|
||||
try:
|
||||
rebuilt: Final = build_selector(self._strategy, args, self._cache)
|
||||
rebuilt: Final = build_selector(self._strategy, args, self._cache, self._host)
|
||||
except (TypeError, ValidationError):
|
||||
verbose_router_logger.exception(
|
||||
"Invalid routing_strategy_args %s for '%s'; keeping the previous ones", dict(args), self._strategy
|
||||
|
|
@ -153,7 +155,8 @@ class StrategySelectors:
|
|||
built: Final = {
|
||||
name: selector
|
||||
for name, strategy, args in groups
|
||||
if (selector := build_selector(None if strategy == "priority" else strategy, args, self._cache)) is not None
|
||||
if (selector := build_selector(None if strategy == "priority" else strategy, args, self._cache, self._host))
|
||||
is not None
|
||||
}
|
||||
for selector in built.values():
|
||||
self._register(selector)
|
||||
|
|
@ -211,7 +214,7 @@ class StrategySelectors:
|
|||
return self._default
|
||||
with self._lock:
|
||||
if strategy not in self._overrides:
|
||||
self._overrides[strategy] = build_selector(strategy, {}, self._cache)
|
||||
self._overrides[strategy] = build_selector(strategy, {}, self._cache, self._host)
|
||||
selector: Final = self._overrides[strategy]
|
||||
if selector is not None and request is not None and strategy not in self._registered_strategies():
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
|
|
|||
|
|
@ -423,7 +423,8 @@ def _mid_stream_continuation_eligible(
|
|||
lookup_groups: Final = fallback_lookup_groups(
|
||||
request, requested_group if isinstance(requested_group, str) else None
|
||||
)
|
||||
targets: Final = continuation_fallback_targets(core.settings().fallbacks, lookup_groups) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # fallbacks as Python spells them
|
||||
providers: Final = core._host.dynamic().provider_list # pyright: ignore[reportPrivateUsage] # the core's own host
|
||||
targets: Final = continuation_fallback_targets(core.settings().fallbacks, lookup_groups, providers) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # fallbacks as Python spells them
|
||||
if any(fallback_target_overrides_request_params(target) for target in targets):
|
||||
return False
|
||||
if any(_names_deployment_that_cannot_continue(core, target) for target in targets):
|
||||
|
|
@ -561,6 +562,7 @@ async def _chat_continuation(
|
|||
str(request.get("model") or ""),
|
||||
complete,
|
||||
checks_post_call_rules=request.get(MID_STREAM_CONTINUATION_KWARG) is not MID_STREAM_CONTINUATION_MARKER,
|
||||
host=attempt.core._host, # pyright: ignore[reportPrivateUsage] # the attempt's own core's host
|
||||
)
|
||||
if continuation_messages is not None
|
||||
else None
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Final
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router_python_host import PythonHost
|
||||
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
|
||||
|
||||
# `RETRY_BREADCRUMB_LIMIT`: how many failures `previous_models` keeps.
|
||||
|
|
@ -225,9 +226,10 @@ class FallbackOutcomes:
|
|||
custom loggers and, under ``include_fallback_errors``, lists the level's errors (its own
|
||||
first, then each hop that failed) in the response's ``x-litellm-fallback-errors``."""
|
||||
|
||||
__slots__ = ("_failed", "_header_errors", "_include", "_pending")
|
||||
__slots__ = ("_failed", "_header_errors", "_host", "_include", "_pending")
|
||||
|
||||
def __init__(self, request: Mapping[str, object]) -> None:
|
||||
def __init__(self, request: Mapping[str, object], host: "PythonHost") -> None:
|
||||
self._host: Final = host
|
||||
self._include: Final = request.get("include_fallback_errors", False) is True
|
||||
self._failed: Final[dict[object, list[object]]] = {} # mutable-ok: each level's failed hops, in order
|
||||
self._header_errors: Final[list[list[object]]] = [] # mutable-ok: each successful hop's errors, innermost first
|
||||
|
|
@ -253,7 +255,7 @@ class FallbackOutcomes:
|
|||
self._header_errors.append([get_fallback_error_info(level_error), *failed])
|
||||
self._pending.append(
|
||||
handlers.log_success_fallback_event( # pyright: ignore[reportUnknownMemberType] # takes the kwargs as an untyped dict
|
||||
original_model_group=original, kwargs=dict(kwargs), original_exception=level_error
|
||||
original_model_group=original, kwargs=dict(kwargs), original_exception=level_error, host=self._host
|
||||
)
|
||||
)
|
||||
return
|
||||
|
|
@ -261,7 +263,7 @@ class FallbackOutcomes:
|
|||
failed.append(get_fallback_error_info(hop_error))
|
||||
self._pending.append(
|
||||
handlers.log_failure_fallback_event( # pyright: ignore[reportUnknownMemberType] # see above
|
||||
original_model_group=original, kwargs=dict(kwargs), original_exception=level_error
|
||||
original_model_group=original, kwargs=dict(kwargs), original_exception=level_error, host=self._host
|
||||
)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,11 +26,10 @@ from collections.abc import Callable, Mapping, Sequence
|
|||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import groupby
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
|
|
@ -50,6 +49,9 @@ from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, RouterErro
|
|||
from litellm.types.utils import BudgetConfig, GenericBudgetConfigType, StandardLoggingPayload
|
||||
from litellm.types.utils import BudgetConfig as GenericBudgetInfo
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router_python_host import PythonHost
|
||||
|
||||
DEFAULT_REDIS_SYNC_INTERVAL: Final = 1
|
||||
_CALLBACK_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_BATCH_CHARGE_VALUES: Final = TypeAdapter(tuple[object, float])
|
||||
|
|
@ -121,7 +123,12 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
provider_budget_config: dict | None,
|
||||
model_list: list[DeploymentTypedDict | dict[str, Any]] | None = None,
|
||||
clock: Callable[[], float] = time.time,
|
||||
host: "PythonHost | None" = None,
|
||||
):
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
# The router core's host: the process settings and litellm functions are read through it.
|
||||
self._host: Final = python_host if host is None else host
|
||||
self.dual_cache = dual_cache
|
||||
self._clock = clock
|
||||
self.redis_increment_operation_queue: list[RedisPipelineIncrementOperation] = []
|
||||
|
|
@ -141,8 +148,8 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
self._start_background_work()
|
||||
|
||||
# Add self to litellm callbacks if it's a list
|
||||
if isinstance(litellm.callbacks, list):
|
||||
litellm.logging_callback_manager.add_litellm_callback(self)
|
||||
if isinstance(self._host.dynamic().callbacks, list):
|
||||
self._host.callbacks.add_litellm_callback(self)
|
||||
|
||||
def _start_background_work(self) -> None:
|
||||
"""Start the periodic Redis sync and the provider budget cache seeding once a loop runs.
|
||||
|
|
@ -810,7 +817,7 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
model = ""
|
||||
provider_resolution_params = _LiteLLMParamsDictView({})
|
||||
|
||||
_, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
_, custom_llm_provider, _, _ = self._host.invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # the host op table
|
||||
model=str(model),
|
||||
litellm_params=provider_resolution_params,
|
||||
)
|
||||
|
|
@ -826,7 +833,7 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
This is helpful for debugging and monitoring provider budget limits.
|
||||
"""
|
||||
|
||||
prometheus_logger: Final = get_prometheus_logger_from_callbacks()
|
||||
prometheus_logger: Final = get_prometheus_logger_from_callbacks(self._host)
|
||||
if prometheus_logger:
|
||||
prometheus_logger.track_provider_remaining_budget(
|
||||
provider=provider,
|
||||
|
|
@ -906,6 +913,7 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
def should_init_router_budget_limiter(
|
||||
provider_budget_config: dict | None,
|
||||
model_list: list[DeploymentTypedDict] | list[dict[str, Any]] | None = None,
|
||||
host: "PythonHost | None" = None,
|
||||
):
|
||||
"""
|
||||
Returns `True` if the router budget routing settings are set and RouterBudgetLimiting should be initialized
|
||||
|
|
@ -918,7 +926,9 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
if provider_budget_config is not None:
|
||||
return True
|
||||
|
||||
if litellm.tag_budget_config is not None:
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
if (python_host if host is None else host).dynamic().tag_budget_config is not None:
|
||||
return True
|
||||
|
||||
if model_list is None:
|
||||
|
|
@ -995,7 +1005,8 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
self.deployment_budget_config = None
|
||||
|
||||
def _init_tag_budgets(self):
|
||||
if litellm.tag_budget_config is None:
|
||||
tag_budget_config: Final = self._host.dynamic().tag_budget_config
|
||||
if tag_budget_config is None:
|
||||
return
|
||||
from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
|
||||
|
||||
|
|
@ -1005,7 +1016,7 @@ class RouterBudgetLimiting(CustomLogger):
|
|||
if self.tag_budget_config is None:
|
||||
self.tag_budget_config = {}
|
||||
|
||||
for _tag, _tag_budget_config in litellm.tag_budget_config.items():
|
||||
for _tag, _tag_budget_config in tag_budget_config.items():
|
||||
if isinstance(_tag_budget_config, dict):
|
||||
_tag_budget_config = BudgetConfig(**_tag_budget_config)
|
||||
_generic_budget_config = GenericBudgetInfo(
|
||||
|
|
|
|||
|
|
@ -2410,7 +2410,12 @@ class ComplexityRouter(CustomLogger):
|
|||
)
|
||||
|
||||
budgeted: Final = await offload_token_count(fit_jev_request)(
|
||||
build_request, prompt, system_prompt, jev_input_targets(self.ctx, config), _truncate
|
||||
build_request,
|
||||
prompt,
|
||||
system_prompt,
|
||||
jev_input_targets(self.ctx, config),
|
||||
_truncate,
|
||||
self.ctx.host().dynamic().disable_token_counter,
|
||||
)
|
||||
request: Final = budgeted.request
|
||||
if request is None:
|
||||
|
|
@ -2457,7 +2462,10 @@ class ComplexityRouter(CustomLogger):
|
|||
cost=response.classifier_cost
|
||||
if config.deployment_name
|
||||
else jev_classifier_cost(
|
||||
response, config.model, "typesafe" if config.provider == "jev" else config.provider
|
||||
response,
|
||||
config.model,
|
||||
"typesafe" if config.provider == "jev" else config.provider,
|
||||
cost_map=self.ctx.host().dynamic().model_cost,
|
||||
),
|
||||
provider=accounting_provider,
|
||||
)
|
||||
|
|
@ -3597,15 +3605,15 @@ class ComplexityRouter(CustomLogger):
|
|||
) -> int | None:
|
||||
"""Real-tokenizer count of the resolved messages plus the out-of-band carriers, off the
|
||||
event loop; None when counting fails, and the gate then leaves the placement alone."""
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
|
||||
token_counter: Final = self.ctx.host().invoke("token_counter") # pyright: ignore[reportUnknownMemberType] # the host op table
|
||||
out_of_band: Final = self._out_of_band_request_text(request_kwargs)
|
||||
try:
|
||||
counted: Final = await offload_token_count(litellm.token_counter)(
|
||||
counted: Final = await offload_token_count(token_counter)(
|
||||
messages=cast(list, resolved_messages) # cast-ok: token_counter only iterates the sequence
|
||||
)
|
||||
return counted + (await offload_token_count(litellm.token_counter)(text=out_of_band) if out_of_band else 0)
|
||||
return counted + (await offload_token_count(token_counter)(text=out_of_band) if out_of_band else 0)
|
||||
except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request
|
||||
verbose_router_logger.debug("ComplexityRouter: context-window token count failed. Got - %s", e)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ from uuid import uuid4
|
|||
import httpx
|
||||
from pydantic import ConfigDict, Field, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
|
||||
from litellm.litellm_core_utils.classifier_logging import masked_originating_request
|
||||
|
|
@ -115,12 +114,14 @@ def _jev_input_target(params: Mapping[str, object], explicit_limit: object) -> J
|
|||
return JevInputTarget(qualified_model, None)
|
||||
|
||||
|
||||
def _jev_request_count(request: JevSystemOneRequest, models: Sequence[str], budget: int) -> tuple[int, str]:
|
||||
def _jev_request_count(
|
||||
request: JevSystemOneRequest, models: Sequence[str], budget: int, disable_token_counter: bool
|
||||
) -> tuple[int, str]:
|
||||
from litellm._lazy_imports import get_token_counter_new
|
||||
|
||||
payload: Final = request.model_dump_json(exclude={"model"})
|
||||
byte_count: Final = len(payload.encode("utf-8"))
|
||||
if byte_count <= budget or litellm.disable_token_counter:
|
||||
if byte_count <= budget or disable_token_counter:
|
||||
return byte_count, "utf8_bytes"
|
||||
try:
|
||||
counts: Final = tuple(get_token_counter_new()(model=model, text=payload) for model in models)
|
||||
|
|
@ -137,7 +138,9 @@ def fit_jev_request(
|
|||
system_prompt: str | None,
|
||||
targets: Sequence[JevInputTarget],
|
||||
truncate: Callable[[str, int], str],
|
||||
disable_token_counter: bool,
|
||||
) -> JevBudgetedRequest:
|
||||
"""``disable_token_counter`` is the process setting, read through the router core's host."""
|
||||
known: Final = tuple(target for target in targets if target.max_tokens is not None)
|
||||
if not known:
|
||||
return JevBudgetedRequest(
|
||||
|
|
@ -158,6 +161,7 @@ def fit_jev_request(
|
|||
truncate,
|
||||
len(known) != len(targets),
|
||||
bounded_prompt != prompt or bounded_system != system_prompt,
|
||||
disable_token_counter,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -170,6 +174,7 @@ def _fit_jev_components(
|
|||
truncate: Callable[[str, int], str],
|
||||
unknown_limits: bool,
|
||||
truncated: bool,
|
||||
disable_token_counter: bool,
|
||||
) -> JevBudgetedRequest:
|
||||
attempts: Final = len(prompt).bit_length() + len(system_prompt or "").bit_length() + 1
|
||||
components: tuple[str, str | None, bool, int] = ( # rebind-ok: bounded iterative fitting state
|
||||
|
|
@ -181,7 +186,7 @@ def _fit_jev_components(
|
|||
while True:
|
||||
current_prompt, current_system, was_truncated, remaining = components
|
||||
request = build_request(current_prompt, current_system)
|
||||
count, method = _jev_request_count(request, models, budget)
|
||||
count, method = _jev_request_count(request, models, budget, disable_token_counter)
|
||||
telemetry: StandardLoggingRoutingDecision = {
|
||||
"classifier_input_budget_tokens": budget,
|
||||
"classifier_input_tokens": count,
|
||||
|
|
@ -452,19 +457,22 @@ def build_jev_request(
|
|||
|
||||
|
||||
def jev_classifier_cost(
|
||||
response: JevSystemOneResponse, configured_model: str, provider: ClassifierProvider = "typesafe"
|
||||
response: JevSystemOneResponse,
|
||||
configured_model: str,
|
||||
provider: ClassifierProvider = "typesafe",
|
||||
*,
|
||||
cost_map: Mapping[str, object],
|
||||
) -> float | None:
|
||||
"""The call's cost from ``cost_map`` (the process cost map, read through the router core's host)."""
|
||||
usage: Final = response.usage
|
||||
if usage is None:
|
||||
return None
|
||||
model: Final = response.model or configured_model
|
||||
model_key: Final = f"{provider}/{model}"
|
||||
if model_key not in litellm.model_cost: # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed
|
||||
if model_key not in cost_map:
|
||||
return None
|
||||
try:
|
||||
pricing: Final = _REGISTRY_PRICING_ADAPTER.validate_python(
|
||||
litellm.model_cost[model_key] # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed
|
||||
)
|
||||
pricing: Final = _REGISTRY_PRICING_ADAPTER.validate_python(cost_map[model_key])
|
||||
except ValidationError:
|
||||
return None
|
||||
return usage.input_tokens * pricing.input_cost_per_token + usage.output_tokens * pricing.output_cost_per_token
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@
|
|||
# picks based on response time (for streaming, this is time to first token)
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
from litellm import ModelResponse, token_counter, verbose_logger
|
||||
from litellm import ModelResponse, Usage, token_counter, verbose_logger
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
|
|
@ -13,22 +12,21 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router_python_host import PythonHost
|
||||
|
||||
def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]:
|
||||
"""Input and output per-token prices from the first cost-map row among the deployment's
|
||||
|
||||
def _deployment_cost_map_rates(deployment: object, cost_map: Mapping[str, object]) -> tuple[float | None, float | None]:
|
||||
"""Input and output per-token prices from the first ``cost_map`` row among the deployment's
|
||||
pricing models that carries one; (None, None) when no row does, so the caller falls back to
|
||||
its unmapped default."""
|
||||
rows: Final = (str_keyed(_cost_map().get(model)) for model in deployment_pricing_models(deployment))
|
||||
rows: Final = (str_keyed(cost_map.get(model)) for model in deployment_pricing_models(deployment))
|
||||
row: Final = next((row for row in rows if _declares_token_price(row)), None)
|
||||
if row is None:
|
||||
return None, None
|
||||
return _as_rate(row.get("input_cost_per_token")), _as_rate(row.get("output_cost_per_token"))
|
||||
|
||||
|
||||
def _cost_map() -> Mapping[str, object]:
|
||||
return litellm.model_cost
|
||||
|
||||
|
||||
def _declares_token_price(row: Mapping[str, object]) -> bool:
|
||||
return row.get("input_cost_per_token") is not None or row.get("output_cost_per_token") is not None
|
||||
|
||||
|
|
@ -42,8 +40,12 @@ class LowestCostLoggingHandler(CustomLogger):
|
|||
logged_success: int = 0
|
||||
logged_failure: int = 0
|
||||
|
||||
def __init__(self, router_cache: DualCache, routing_args: dict = {}):
|
||||
def __init__(self, router_cache: DualCache, routing_args: dict = {}, host: "PythonHost | None" = None):
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
self.router_cache = router_cache
|
||||
# The router core's host: the cost map is read through it.
|
||||
self._host: Final = python_host if host is None else host
|
||||
|
||||
@with_service_target("router_usage")
|
||||
def log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
|
|
@ -86,7 +88,7 @@ class LowestCostLoggingHandler(CustomLogger):
|
|||
|
||||
if isinstance(response_obj, ModelResponse):
|
||||
_usage: Final = getattr(response_obj, "usage", None)
|
||||
if _usage is not None and isinstance(_usage, litellm.Usage):
|
||||
if _usage is not None and isinstance(_usage, Usage):
|
||||
total_tokens = _usage.total_tokens
|
||||
|
||||
# ------------
|
||||
|
|
@ -163,7 +165,7 @@ class LowestCostLoggingHandler(CustomLogger):
|
|||
|
||||
if isinstance(response_obj, ModelResponse):
|
||||
_usage: Final = getattr(response_obj, "usage", None)
|
||||
if _usage is not None and isinstance(_usage, litellm.Usage):
|
||||
if _usage is not None and isinstance(_usage, Usage):
|
||||
total_tokens = _usage.total_tokens
|
||||
|
||||
# ------------
|
||||
|
|
@ -286,7 +288,9 @@ class LowestCostLoggingHandler(CustomLogger):
|
|||
or float("inf")
|
||||
)
|
||||
deployment_view: object = _deployment
|
||||
map_input_cost, map_output_cost = _deployment_cost_map_rates(deployment_view)
|
||||
map_input_cost, map_output_cost = _deployment_cost_map_rates(
|
||||
deployment_view, self._host.dynamic().model_cost
|
||||
)
|
||||
|
||||
# check if user provided input_cost_per_token and output_cost_per_token in litellm_params
|
||||
item_input_cost = None
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.router_python_host import PythonHost
|
||||
|
||||
|
||||
_REFERENCE_REQUEST: Final = Usage(
|
||||
|
|
@ -46,7 +47,7 @@ class Baseline(NamedTuple):
|
|||
deployment_id: str | None = None
|
||||
|
||||
|
||||
def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None:
|
||||
def canonical_model(model: str, custom_llm_provider: str | None = None, host: "PythonHost | None" = None) -> str | None:
|
||||
"""``provider/model``, or ``None`` when the pair names no known provider.
|
||||
|
||||
A deployment may name its vendor in the model prefix or in a separate
|
||||
|
|
@ -56,15 +57,19 @@ def canonical_model(model: str, custom_llm_provider: str | None = None) -> str |
|
|||
A github_copilot or chatgpt candidate is qualified by string alone: resolving either
|
||||
provider runs its OAuth device flow, and for a declared pair the resolver's answer is
|
||||
the declaration itself, so asking it buys nothing but the block.
|
||||
|
||||
The provider is resolved through ``host`` (the router core's host; the process host when none is given).
|
||||
"""
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
|
||||
if declared is not None:
|
||||
return f"{declared}/{model.removeprefix(f'{declared}/')}"
|
||||
try:
|
||||
resolved, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
resolved, provider, _, _ = (python_host if host is None else host).invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # the host op table
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline
|
||||
verbose_router_logger.debug("savings baseline: cannot resolve candidate %s (%s)", model, e)
|
||||
return None
|
||||
|
|
@ -83,7 +88,7 @@ def _models_in(ctx: "RoutingContext", group_name: str) -> tuple[Baseline, ...]:
|
|||
"""
|
||||
deployments: Final = ctx.registry().deployments(group_name, source="named")
|
||||
if not deployments:
|
||||
return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else ()
|
||||
return (Baseline(qualified),) if (qualified := canonical_model(group_name, host=ctx.host())) else ()
|
||||
|
||||
def candidate(deployment: Mapping[str, object]) -> Baseline | None:
|
||||
params: Final = deployment.get("litellm_params")
|
||||
|
|
@ -92,7 +97,7 @@ def _models_in(ctx: "RoutingContext", group_name: str) -> tuple[Baseline, ...]:
|
|||
info: Final = deployment.get("model_info")
|
||||
base: Final = info.get("base_model") if isinstance(info, dict) else None
|
||||
model: Final = base or params.get("base_model") or params.get("model")
|
||||
qualified: Final = canonical_model(model, params.get("custom_llm_provider")) if model else None
|
||||
qualified: Final = canonical_model(model, params.get("custom_llm_provider"), ctx.host()) if model else None
|
||||
if qualified is None:
|
||||
return None
|
||||
deployment_id: Final = info.get("id") if isinstance(info, dict) else None
|
||||
|
|
|
|||
|
|
@ -5,13 +5,13 @@ Callbacks triggered on cooling down deployments
|
|||
import copy
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.router_deprecated import renamed_router_param
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.router_python_host import PythonHost
|
||||
else:
|
||||
PrometheusLogger = Any
|
||||
|
||||
|
|
@ -54,14 +54,17 @@ def emit_router_cooldown_event(
|
|||
temp_litellm_params = copy.deepcopy(_litellm_params)
|
||||
temp_litellm_params = dict(temp_litellm_params)
|
||||
_model_name: Final = _deployment.get("model_name", None) or ""
|
||||
_api_base: Final = litellm.get_api_base(model=_model_name, optional_params=temp_litellm_params) or ""
|
||||
host: Final = ctx.host()
|
||||
_api_base: Final = str(
|
||||
host.invoke("get_api_base")(model=_model_name, optional_params=temp_litellm_params) or "" # pyright: ignore[reportUnknownMemberType] # the host op table
|
||||
)
|
||||
model_info: Final = _deployment["model_info"]
|
||||
model_id: Final = model_info.id
|
||||
|
||||
litellm_model_name: Final = temp_litellm_params.get("model") or ""
|
||||
llm_provider = ""
|
||||
try:
|
||||
_, llm_provider, _, _ = litellm.get_llm_provider(
|
||||
_, llm_provider, _, _ = host.invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # the host op table
|
||||
model=litellm_model_name,
|
||||
custom_llm_provider=temp_litellm_params.get("custom_llm_provider"),
|
||||
)
|
||||
|
|
@ -69,7 +72,7 @@ def emit_router_cooldown_event(
|
|||
pass
|
||||
|
||||
# get the prometheus logger from in memory loggers
|
||||
prometheusLogger: Final[PrometheusLogger | None] = get_prometheus_logger_from_callbacks()
|
||||
prometheusLogger: Final[PrometheusLogger | None] = get_prometheus_logger_from_callbacks(host)
|
||||
|
||||
if prometheusLogger is not None:
|
||||
prometheusLogger.set_deployment_complete_outage(
|
||||
|
|
@ -88,19 +91,22 @@ def emit_router_cooldown_event(
|
|||
)
|
||||
|
||||
|
||||
def get_prometheus_logger_from_callbacks() -> PrometheusLogger | None:
|
||||
def get_prometheus_logger_from_callbacks(host: "PythonHost | None" = None) -> PrometheusLogger | None:
|
||||
"""
|
||||
Checks if prometheus is a initalized callback, if yes returns it
|
||||
Checks if prometheus is a initalized callback, if yes returns it. The callback lists are read
|
||||
through ``host`` (the router core's host; the process host when none is given).
|
||||
"""
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
if PrometheusLogger is None:
|
||||
return None
|
||||
|
||||
for _callback in litellm._async_success_callback:
|
||||
source: Final = python_host if host is None else host
|
||||
for _callback in source.callback_list("_async_success_callback"):
|
||||
if isinstance(_callback, PrometheusLogger):
|
||||
return _callback
|
||||
for global_callback in litellm.callbacks:
|
||||
for global_callback in source.dynamic().callbacks:
|
||||
if isinstance(global_callback, PrometheusLogger):
|
||||
return global_callback
|
||||
|
||||
|
|
|
|||
|
|
@ -453,7 +453,9 @@ def _has_fallback_group(ctx: "RoutingContext", requested_model_group: str | None
|
|||
chains: Final = tuple(
|
||||
chain for chain in map(_fallback_chain_names, _fallback_entries(configured)) if chain is not None
|
||||
)
|
||||
fallback_group, _ = get_fallback_model_group(fallbacks=chains, model_group=requested_model_group)
|
||||
fallback_group, _ = get_fallback_model_group(
|
||||
fallbacks=chains, model_group=requested_model_group, provider_list=ctx.host().dynamic().provider_list
|
||||
)
|
||||
return bool(fallback_group)
|
||||
|
||||
|
||||
|
|
@ -544,7 +546,7 @@ def _should_cooldown_deployment(
|
|||
# Only apply error rate cooldown when we have enough requests to make the percentage meaningful
|
||||
return True
|
||||
|
||||
elif litellm.should_retry(status_code=exception_status_int) is False:
|
||||
elif ctx.host().invoke("should_retry")(status_code=exception_status_int) is False: # pyright: ignore[reportUnknownMemberType] # the host op table
|
||||
return True
|
||||
|
||||
return False
|
||||
|
|
@ -819,7 +821,7 @@ def _is_allowed_fails_set_on_router(
|
|||
return False
|
||||
if ctx.explicit_allowed_fails is not None:
|
||||
return True
|
||||
if ctx.settings().allowed_fails != litellm.allowed_fails:
|
||||
if ctx.settings().allowed_fails != ctx.host().dynamic().allowed_fails:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ from litellm.types.router import (
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router_api import RoutingContext
|
||||
from litellm.router_python_host import PythonHost
|
||||
|
||||
# Status codes a generic API call's caller-supplied resource id can trigger on its own
|
||||
# (e.g. a nonexistent file/batch/thread id), independent of the selected deployment's health.
|
||||
|
|
@ -266,7 +267,7 @@ def is_bad_request_without_fallback(exception: Exception, fallback_on_unsupporte
|
|||
return exception.status_code == 400 and bool(getattr(exception, "llm_provider", ""))
|
||||
|
||||
|
||||
def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool:
|
||||
def _check_stripped_model_group(model_group: str, fallback_key: str, provider_list: Sequence[object]) -> bool:
|
||||
"""
|
||||
Handles wildcard routing scenario
|
||||
|
||||
|
|
@ -279,11 +280,8 @@ def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool:
|
|||
Returns:
|
||||
- True if the stripped model group == fallback_key
|
||||
"""
|
||||
for provider in litellm.provider_list:
|
||||
if isinstance(provider, Enum):
|
||||
_provider = provider.value
|
||||
else:
|
||||
_provider = provider
|
||||
for provider in provider_list:
|
||||
_provider: str = str(provider.value) if isinstance(provider, Enum) else str(provider) # pyright: ignore[reportAny] # an Enum member's value
|
||||
if model_group.startswith(f"{_provider}/"):
|
||||
stripped_model_group = model_group.replace(f"{_provider}/", "")
|
||||
if stripped_model_group == fallback_key:
|
||||
|
|
@ -588,12 +586,16 @@ def _resolved_a_specific_chain(
|
|||
def get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: mirrors the fallbacks config shape
|
||||
lookup_groups: tuple[str, ...],
|
||||
provider_list: Sequence[object] | None = None,
|
||||
) -> tuple[list[str] | None, int | None]: # mutable-ok: mirrors get_fallback_model_group's contract
|
||||
"""
|
||||
First lookup group with a specifically-keyed chain wins; the generic "*" chain applies
|
||||
only after every group missed, so a catch-all cannot shadow a later group's own chain.
|
||||
"""
|
||||
results: Final = tuple(get_fallback_model_group(fallbacks=fallbacks, model_group=group) for group in lookup_groups)
|
||||
results: Final = tuple(
|
||||
get_fallback_model_group(fallbacks=fallbacks, model_group=group, provider_list=provider_list)
|
||||
for group in lookup_groups
|
||||
)
|
||||
specific: Final = next((result for result in results if _resolved_a_specific_chain(fallbacks, result)), None)
|
||||
if specific is not None:
|
||||
return specific
|
||||
|
|
@ -603,8 +605,12 @@ def get_fallback_model_group_for_lookup_groups(
|
|||
def get_fallback_model_group(
|
||||
fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: mirrors the fallbacks config shape
|
||||
model_group: str,
|
||||
provider_list: Sequence[object] | None = None,
|
||||
) -> tuple[list[str] | None, int | None]: # mutable-ok: mirrors get_fallback_model_group's contract
|
||||
"""
|
||||
``provider_list`` is the process provider list, read through the router core's host (the
|
||||
process host's when none is given).
|
||||
|
||||
Returns:
|
||||
- fallback_model_group: List[str] of fallback model groups. example: ["gpt-4", "gpt-3.5-turbo"]
|
||||
- generic_fallback_idx: int of the index of the generic fallback in the fallbacks list.
|
||||
|
|
@ -615,6 +621,9 @@ def get_fallback_model_group(
|
|||
- provider-prefixed model group match
|
||||
- generic fallback
|
||||
"""
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
providers: Final = python_host.dynamic().provider_list if provider_list is None else provider_list
|
||||
generic_fallback_idx: int | None = None
|
||||
stripped_model_fallback: list[str] | None = None
|
||||
fallback_model_group: list[str] | None = None
|
||||
|
|
@ -628,7 +637,7 @@ def get_fallback_model_group(
|
|||
fallback_model_group = item[model_group]
|
||||
break
|
||||
elif fallback_key == prefixed_model_group or _check_stripped_model_group(
|
||||
model_group=model_group, fallback_key=fallback_key
|
||||
model_group=model_group, fallback_key=fallback_key, provider_list=providers
|
||||
): # check generic fallback
|
||||
stripped_model_fallback = item[fallback_key]
|
||||
elif fallback_key == "*": # check generic fallback
|
||||
|
|
@ -645,20 +654,26 @@ def get_fallback_model_group(
|
|||
return fallback_model_group, generic_fallback_idx
|
||||
|
||||
|
||||
def _resolved_fallback_chain(entries: Sequence[object], lookup_groups: tuple[str, ...]) -> Sequence[object]:
|
||||
def _resolved_fallback_chain(
|
||||
entries: Sequence[object], lookup_groups: tuple[str, ...], provider_list: Sequence[object] | None
|
||||
) -> Sequence[object]:
|
||||
group_entries: Final[Sequence[Mapping[str, list[str]]]] = tuple( # mutable-ok: mirrors the fallbacks config shape
|
||||
entry for entry in entries if isinstance(entry, Mapping) and entry
|
||||
)
|
||||
chain, _ = get_fallback_model_group_for_lookup_groups(fallbacks=group_entries, lookup_groups=lookup_groups)
|
||||
chain, _ = get_fallback_model_group_for_lookup_groups(
|
||||
fallbacks=group_entries, lookup_groups=lookup_groups, provider_list=provider_list
|
||||
)
|
||||
return chain if isinstance(chain, list) else ()
|
||||
|
||||
|
||||
def continuation_fallback_targets(fallbacks: object, lookup_groups: tuple[str, ...]) -> Sequence[object]:
|
||||
def continuation_fallback_targets(
|
||||
fallbacks: object, lookup_groups: tuple[str, ...], provider_list: Sequence[object] | None = None
|
||||
) -> Sequence[object]:
|
||||
entries: Final[Sequence[object]] = fallbacks if isinstance(fallbacks, list) else ()
|
||||
return (
|
||||
entries
|
||||
if check_non_standard_fallback_format(fallbacks=entries)
|
||||
else _resolved_fallback_chain(entries, lookup_groups)
|
||||
else _resolved_fallback_chain(entries, lookup_groups, provider_list)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -901,6 +916,7 @@ async def run_async_fallback(
|
|||
original_model_group=original_model_group,
|
||||
kwargs=kwargs,
|
||||
original_exception=original_exception,
|
||||
host=litellm_routing_ctx.host(),
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
|
|
@ -910,6 +926,7 @@ async def run_async_fallback(
|
|||
original_model_group=original_model_group,
|
||||
kwargs=kwargs,
|
||||
original_exception=original_exception,
|
||||
host=litellm_routing_ctx.host(),
|
||||
)
|
||||
if (
|
||||
failure_callbacks_blocked_logging_obj := _failure_callbacks_blocked_logging_object(
|
||||
|
|
@ -925,7 +942,9 @@ async def run_async_fallback(
|
|||
raise error_from_fallbacks
|
||||
|
||||
|
||||
async def log_success_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception):
|
||||
async def log_success_fallback_event(
|
||||
original_model_group: str, kwargs: dict, original_exception: Exception, host: "PythonHost | None" = None
|
||||
):
|
||||
"""
|
||||
Log a successful fallback event to all registered callbacks.
|
||||
|
||||
|
|
@ -940,7 +959,9 @@ async def log_success_fallback_event(original_model_group: str, kwargs: dict, or
|
|||
Errors during logging are caught and reported but do not interrupt the process.
|
||||
"""
|
||||
# Get deduplicated CustomLogger instances from all callback lists
|
||||
custom_loggers: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger)
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
custom_loggers: Final = (python_host if host is None else host).callbacks.get_custom_loggers_for_type(CustomLogger)
|
||||
|
||||
for _callback_custom_logger in custom_loggers:
|
||||
try:
|
||||
|
|
@ -953,7 +974,9 @@ async def log_success_fallback_event(original_model_group: str, kwargs: dict, or
|
|||
verbose_router_logger.error("Error in log_success_fallback_event: %s", e)
|
||||
|
||||
|
||||
async def log_failure_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception):
|
||||
async def log_failure_fallback_event(
|
||||
original_model_group: str, kwargs: dict, original_exception: Exception, host: "PythonHost | None" = None
|
||||
):
|
||||
"""
|
||||
Log a failed fallback event to all registered callbacks.
|
||||
|
||||
|
|
@ -968,7 +991,9 @@ async def log_failure_fallback_event(original_model_group: str, kwargs: dict, or
|
|||
Errors during logging are caught and reported but do not interrupt the process.
|
||||
"""
|
||||
# Get deduplicated CustomLogger instances from all callback lists
|
||||
custom_loggers: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger)
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
custom_loggers: Final = (python_host if host is None else host).callbacks.get_custom_loggers_for_type(CustomLogger)
|
||||
|
||||
for _callback_custom_logger in custom_loggers:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -7,11 +7,10 @@ from dataclasses import dataclass
|
|||
from itertools import accumulate
|
||||
from os.path import commonprefix
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeVar
|
||||
from typing import TYPE_CHECKING, Final, TypeVar
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.litellm_core_utils.routing_common_utils import MID_STREAM_CONTINUATION_KWARG
|
||||
from litellm.litellm_core_utils.rules import Rules
|
||||
|
|
@ -19,6 +18,9 @@ from litellm.types.llms.openai import AllMessageValues, OutputTextDeltaEvent
|
|||
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
|
||||
from litellm.utils import supports_assistant_prefill
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router_python_host import PythonHost
|
||||
|
||||
|
||||
class _ContinuationMarker:
|
||||
"""Only the router constructs one, so a JSON request body cannot forge the continuation flag."""
|
||||
|
|
@ -219,7 +221,7 @@ class ContinuationOutput:
|
|||
"""Splices a fallback's continuation onto the stream the client already received. Drops a leading echo of the
|
||||
prefill, since some providers (Mistral) return the prefix inside the completion, and the prefill's trailing
|
||||
whitespace sent again by a provider that strips it (Anthropic), keeps the primary's chunk id, created time and single role delta, and
|
||||
runs litellm.post_call_rules over the joined text, since each stream wrapper only sees its own fragment. A
|
||||
runs the process post_call_rules (read through the router core's host) over the joined text, since each stream wrapper only sees its own fragment. A
|
||||
continuation nested in another leaves the rules to the outermost one, the only one holding the whole text."""
|
||||
|
||||
def __init__(
|
||||
|
|
@ -229,7 +231,11 @@ class ContinuationOutput:
|
|||
model: str,
|
||||
primary_response: object,
|
||||
checks_post_call_rules: bool = True,
|
||||
host: "PythonHost | None" = None,
|
||||
) -> None:
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
self._host: Final = python_host if host is None else host
|
||||
self._text = emitted_text
|
||||
self._checks_post_call_rules: Final = checks_post_call_rules
|
||||
self._prefill_text: Final = _prefill_text(prefill_message)
|
||||
|
|
@ -283,7 +289,7 @@ class ContinuationOutput:
|
|||
return _without_leading_text(chunk, matched)
|
||||
|
||||
def _observed(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]:
|
||||
if not self._checks_post_call_rules or not litellm.post_call_rules:
|
||||
if not self._checks_post_call_rules or not self._host.dynamic().post_call_rules:
|
||||
return chunks
|
||||
for chunk in chunks:
|
||||
self._text += _delta_text(chunk)
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ from litellm.utils import get_utc_datetime
|
|||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.router_python_host import PythonHost
|
||||
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
|
@ -179,11 +181,11 @@ def _estimate_input_tokens(request_kwargs: dict[str, Any] | None, model: str = "
|
|||
return 0
|
||||
|
||||
|
||||
def _model_max_output_tokens(model_name: str) -> int | None:
|
||||
# litellm.get_model_info raises a bare Exception for an unrecognized model;
|
||||
# this lookup is a fallback default and must never fail the request.
|
||||
def _model_max_output_tokens(model_name: str, host: PythonHost) -> int | None:
|
||||
# get_model_info (through the router core's host) raises a bare Exception for an
|
||||
# unrecognized model; this lookup is a fallback default and must never fail the request.
|
||||
with contextlib.suppress(Exception):
|
||||
info: Final = litellm.get_model_info(model=model_name)
|
||||
info: Final = host.invoke("get_model_info")(model=model_name) # pyright: ignore[reportUnknownMemberType] # the host op table
|
||||
model_max: Final = info.get("max_output_tokens")
|
||||
if model_max is not None:
|
||||
return max(0, int(model_max))
|
||||
|
|
@ -227,8 +229,8 @@ def _deployment_output_cap(request_kwargs: Mapping[str, object] | None, deployme
|
|||
)
|
||||
|
||||
|
||||
def _default_otpm_reservation(model_name: str, otpm_limit: int) -> int:
|
||||
model_max: Final = _model_max_output_tokens(model_name) if model_name else None
|
||||
def _default_otpm_reservation(model_name: str, otpm_limit: int, host: PythonHost) -> int:
|
||||
model_max: Final = _model_max_output_tokens(model_name, host) if model_name else None
|
||||
share_of_limit: Final = max(1, int(otpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION))
|
||||
return min(4096 if model_max is None else model_max, share_of_limit)
|
||||
|
||||
|
|
@ -238,11 +240,12 @@ def _otpm_reservation(
|
|||
deployment: Mapping[str, object],
|
||||
model_name: str,
|
||||
otpm_limit: int,
|
||||
host: PythonHost,
|
||||
) -> int:
|
||||
from_request: Final = _explicit_max_tokens(request_kwargs)
|
||||
if from_request is not None:
|
||||
return from_request
|
||||
default: Final = _default_otpm_reservation(model_name, otpm_limit)
|
||||
default: Final = _default_otpm_reservation(model_name, otpm_limit, host)
|
||||
from_deployment: Final = _deployment_output_cap(request_kwargs, deployment)
|
||||
return default if from_deployment is None else min(from_deployment, default)
|
||||
|
||||
|
|
@ -515,7 +518,11 @@ async def _increment_with_rollback(
|
|||
def io_token_pre_call_check(
|
||||
dual_cache: DualCache,
|
||||
deployment: dict,
|
||||
host: PythonHost | None = None,
|
||||
) -> dict | None:
|
||||
"""``host`` is the router core's host (the process host when none is given)."""
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
itpm_limit, otpm_limit = get_deployment_io_token_limits(deployment)
|
||||
if itpm_limit is None and otpm_limit is None:
|
||||
return deployment
|
||||
|
|
@ -525,7 +532,9 @@ def io_token_pre_call_check(
|
|||
_model, served_model = _tokenizer_and_served_models(deployment)
|
||||
estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model)
|
||||
otpm_to_reserve: Final = (
|
||||
_otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0
|
||||
_otpm_reservation(request_kwargs, deployment, served_model, otpm_limit, python_host if host is None else host)
|
||||
if otpm_limit is not None
|
||||
else 0
|
||||
)
|
||||
|
||||
dt: Final = get_utc_datetime()
|
||||
|
|
@ -584,7 +593,11 @@ async def async_io_token_pre_call_check(
|
|||
dual_cache: DualCache,
|
||||
deployment: dict,
|
||||
parent_otel_span: Span | None = None,
|
||||
host: PythonHost | None = None,
|
||||
) -> dict | None:
|
||||
"""``host`` is the router core's host (the process host when none is given)."""
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
itpm_limit, otpm_limit = get_deployment_io_token_limits(deployment)
|
||||
if itpm_limit is None and otpm_limit is None:
|
||||
return deployment
|
||||
|
|
@ -600,7 +613,9 @@ async def async_io_token_pre_call_check(
|
|||
_model, served_model = _tokenizer_and_served_models(deployment)
|
||||
estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model)
|
||||
otpm_to_reserve: Final = (
|
||||
_otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0
|
||||
_otpm_reservation(request_kwargs, deployment, served_model, otpm_limit, python_host if host is None else host)
|
||||
if otpm_limit is not None
|
||||
else 0
|
||||
)
|
||||
|
||||
dt: Final = get_utc_datetime()
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ from litellm.utils import get_utc_datetime
|
|||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.router_python_host import PythonHost
|
||||
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
|
@ -134,7 +136,14 @@ class ModelRateLimitingCheck(CustomLogger):
|
|||
this check actively enforces those limits across ALL routing strategies.
|
||||
"""
|
||||
|
||||
def __init__(self, dual_cache: DualCache, clock: Callable[[], datetime] = get_utc_datetime):
|
||||
def __init__(
|
||||
self,
|
||||
dual_cache: DualCache,
|
||||
clock: Callable[[], datetime] = get_utc_datetime,
|
||||
host: "PythonHost | None" = None,
|
||||
):
|
||||
# The router core's host: the io-token checks read model info through it.
|
||||
self._host: PythonHost | None = host
|
||||
self.dual_cache = dual_cache
|
||||
self.clock = clock
|
||||
# model_ids already warned about itpm/otpm + tpm/rpm on the same deployment,
|
||||
|
|
@ -312,6 +321,7 @@ class ModelRateLimitingCheck(CustomLogger):
|
|||
io_token_pre_call_check(
|
||||
self.dual_cache,
|
||||
deployment,
|
||||
host=self._host,
|
||||
)
|
||||
io_reservation_made = True
|
||||
|
||||
|
|
@ -397,6 +407,7 @@ class ModelRateLimitingCheck(CustomLogger):
|
|||
self.dual_cache,
|
||||
deployment,
|
||||
parent_otel_span=parent_otel_span,
|
||||
host=self._host,
|
||||
)
|
||||
io_reservation_made = True
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router_python_host import PythonHost
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InvalidPriority:
|
||||
|
|
@ -84,9 +87,15 @@ def request_drops_params(
|
|||
kwargs: Mapping[str, object],
|
||||
router_defaults: Mapping[str, object],
|
||||
deployment_params: Iterable[Mapping[str, object]],
|
||||
host: "PythonHost | None" = None,
|
||||
) -> bool:
|
||||
"""``host`` is the router core's host, whose process ``drop_params`` counts (the process host when none is given)."""
|
||||
from litellm.router_python_host import python_host
|
||||
|
||||
deployments: Final = tuple(deployment_params)
|
||||
if litellm.drop_params is True or any(_drops_priority_by_name(s) for s in (kwargs, router_defaults, *deployments)):
|
||||
if (python_host if host is None else host).dynamic().drop_params is True or any(
|
||||
_drops_priority_by_name(s) for s in (kwargs, router_defaults, *deployments)
|
||||
):
|
||||
return True
|
||||
request_flag: Final = _explicit_drop_params(kwargs)
|
||||
if request_flag is not None:
|
||||
|
|
|
|||
|
|
@ -874,7 +874,7 @@ def test_jev_classifier_cost_uses_registry_pricing(monkeypatch: pytest.MonkeyPat
|
|||
answers={"tier": _answer()},
|
||||
usage=JevUsage(input_tokens=3, output_tokens=4),
|
||||
)
|
||||
assert jev_classifier_cost(response, "jev-latest") == pytest.approx(0.0011)
|
||||
assert jev_classifier_cost(response, "jev-latest", cost_map=litellm.model_cost) == pytest.approx(0.0011)
|
||||
|
||||
|
||||
def test_jev_classifier_cost_is_none_without_registry_pricing() -> None:
|
||||
|
|
@ -883,7 +883,7 @@ def test_jev_classifier_cost_is_none_without_registry_pricing() -> None:
|
|||
answers={"tier": _answer()},
|
||||
usage=JevUsage(input_tokens=3, output_tokens=4),
|
||||
)
|
||||
assert jev_classifier_cost(response, "jev-unpriced") is None
|
||||
assert jev_classifier_cost(response, "jev-unpriced", cost_map=litellm.model_cost) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ async def test_get_llm_provider_for_deployment_dict_view_supports_mapping_and_at
|
|||
return model, "openai", None, None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"litellm.router_strategy.budget_limiter.litellm.get_llm_provider",
|
||||
"litellm.get_llm_provider",
|
||||
_future_style_get_llm_provider,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -425,10 +425,10 @@ class FallbackEvents(CustomLogger):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_events_reach_the_process_custom_loggers_innermost_hop_first(
|
||||
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
events = FallbackEvents()
|
||||
monkeypatch.setattr(litellm, "callbacks", [events])
|
||||
host.callbacks.add_litellm_callback(events)
|
||||
core = build(make_core, fallbacks=[{"a": ["b", "c"]}])
|
||||
host.answer("openai/a", rate_limit())
|
||||
host.answer("openai/b", server_error())
|
||||
|
|
@ -443,10 +443,10 @@ async def test_fallback_events_reach_the_process_custom_loggers_innermost_hop_fi
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_fallback_that_fails_everywhere_logs_failure_events(
|
||||
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
events = FallbackEvents()
|
||||
monkeypatch.setattr(litellm, "callbacks", [events])
|
||||
host.callbacks.add_litellm_callback(events)
|
||||
core = build(make_core, fallbacks=[{"a": ["b"]}])
|
||||
host.answer("*", rate_limit())
|
||||
|
||||
|
|
|
|||
|
|
@ -267,10 +267,10 @@ def _stale(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_stale_routing_group_at_a_fallback_level_is_that_levels_error(
|
||||
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
events: Final = FallbackEvents()
|
||||
monkeypatch.setattr(litellm, "callbacks", [events])
|
||||
host.callbacks.add_litellm_callback(events)
|
||||
core: Final = _stale(make_core)
|
||||
host.answer("*", rate_limit())
|
||||
|
||||
|
|
|
|||
|
|
@ -531,6 +531,9 @@ async def test_a_deployment_cools_down_once_its_failures_pass_allowed_fails(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_allowed_fails_is_the_default(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
"""The host's process ``allowed_fails`` is the router's default, and a router left on the default
|
||||
does not count it as set on the router: one failure cools nothing down, as in a process whose
|
||||
``litellm.allowed_fails`` is 0."""
|
||||
host.set_dynamic(allowed_fails=0)
|
||||
core: Final = make_core(model_list=two_deployments(), num_retries=0)
|
||||
host.answer("*", fresh(server_error))
|
||||
|
|
@ -538,6 +541,20 @@ async def test_process_allowed_fails_is_the_default(make_core: MakeCore, host: S
|
|||
await attempt(core)
|
||||
|
||||
assert core.settings().allowed_fails == 0
|
||||
assert list(cooled_down(core, ["a", "b"])) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_allowed_fails_set_on_the_router_cools_down_after_that_many_failures(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""An ``allowed_fails`` passed to the router counts as set on it, even when it equals the host's default."""
|
||||
host.set_dynamic(allowed_fails=0)
|
||||
core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0)
|
||||
host.answer("*", fresh(server_error))
|
||||
|
||||
await attempt(core)
|
||||
|
||||
assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion")
|
||||
|
||||
|
||||
|
|
|
|||
54
tests/unit/test_router/core/test_router_host_boundary.py
Normal file
54
tests/unit/test_router/core/test_router_host_boundary.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
"""
|
||||
Both router cores and the helpers they share reach the process only through their host: no router
|
||||
module reads a ``litellm.<setting>`` or calls a ``litellm.<function>`` directly. Exception classes,
|
||||
types and modules (``litellm.RateLimitError``, ``litellm.Usage``, ``litellm.utils``) are code, not
|
||||
process state, and stay importable.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
||||
_PACKAGE: Final = Path(litellm.__file__).parent
|
||||
# router_config.py is the host's own view of the process settings; router_python_host.py is the host.
|
||||
_HOST_MODULES: Final = frozenset({"router_config.py", "router_python_host.py"})
|
||||
|
||||
|
||||
def _router_modules() -> tuple[Path, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
path
|
||||
for path in (
|
||||
*_PACKAGE.glob("router_*.py"),
|
||||
*(_PACKAGE / "router_utils").rglob("*.py"),
|
||||
*(_PACKAGE / "router_strategy").rglob("*.py"),
|
||||
)
|
||||
if path.name not in _HOST_MODULES
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _is_code(name: str) -> bool:
|
||||
value: Final[object] = getattr(litellm, name, None)
|
||||
return inspect.isclass(value) or inspect.ismodule(value)
|
||||
|
||||
|
||||
def _process_reads(path: Path) -> tuple[str, ...]:
|
||||
tree: Final = ast.parse(path.read_text(encoding="utf-8"))
|
||||
return tuple(
|
||||
f"{path.relative_to(_PACKAGE.parent)}:{node.lineno} litellm.{node.attr}"
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Attribute)
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id == "litellm"
|
||||
and not _is_code(node.attr)
|
||||
)
|
||||
|
||||
|
||||
def test_router_code_reaches_litellm_settings_and_functions_only_through_the_host() -> None:
|
||||
modules: Final = _router_modules()
|
||||
assert len(modules) > 50
|
||||
assert [read for path in modules for read in _process_reads(path)] == []
|
||||
Loading…
Add table
Reference in a new issue