refactor(router): route every router read of litellm settings and functions through the host

Both router cores and the Python helpers they share (cooldown handlers and callbacks, budget limiter,
lowest-cost strategy, complexity router, savings baseline, continuation prefill, io-token and model
rate-limit checks, request priority, fallback event handlers) now read process settings through
host.dynamic() and call litellm functions through host.invoke(). RustCore's allowed_fails default
comes from the host too. A guard test fails if any router module reads litellm.<setting> again.
This commit is contained in:
narmstrong-berry 2026-11-02 04:59:06 +00:00
parent 1a7deafe09
commit 1e08643bd3
No known key found for this signature in database
27 changed files with 349 additions and 122 deletions

View file

@ -59,6 +59,7 @@ if TYPE_CHECKING:
from litellm.messages.dispatch import MessagesResult
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.router import Router
from litellm.router_python_host import PythonHost
from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter
from litellm.router_strategy.auto_router.auto_router import AutoRouter
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
@ -1474,6 +1475,16 @@ class RoutingContext:
def belongs_to(self, router: "Router") -> bool:
return self._router is router or router.__dict__.get("_core") is self._router
def host(self) -> "PythonHost":
"""The router core's host: the process settings (``dynamic()``) and litellm functions
(``invoke``) a plugin reads go through it, never ``litellm.*`` directly."""
from litellm.router_python_host import PythonHost, python_host
core: Final[object] = self._router.__dict__.get("_core", self._router) # pyright: ignore[reportAny] # __dict__ is untyped
host: Final[object] = getattr(core, "_host", None)
# Every core keeps its host as _host; a stand-in router without one reads the process host.
return host if isinstance(host, PythonHost) else python_host
@property
def explicit_allowed_fails(self) -> int | None:
"""``allowed_fails`` as the caller passed it, ``None`` when it came from the global default."""

View file

@ -18,6 +18,7 @@ import litellm
if TYPE_CHECKING:
from litellm.caching.caching import Cache
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
from litellm.types.utils import BudgetConfig
@dataclass(frozen=True, slots=True)
@ -59,6 +60,9 @@ class RouterDynamicConfig:
callback_manager: "LoggingCallbackManager"
cache: "Cache | None"
request_timeout_explicitly_set: bool = False
tag_budget_config: "Mapping[str, BudgetConfig] | None" = None
disable_token_counter: bool = False
post_call_rules: Sequence[object] = ()
_versions: Final = itertools.count(1)
@ -112,6 +116,9 @@ def current_dynamic_config() -> RouterDynamicConfig:
callback_manager=litellm.logging_callback_manager,
cache=litellm.cache,
request_timeout_explicitly_set=litellm.request_timeout_explicitly_set,
tag_budget_config=litellm.tag_budget_config,
disable_token_counter=litellm.disable_token_counter,
post_call_rules=litellm.post_call_rules,
)
@ -141,6 +148,9 @@ _LIVE_SOURCES: Final = MappingProxyType(
"input_callback": "input_callback",
"callback_manager": "logging_callback_manager",
"cache": "cache",
"tag_budget_config": "tag_budget_config",
"disable_token_counter": "disable_token_counter",
"post_call_rules": "post_call_rules",
}
)

View file

@ -2185,7 +2185,7 @@ class PythonCore(DeprecatedRouterMembers):
self._core_provider_budget_config = provider_budget_config
self.router_budget_logger: RouterBudgetLimiting | None = None
if RouterBudgetLimiting.should_init_router_budget_limiter(
model_list=model_list, provider_budget_config=self._core_provider_budget_config
model_list=model_list, provider_budget_config=self._core_provider_budget_config, host=self._host
):
if optional_pre_call_checks is not None:
optional_pre_call_checks.append("router_budget_limiting")
@ -2473,6 +2473,7 @@ class PythonCore(DeprecatedRouterMembers):
selector = LowestCostLoggingHandler(
router_cache=self._core_cache,
routing_args={},
host=self._host,
)
case _:
pass
@ -3204,10 +3205,11 @@ class PythonCore(DeprecatedRouterMembers):
dual_cache=self._core_cache,
provider_budget_config=self._core_provider_budget_config,
model_list=self._core_model_list,
host=self._host,
)
self.router_budget_logger = _callback
elif pre_call_check == "enforce_model_rate_limits":
_callback = ModelRateLimitingCheck(dual_cache=self._core_cache)
_callback = ModelRateLimitingCheck(dual_cache=self._core_cache, host=self._host)
if _callback is None:
continue
@ -3898,6 +3900,7 @@ class PythonCore(DeprecatedRouterMembers):
complete_response_object,
checks_post_call_rules=initial_kwargs.get(MID_STREAM_CONTINUATION_KWARG)
is not MID_STREAM_CONTINUATION_MARKER,
host=self._host,
)
if continuation_messages is not None
else None
@ -4177,7 +4180,7 @@ class PythonCore(DeprecatedRouterMembers):
request_kwargs, requested_group if isinstance(requested_group, str) else None
)
targets: Final = continuation_fallback_targets(
request_kwargs.get("fallbacks", self._core_fallbacks), lookup_groups
request_kwargs.get("fallbacks", self._core_fallbacks), lookup_groups, self._host.dynamic().provider_list
)
if any(fallback_target_overrides_request_params(target) for target in targets):
return False
@ -5740,7 +5743,10 @@ class PythonCore(DeprecatedRouterMembers):
def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool:
return request_drops_params(
kwargs, self._core_default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs)
kwargs,
self._core_default_litellm_params,
self._request_deployment_params(model=model, kwargs=kwargs),
self._host,
)
async def _schedule_factory(
@ -8852,6 +8858,7 @@ class PythonCore(DeprecatedRouterMembers):
external_fallback_group, generic_idx = get_fallback_model_group_for_lookup_groups(
fallbacks=fallbacks,
lookup_groups=lookup_groups,
provider_list=self._host.dynamic().provider_list,
)
if external_fallback_group is None and generic_idx is not None:
external_fallback_group = fallbacks[generic_idx]["*"]
@ -8981,6 +8988,7 @@ class PythonCore(DeprecatedRouterMembers):
) = get_fallback_model_group_for_lookup_groups(
fallbacks=fallbacks, # if fallbacks = [{"gpt-3.5-turbo": ["claude-3-haiku"]}]
lookup_groups=lookup_groups,
provider_list=self._host.dynamic().provider_list,
)
## if none, check for generic fallback
if fallback_model_group is None and generic_fallback_idx is not None:
@ -9537,7 +9545,7 @@ class PythonCore(DeprecatedRouterMembers):
lookup_groups: tuple[str, ...],
) -> list[str] | None: # mutable-ok: mirrors the shared resolver's contract
fallback_model_group, _ = get_fallback_model_group_for_lookup_groups(
fallbacks=fallbacks, lookup_groups=lookup_groups
fallbacks=fallbacks, lookup_groups=lookup_groups, provider_list=self._host.dynamic().provider_list
)
return fallback_model_group
@ -10143,6 +10151,7 @@ class PythonCore(DeprecatedRouterMembers):
resolved, _ = get_fallback_model_group_for_lookup_groups(
fallbacks=fallbacks,
lookup_groups=fallback_lookup_groups(kwargs, model_group),
provider_list=self._host.dynamic().provider_list,
)
return has_unattempted_fallback_target(resolved, kwargs)
@ -10193,6 +10202,7 @@ class PythonCore(DeprecatedRouterMembers):
resolved, _ = get_fallback_model_group_for_lookup_groups(
fallbacks=fallbacks,
lookup_groups=fallback_lookup_groups(kwargs, model_group),
provider_list=self._host.dynamic().provider_list,
)
return has_unattempted_fallback_target(resolved, kwargs)

View file

@ -143,6 +143,7 @@ class OptionalChecks:
if RouterBudgetLimiting.should_init_router_budget_limiter( # pyright: ignore[reportUnknownMemberType] # untyped model_list
model_list=model_list if isinstance(model_list, list) else None, # pyright: ignore[reportUnknownArgumentType] # the argument as passed
provider_budget_config=self.provider_budget_config, # pyright: ignore[reportArgumentType] # the argument as passed
host=self._host,
):
if listed is not None:
listed.append("router_budget_limiting")
@ -255,10 +256,11 @@ class OptionalChecks:
dual_cache=self._cache,
provider_budget_config=self.provider_budget_config, # pyright: ignore[reportArgumentType] # the argument as passed
model_list=self._hosted._core_model_list, # pyright: ignore[reportPrivateUsage, reportArgumentType] # the live model_list
host=self._host,
)
self.router_budget_logger = callback
elif name == "enforce_model_rate_limits":
callback = ModelRateLimitingCheck(dual_cache=self._cache)
callback = ModelRateLimitingCheck(dual_cache=self._cache, host=self._host)
if callback is None:
continue
if self.optional_callbacks is not None and any(

View file

@ -721,7 +721,10 @@ class Resolved:
# An explicit `routing_strategy=None` reaches the Rust core as "None": no strategy picks.
return kept if self.routing_strategy is not None else {**kept, "routing_strategy": "None"}
def dynamic_document(self, version: int, model_names: Sequence[str]) -> Mapping[str, object]:
def dynamic_document(
self, version: int, model_names: Sequence[str], process_allowed_fails: object
) -> Mapping[str, object]:
"""``process_allowed_fails`` is the process default, read through the host."""
values: Final = {
"version": version,
"model_group_alias": _plain(dict(self.model_group_alias)),
@ -730,21 +733,19 @@ class Resolved:
"content_policy_fallbacks": _plain(self.content_policy_fallbacks),
"num_retries": _crate_int(self.num_retries),
"max_fallbacks": _crate_int(self.max_fallbacks),
"allowed_fails": _crate_int(self._router_allowed_fails()),
"allowed_fails": _crate_int(self._router_allowed_fails(process_allowed_fails)),
"fallback_keys": self._fallback_keys(model_names),
"default_deployment": _plain(served_default_deployment(self.default_deployment)[0]),
"default_deployment_raises": True if served_default_deployment(self.default_deployment)[1] else None,
}
return {name: value for name, value in values.items() if value is not None}
def _router_allowed_fails(self) -> object:
def _router_allowed_fails(self, process_allowed_fails: object) -> object:
"""``allowed_fails`` when it counts as set on the router (``_is_allowed_fails_set_on_router``):
passed explicitly, or different from the process default."""
import litellm
passed explicitly, or different from the process default ``process_allowed_fails``."""
if self.explicit_allowed_fails is not None:
return self.explicit_allowed_fails
if self.allowed_fails != litellm.allowed_fails:
if self.allowed_fails != process_allowed_fails:
return self.allowed_fails
return None
@ -1218,7 +1219,9 @@ class RustCore(
**{name: value for name, value in failover_document(self._plain).items() if value is not None},
**({"provider_default_deployment_ids": list(default_ids)} if default_ids else {}),
}
return RouterCoreHandle(static, self._resolved.dynamic_document(self._version, names))
return RouterCoreHandle(
static, self._resolved.dynamic_document(self._version, names, self._host.dynamic().allowed_fails)
)
def _provider_default_ids(self) -> tuple[str, ...]:
"""``provider_default_deployment_ids`` as the cooldown check reads it (``deployment in
@ -1609,7 +1612,9 @@ class RustCore(
self._rebuild()
else:
names: Final = [str(d.get("model_name")) for d in self._handle.deployments()]
self._handle.apply({"settings": resolved.dynamic_document(self._version, names)})
self._handle.apply(
{"settings": resolved.dynamic_document(self._version, names, self._host.dynamic().allowed_fails)}
)
def _changes_strategy(self, updated: Mapping[str, object]) -> bool:
"""``update_settings`` names a ``routing_strategy`` other than the router's."""
@ -3016,7 +3021,7 @@ class _Driver:
self._served: tuple[str | None, Mapping[str, object]] = (None, MappingProxyType({}))
self._attempts: int = 0
self._walk: Final = WalkMetadata(request, plan.metadata_field, self._params_base_model)
self.outcomes: Final = FallbackOutcomes(request)
self.outcomes: Final = FallbackOutcomes(request, core._host) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
# The fallback level the walk is at, from its fallback events.
self._level: int = 0
@ -3581,7 +3586,7 @@ class _Driver:
source: Final = own if own is not None else fallbacks
entries: Final = [dict(m) for entry in _as_list(source) if (m := as_str_mapping(entry))]
found, generic = (
get_fallback_model_group_for_lookup_groups(entries, lookup) # pyright: ignore[reportArgumentType] # the fallbacks config shape
get_fallback_model_group_for_lookup_groups(entries, lookup, self._core._host.dynamic().provider_list) # pyright: ignore[reportArgumentType, reportPrivateUsage] # the fallbacks config shape; the driver's own core's host
if entries
else (None, None)
)

View file

@ -1306,6 +1306,7 @@ class RustEndpoints:
kwargs,
self._core._resolved.default_litellm_params, # pyright: ignore[reportPrivateUsage] # see above
deployment_params(),
self._core._host, # pyright: ignore[reportPrivateUsage] # see above
)
def _split_priority(

View file

@ -58,7 +58,9 @@ def normalize_strategy(strategy: object) -> str | None:
return strategy if isinstance(strategy, str) else None
def build_selector(strategy: str | None, args: Mapping[str, object], cache: "DualCache") -> Selector | None:
def build_selector(
strategy: str | None, args: Mapping[str, object], cache: "DualCache", host: "PythonHost"
) -> Selector | None:
"""``_build_strategy_selector`` without registering: ``None`` for simple-shuffle and every
strategy without a selector."""
routing_args: Final = dict(args)
@ -72,7 +74,7 @@ def build_selector(strategy: str | None, args: Mapping[str, object], cache: "Dua
case "latency-based-routing":
return LowestLatencyLoggingHandler(router_cache=cache, routing_args=routing_args)
case "cost-based-routing":
return LowestCostLoggingHandler(router_cache=cache, routing_args={})
return LowestCostLoggingHandler(router_cache=cache, routing_args={}, host=host)
case _:
return None
@ -122,7 +124,7 @@ class StrategySelectors:
self._unregister([self._default, *self._overrides.values()])
self._overrides = {}
self._strategy = normalize_strategy(strategy)
self._default = build_selector(self._strategy, args, self._cache)
self._default = build_selector(self._strategy, args, self._cache, self._host)
if self._default is not None:
self._register(self._default)
@ -136,7 +138,7 @@ class StrategySelectors:
if self._default is None:
return
try:
rebuilt: Final = build_selector(self._strategy, args, self._cache)
rebuilt: Final = build_selector(self._strategy, args, self._cache, self._host)
except (TypeError, ValidationError):
verbose_router_logger.exception(
"Invalid routing_strategy_args %s for '%s'; keeping the previous ones", dict(args), self._strategy
@ -153,7 +155,8 @@ class StrategySelectors:
built: Final = {
name: selector
for name, strategy, args in groups
if (selector := build_selector(None if strategy == "priority" else strategy, args, self._cache)) is not None
if (selector := build_selector(None if strategy == "priority" else strategy, args, self._cache, self._host))
is not None
}
for selector in built.values():
self._register(selector)
@ -211,7 +214,7 @@ class StrategySelectors:
return self._default
with self._lock:
if strategy not in self._overrides:
self._overrides[strategy] = build_selector(strategy, {}, self._cache)
self._overrides[strategy] = build_selector(strategy, {}, self._cache, self._host)
selector: Final = self._overrides[strategy]
if selector is not None and request is not None and strategy not in self._registered_strategies():
from litellm.litellm_core_utils.litellm_logging import Logging

View file

@ -423,7 +423,8 @@ def _mid_stream_continuation_eligible(
lookup_groups: Final = fallback_lookup_groups(
request, requested_group if isinstance(requested_group, str) else None
)
targets: Final = continuation_fallback_targets(core.settings().fallbacks, lookup_groups) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # fallbacks as Python spells them
providers: Final = core._host.dynamic().provider_list # pyright: ignore[reportPrivateUsage] # the core's own host
targets: Final = continuation_fallback_targets(core.settings().fallbacks, lookup_groups, providers) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # fallbacks as Python spells them
if any(fallback_target_overrides_request_params(target) for target in targets):
return False
if any(_names_deployment_that_cannot_continue(core, target) for target in targets):
@ -561,6 +562,7 @@ async def _chat_continuation(
str(request.get("model") or ""),
complete,
checks_post_call_rules=request.get(MID_STREAM_CONTINUATION_KWARG) is not MID_STREAM_CONTINUATION_MARKER,
host=attempt.core._host, # pyright: ignore[reportPrivateUsage] # the attempt's own core's host
)
if continuation_messages is not None
else None

View file

@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Final
from litellm.litellm_core_utils.core_helpers import as_str_mapping
if TYPE_CHECKING:
from litellm.router_python_host import PythonHost
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
# `RETRY_BREADCRUMB_LIMIT`: how many failures `previous_models` keeps.
@ -225,9 +226,10 @@ class FallbackOutcomes:
custom loggers and, under ``include_fallback_errors``, lists the level's errors (its own
first, then each hop that failed) in the response's ``x-litellm-fallback-errors``."""
__slots__ = ("_failed", "_header_errors", "_include", "_pending")
__slots__ = ("_failed", "_header_errors", "_host", "_include", "_pending")
def __init__(self, request: Mapping[str, object]) -> None:
def __init__(self, request: Mapping[str, object], host: "PythonHost") -> None:
self._host: Final = host
self._include: Final = request.get("include_fallback_errors", False) is True
self._failed: Final[dict[object, list[object]]] = {} # mutable-ok: each level's failed hops, in order
self._header_errors: Final[list[list[object]]] = [] # mutable-ok: each successful hop's errors, innermost first
@ -253,7 +255,7 @@ class FallbackOutcomes:
self._header_errors.append([get_fallback_error_info(level_error), *failed])
self._pending.append(
handlers.log_success_fallback_event( # pyright: ignore[reportUnknownMemberType] # takes the kwargs as an untyped dict
original_model_group=original, kwargs=dict(kwargs), original_exception=level_error
original_model_group=original, kwargs=dict(kwargs), original_exception=level_error, host=self._host
)
)
return
@ -261,7 +263,7 @@ class FallbackOutcomes:
failed.append(get_fallback_error_info(hop_error))
self._pending.append(
handlers.log_failure_fallback_event( # pyright: ignore[reportUnknownMemberType] # see above
original_model_group=original, kwargs=dict(kwargs), original_exception=level_error
original_model_group=original, kwargs=dict(kwargs), original_exception=level_error, host=self._host
)
)

View file

@ -26,11 +26,10 @@ from collections.abc import Callable, Mapping, Sequence
from datetime import datetime, timedelta, timezone
from itertools import groupby
from types import MappingProxyType
from typing import Any, Final
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
import litellm
from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
@ -50,6 +49,9 @@ from litellm.types.router import DeploymentTypedDict, LiteLLM_Params, RouterErro
from litellm.types.utils import BudgetConfig, GenericBudgetConfigType, StandardLoggingPayload
from litellm.types.utils import BudgetConfig as GenericBudgetInfo
if TYPE_CHECKING:
from litellm.router_python_host import PythonHost
DEFAULT_REDIS_SYNC_INTERVAL: Final = 1
_CALLBACK_FIELDS: Final = TypeAdapter(dict[str, object])
_BATCH_CHARGE_VALUES: Final = TypeAdapter(tuple[object, float])
@ -121,7 +123,12 @@ class RouterBudgetLimiting(CustomLogger):
provider_budget_config: dict | None,
model_list: list[DeploymentTypedDict | dict[str, Any]] | None = None,
clock: Callable[[], float] = time.time,
host: "PythonHost | None" = None,
):
from litellm.router_python_host import python_host
# The router core's host: the process settings and litellm functions are read through it.
self._host: Final = python_host if host is None else host
self.dual_cache = dual_cache
self._clock = clock
self.redis_increment_operation_queue: list[RedisPipelineIncrementOperation] = []
@ -141,8 +148,8 @@ class RouterBudgetLimiting(CustomLogger):
self._start_background_work()
# Add self to litellm callbacks if it's a list
if isinstance(litellm.callbacks, list):
litellm.logging_callback_manager.add_litellm_callback(self)
if isinstance(self._host.dynamic().callbacks, list):
self._host.callbacks.add_litellm_callback(self)
def _start_background_work(self) -> None:
"""Start the periodic Redis sync and the provider budget cache seeding once a loop runs.
@ -810,7 +817,7 @@ class RouterBudgetLimiting(CustomLogger):
model = ""
provider_resolution_params = _LiteLLMParamsDictView({})
_, custom_llm_provider, _, _ = litellm.get_llm_provider(
_, custom_llm_provider, _, _ = self._host.invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # the host op table
model=str(model),
litellm_params=provider_resolution_params,
)
@ -826,7 +833,7 @@ class RouterBudgetLimiting(CustomLogger):
This is helpful for debugging and monitoring provider budget limits.
"""
prometheus_logger: Final = get_prometheus_logger_from_callbacks()
prometheus_logger: Final = get_prometheus_logger_from_callbacks(self._host)
if prometheus_logger:
prometheus_logger.track_provider_remaining_budget(
provider=provider,
@ -906,6 +913,7 @@ class RouterBudgetLimiting(CustomLogger):
def should_init_router_budget_limiter(
provider_budget_config: dict | None,
model_list: list[DeploymentTypedDict] | list[dict[str, Any]] | None = None,
host: "PythonHost | None" = None,
):
"""
Returns `True` if the router budget routing settings are set and RouterBudgetLimiting should be initialized
@ -918,7 +926,9 @@ class RouterBudgetLimiting(CustomLogger):
if provider_budget_config is not None:
return True
if litellm.tag_budget_config is not None:
from litellm.router_python_host import python_host
if (python_host if host is None else host).dynamic().tag_budget_config is not None:
return True
if model_list is None:
@ -995,7 +1005,8 @@ class RouterBudgetLimiting(CustomLogger):
self.deployment_budget_config = None
def _init_tag_budgets(self):
if litellm.tag_budget_config is None:
tag_budget_config: Final = self._host.dynamic().tag_budget_config
if tag_budget_config is None:
return
from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
@ -1005,7 +1016,7 @@ class RouterBudgetLimiting(CustomLogger):
if self.tag_budget_config is None:
self.tag_budget_config = {}
for _tag, _tag_budget_config in litellm.tag_budget_config.items():
for _tag, _tag_budget_config in tag_budget_config.items():
if isinstance(_tag_budget_config, dict):
_tag_budget_config = BudgetConfig(**_tag_budget_config)
_generic_budget_config = GenericBudgetInfo(

View file

@ -2410,7 +2410,12 @@ class ComplexityRouter(CustomLogger):
)
budgeted: Final = await offload_token_count(fit_jev_request)(
build_request, prompt, system_prompt, jev_input_targets(self.ctx, config), _truncate
build_request,
prompt,
system_prompt,
jev_input_targets(self.ctx, config),
_truncate,
self.ctx.host().dynamic().disable_token_counter,
)
request: Final = budgeted.request
if request is None:
@ -2457,7 +2462,10 @@ class ComplexityRouter(CustomLogger):
cost=response.classifier_cost
if config.deployment_name
else jev_classifier_cost(
response, config.model, "typesafe" if config.provider == "jev" else config.provider
response,
config.model,
"typesafe" if config.provider == "jev" else config.provider,
cost_map=self.ctx.host().dynamic().model_cost,
),
provider=accounting_provider,
)
@ -3597,15 +3605,15 @@ class ComplexityRouter(CustomLogger):
) -> int | None:
"""Real-tokenizer count of the resolved messages plus the out-of-band carriers, off the
event loop; None when counting fails, and the gate then leaves the placement alone."""
import litellm
from litellm.litellm_core_utils.token_counter import offload_token_count
token_counter: Final = self.ctx.host().invoke("token_counter") # pyright: ignore[reportUnknownMemberType] # the host op table
out_of_band: Final = self._out_of_band_request_text(request_kwargs)
try:
counted: Final = await offload_token_count(litellm.token_counter)(
counted: Final = await offload_token_count(token_counter)(
messages=cast(list, resolved_messages) # cast-ok: token_counter only iterates the sequence
)
return counted + (await offload_token_count(litellm.token_counter)(text=out_of_band) if out_of_band else 0)
return counted + (await offload_token_count(token_counter)(text=out_of_band) if out_of_band else 0)
except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request
verbose_router_logger.debug("ComplexityRouter: context-window token count failed. Got - %s", e)
return None

View file

@ -8,7 +8,6 @@ from uuid import uuid4
import httpx
from pydantic import ConfigDict, Field, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_router_logger
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.litellm_core_utils.classifier_logging import masked_originating_request
@ -115,12 +114,14 @@ def _jev_input_target(params: Mapping[str, object], explicit_limit: object) -> J
return JevInputTarget(qualified_model, None)
def _jev_request_count(request: JevSystemOneRequest, models: Sequence[str], budget: int) -> tuple[int, str]:
def _jev_request_count(
request: JevSystemOneRequest, models: Sequence[str], budget: int, disable_token_counter: bool
) -> tuple[int, str]:
from litellm._lazy_imports import get_token_counter_new
payload: Final = request.model_dump_json(exclude={"model"})
byte_count: Final = len(payload.encode("utf-8"))
if byte_count <= budget or litellm.disable_token_counter:
if byte_count <= budget or disable_token_counter:
return byte_count, "utf8_bytes"
try:
counts: Final = tuple(get_token_counter_new()(model=model, text=payload) for model in models)
@ -137,7 +138,9 @@ def fit_jev_request(
system_prompt: str | None,
targets: Sequence[JevInputTarget],
truncate: Callable[[str, int], str],
disable_token_counter: bool,
) -> JevBudgetedRequest:
"""``disable_token_counter`` is the process setting, read through the router core's host."""
known: Final = tuple(target for target in targets if target.max_tokens is not None)
if not known:
return JevBudgetedRequest(
@ -158,6 +161,7 @@ def fit_jev_request(
truncate,
len(known) != len(targets),
bounded_prompt != prompt or bounded_system != system_prompt,
disable_token_counter,
)
@ -170,6 +174,7 @@ def _fit_jev_components(
truncate: Callable[[str, int], str],
unknown_limits: bool,
truncated: bool,
disable_token_counter: bool,
) -> JevBudgetedRequest:
attempts: Final = len(prompt).bit_length() + len(system_prompt or "").bit_length() + 1
components: tuple[str, str | None, bool, int] = ( # rebind-ok: bounded iterative fitting state
@ -181,7 +186,7 @@ def _fit_jev_components(
while True:
current_prompt, current_system, was_truncated, remaining = components
request = build_request(current_prompt, current_system)
count, method = _jev_request_count(request, models, budget)
count, method = _jev_request_count(request, models, budget, disable_token_counter)
telemetry: StandardLoggingRoutingDecision = {
"classifier_input_budget_tokens": budget,
"classifier_input_tokens": count,
@ -452,19 +457,22 @@ def build_jev_request(
def jev_classifier_cost(
response: JevSystemOneResponse, configured_model: str, provider: ClassifierProvider = "typesafe"
response: JevSystemOneResponse,
configured_model: str,
provider: ClassifierProvider = "typesafe",
*,
cost_map: Mapping[str, object],
) -> float | None:
"""The call's cost from ``cost_map`` (the process cost map, read through the router core's host)."""
usage: Final = response.usage
if usage is None:
return None
model: Final = response.model or configured_model
model_key: Final = f"{provider}/{model}"
if model_key not in litellm.model_cost: # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed
if model_key not in cost_map:
return None
try:
pricing: Final = _REGISTRY_PRICING_ADAPTER.validate_python(
litellm.model_cost[model_key] # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed
)
pricing: Final = _REGISTRY_PRICING_ADAPTER.validate_python(cost_map[model_key])
except ValidationError:
return None
return usage.input_tokens * pricing.input_cost_per_token + usage.output_tokens * pricing.output_cost_per_token

View file

@ -2,10 +2,9 @@
# picks based on response time (for streaming, this is time to first token)
from collections.abc import Mapping
from datetime import datetime
from typing import Final
from typing import TYPE_CHECKING, Final
import litellm
from litellm import ModelResponse, token_counter, verbose_logger
from litellm import ModelResponse, Usage, token_counter, verbose_logger
from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
@ -13,22 +12,21 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed
if TYPE_CHECKING:
from litellm.router_python_host import PythonHost
def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]:
"""Input and output per-token prices from the first cost-map row among the deployment's
def _deployment_cost_map_rates(deployment: object, cost_map: Mapping[str, object]) -> tuple[float | None, float | None]:
"""Input and output per-token prices from the first ``cost_map`` row among the deployment's
pricing models that carries one; (None, None) when no row does, so the caller falls back to
its unmapped default."""
rows: Final = (str_keyed(_cost_map().get(model)) for model in deployment_pricing_models(deployment))
rows: Final = (str_keyed(cost_map.get(model)) for model in deployment_pricing_models(deployment))
row: Final = next((row for row in rows if _declares_token_price(row)), None)
if row is None:
return None, None
return _as_rate(row.get("input_cost_per_token")), _as_rate(row.get("output_cost_per_token"))
def _cost_map() -> Mapping[str, object]:
return litellm.model_cost
def _declares_token_price(row: Mapping[str, object]) -> bool:
return row.get("input_cost_per_token") is not None or row.get("output_cost_per_token") is not None
@ -42,8 +40,12 @@ class LowestCostLoggingHandler(CustomLogger):
logged_success: int = 0
logged_failure: int = 0
def __init__(self, router_cache: DualCache, routing_args: dict = {}):
def __init__(self, router_cache: DualCache, routing_args: dict = {}, host: "PythonHost | None" = None):
from litellm.router_python_host import python_host
self.router_cache = router_cache
# The router core's host: the cost map is read through it.
self._host: Final = python_host if host is None else host
@with_service_target("router_usage")
def log_success_event(self, kwargs, response_obj, start_time, end_time):
@ -86,7 +88,7 @@ class LowestCostLoggingHandler(CustomLogger):
if isinstance(response_obj, ModelResponse):
_usage: Final = getattr(response_obj, "usage", None)
if _usage is not None and isinstance(_usage, litellm.Usage):
if _usage is not None and isinstance(_usage, Usage):
total_tokens = _usage.total_tokens
# ------------
@ -163,7 +165,7 @@ class LowestCostLoggingHandler(CustomLogger):
if isinstance(response_obj, ModelResponse):
_usage: Final = getattr(response_obj, "usage", None)
if _usage is not None and isinstance(_usage, litellm.Usage):
if _usage is not None and isinstance(_usage, Usage):
total_tokens = _usage.total_tokens
# ------------
@ -286,7 +288,9 @@ class LowestCostLoggingHandler(CustomLogger):
or float("inf")
)
deployment_view: object = _deployment
map_input_cost, map_output_cost = _deployment_cost_map_rates(deployment_view)
map_input_cost, map_output_cost = _deployment_cost_map_rates(
deployment_view, self._host.dynamic().model_cost
)
# check if user provided input_cost_per_token and output_cost_per_token in litellm_params
item_input_cost = None

View file

@ -23,6 +23,7 @@ from litellm.types.utils import PromptTokensDetailsWrapper, Usage
if TYPE_CHECKING:
from litellm.router_api import RoutingContext
from litellm.router_python_host import PythonHost
_REFERENCE_REQUEST: Final = Usage(
@ -46,7 +47,7 @@ class Baseline(NamedTuple):
deployment_id: str | None = None
def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None:
def canonical_model(model: str, custom_llm_provider: str | None = None, host: "PythonHost | None" = None) -> str | None:
"""``provider/model``, or ``None`` when the pair names no known provider.
A deployment may name its vendor in the model prefix or in a separate
@ -56,15 +57,19 @@ def canonical_model(model: str, custom_llm_provider: str | None = None) -> str |
A github_copilot or chatgpt candidate is qualified by string alone: resolving either
provider runs its OAuth device flow, and for a declared pair the resolver's answer is
the declaration itself, so asking it buys nothing but the block.
The provider is resolved through ``host`` (the router core's host; the process host when none is given).
"""
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.router_python_host import python_host
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
if declared is not None:
return f"{declared}/{model.removeprefix(f'{declared}/')}"
try:
resolved, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
resolved, provider, _, _ = (python_host if host is None else host).invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # the host op table
model=model, custom_llm_provider=custom_llm_provider
)
except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline
verbose_router_logger.debug("savings baseline: cannot resolve candidate %s (%s)", model, e)
return None
@ -83,7 +88,7 @@ def _models_in(ctx: "RoutingContext", group_name: str) -> tuple[Baseline, ...]:
"""
deployments: Final = ctx.registry().deployments(group_name, source="named")
if not deployments:
return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else ()
return (Baseline(qualified),) if (qualified := canonical_model(group_name, host=ctx.host())) else ()
def candidate(deployment: Mapping[str, object]) -> Baseline | None:
params: Final = deployment.get("litellm_params")
@ -92,7 +97,7 @@ def _models_in(ctx: "RoutingContext", group_name: str) -> tuple[Baseline, ...]:
info: Final = deployment.get("model_info")
base: Final = info.get("base_model") if isinstance(info, dict) else None
model: Final = base or params.get("base_model") or params.get("model")
qualified: Final = canonical_model(model, params.get("custom_llm_provider")) if model else None
qualified: Final = canonical_model(model, params.get("custom_llm_provider"), ctx.host()) if model else None
if qualified is None:
return None
deployment_id: Final = info.get("id") if isinstance(info, dict) else None

View file

@ -5,13 +5,13 @@ Callbacks triggered on cooling down deployments
import copy
from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm._logging import verbose_logger
from litellm.router_deprecated import renamed_router_param
if TYPE_CHECKING:
from litellm.integrations.prometheus import PrometheusLogger
from litellm.router_api import RoutingContext
from litellm.router_python_host import PythonHost
else:
PrometheusLogger = Any
@ -54,14 +54,17 @@ def emit_router_cooldown_event(
temp_litellm_params = copy.deepcopy(_litellm_params)
temp_litellm_params = dict(temp_litellm_params)
_model_name: Final = _deployment.get("model_name", None) or ""
_api_base: Final = litellm.get_api_base(model=_model_name, optional_params=temp_litellm_params) or ""
host: Final = ctx.host()
_api_base: Final = str(
host.invoke("get_api_base")(model=_model_name, optional_params=temp_litellm_params) or "" # pyright: ignore[reportUnknownMemberType] # the host op table
)
model_info: Final = _deployment["model_info"]
model_id: Final = model_info.id
litellm_model_name: Final = temp_litellm_params.get("model") or ""
llm_provider = ""
try:
_, llm_provider, _, _ = litellm.get_llm_provider(
_, llm_provider, _, _ = host.invoke("get_llm_provider")( # pyright: ignore[reportUnknownMemberType] # the host op table
model=litellm_model_name,
custom_llm_provider=temp_litellm_params.get("custom_llm_provider"),
)
@ -69,7 +72,7 @@ def emit_router_cooldown_event(
pass
# get the prometheus logger from in memory loggers
prometheusLogger: Final[PrometheusLogger | None] = get_prometheus_logger_from_callbacks()
prometheusLogger: Final[PrometheusLogger | None] = get_prometheus_logger_from_callbacks(host)
if prometheusLogger is not None:
prometheusLogger.set_deployment_complete_outage(
@ -88,19 +91,22 @@ def emit_router_cooldown_event(
)
def get_prometheus_logger_from_callbacks() -> PrometheusLogger | None:
def get_prometheus_logger_from_callbacks(host: "PythonHost | None" = None) -> PrometheusLogger | None:
"""
Checks if prometheus is a initalized callback, if yes returns it
Checks if prometheus is a initalized callback, if yes returns it. The callback lists are read
through ``host`` (the router core's host; the process host when none is given).
"""
from litellm.integrations.prometheus import PrometheusLogger
from litellm.router_python_host import python_host
if PrometheusLogger is None:
return None
for _callback in litellm._async_success_callback:
source: Final = python_host if host is None else host
for _callback in source.callback_list("_async_success_callback"):
if isinstance(_callback, PrometheusLogger):
return _callback
for global_callback in litellm.callbacks:
for global_callback in source.dynamic().callbacks:
if isinstance(global_callback, PrometheusLogger):
return global_callback

View file

@ -453,7 +453,9 @@ def _has_fallback_group(ctx: "RoutingContext", requested_model_group: str | None
chains: Final = tuple(
chain for chain in map(_fallback_chain_names, _fallback_entries(configured)) if chain is not None
)
fallback_group, _ = get_fallback_model_group(fallbacks=chains, model_group=requested_model_group)
fallback_group, _ = get_fallback_model_group(
fallbacks=chains, model_group=requested_model_group, provider_list=ctx.host().dynamic().provider_list
)
return bool(fallback_group)
@ -544,7 +546,7 @@ def _should_cooldown_deployment(
# Only apply error rate cooldown when we have enough requests to make the percentage meaningful
return True
elif litellm.should_retry(status_code=exception_status_int) is False:
elif ctx.host().invoke("should_retry")(status_code=exception_status_int) is False: # pyright: ignore[reportUnknownMemberType] # the host op table
return True
return False
@ -819,7 +821,7 @@ def _is_allowed_fails_set_on_router(
return False
if ctx.explicit_allowed_fails is not None:
return True
if ctx.settings().allowed_fails != litellm.allowed_fails:
if ctx.settings().allowed_fails != ctx.host().dynamic().allowed_fails:
return True
return False

View file

@ -45,6 +45,7 @@ from litellm.types.router import (
if TYPE_CHECKING:
from litellm.router_api import RoutingContext
from litellm.router_python_host import PythonHost
# Status codes a generic API call's caller-supplied resource id can trigger on its own
# (e.g. a nonexistent file/batch/thread id), independent of the selected deployment's health.
@ -266,7 +267,7 @@ def is_bad_request_without_fallback(exception: Exception, fallback_on_unsupporte
return exception.status_code == 400 and bool(getattr(exception, "llm_provider", ""))
def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool:
def _check_stripped_model_group(model_group: str, fallback_key: str, provider_list: Sequence[object]) -> bool:
"""
Handles wildcard routing scenario
@ -279,11 +280,8 @@ def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool:
Returns:
- True if the stripped model group == fallback_key
"""
for provider in litellm.provider_list:
if isinstance(provider, Enum):
_provider = provider.value
else:
_provider = provider
for provider in provider_list:
_provider: str = str(provider.value) if isinstance(provider, Enum) else str(provider) # pyright: ignore[reportAny] # an Enum member's value
if model_group.startswith(f"{_provider}/"):
stripped_model_group = model_group.replace(f"{_provider}/", "")
if stripped_model_group == fallback_key:
@ -588,12 +586,16 @@ def _resolved_a_specific_chain(
def get_fallback_model_group_for_lookup_groups(
fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: mirrors the fallbacks config shape
lookup_groups: tuple[str, ...],
provider_list: Sequence[object] | None = None,
) -> tuple[list[str] | None, int | None]: # mutable-ok: mirrors get_fallback_model_group's contract
"""
First lookup group with a specifically-keyed chain wins; the generic "*" chain applies
only after every group missed, so a catch-all cannot shadow a later group's own chain.
"""
results: Final = tuple(get_fallback_model_group(fallbacks=fallbacks, model_group=group) for group in lookup_groups)
results: Final = tuple(
get_fallback_model_group(fallbacks=fallbacks, model_group=group, provider_list=provider_list)
for group in lookup_groups
)
specific: Final = next((result for result in results if _resolved_a_specific_chain(fallbacks, result)), None)
if specific is not None:
return specific
@ -603,8 +605,12 @@ def get_fallback_model_group_for_lookup_groups(
def get_fallback_model_group(
fallbacks: Sequence[Mapping[str, list[str]]], # mutable-ok: mirrors the fallbacks config shape
model_group: str,
provider_list: Sequence[object] | None = None,
) -> tuple[list[str] | None, int | None]: # mutable-ok: mirrors get_fallback_model_group's contract
"""
``provider_list`` is the process provider list, read through the router core's host (the
process host's when none is given).
Returns:
- fallback_model_group: List[str] of fallback model groups. example: ["gpt-4", "gpt-3.5-turbo"]
- generic_fallback_idx: int of the index of the generic fallback in the fallbacks list.
@ -615,6 +621,9 @@ def get_fallback_model_group(
- provider-prefixed model group match
- generic fallback
"""
from litellm.router_python_host import python_host
providers: Final = python_host.dynamic().provider_list if provider_list is None else provider_list
generic_fallback_idx: int | None = None
stripped_model_fallback: list[str] | None = None
fallback_model_group: list[str] | None = None
@ -628,7 +637,7 @@ def get_fallback_model_group(
fallback_model_group = item[model_group]
break
elif fallback_key == prefixed_model_group or _check_stripped_model_group(
model_group=model_group, fallback_key=fallback_key
model_group=model_group, fallback_key=fallback_key, provider_list=providers
): # check generic fallback
stripped_model_fallback = item[fallback_key]
elif fallback_key == "*": # check generic fallback
@ -645,20 +654,26 @@ def get_fallback_model_group(
return fallback_model_group, generic_fallback_idx
def _resolved_fallback_chain(entries: Sequence[object], lookup_groups: tuple[str, ...]) -> Sequence[object]:
def _resolved_fallback_chain(
entries: Sequence[object], lookup_groups: tuple[str, ...], provider_list: Sequence[object] | None
) -> Sequence[object]:
group_entries: Final[Sequence[Mapping[str, list[str]]]] = tuple( # mutable-ok: mirrors the fallbacks config shape
entry for entry in entries if isinstance(entry, Mapping) and entry
)
chain, _ = get_fallback_model_group_for_lookup_groups(fallbacks=group_entries, lookup_groups=lookup_groups)
chain, _ = get_fallback_model_group_for_lookup_groups(
fallbacks=group_entries, lookup_groups=lookup_groups, provider_list=provider_list
)
return chain if isinstance(chain, list) else ()
def continuation_fallback_targets(fallbacks: object, lookup_groups: tuple[str, ...]) -> Sequence[object]:
def continuation_fallback_targets(
fallbacks: object, lookup_groups: tuple[str, ...], provider_list: Sequence[object] | None = None
) -> Sequence[object]:
entries: Final[Sequence[object]] = fallbacks if isinstance(fallbacks, list) else ()
return (
entries
if check_non_standard_fallback_format(fallbacks=entries)
else _resolved_fallback_chain(entries, lookup_groups)
else _resolved_fallback_chain(entries, lookup_groups, provider_list)
)
@ -901,6 +916,7 @@ async def run_async_fallback(
original_model_group=original_model_group,
kwargs=kwargs,
original_exception=original_exception,
host=litellm_routing_ctx.host(),
)
return response
except Exception as e:
@ -910,6 +926,7 @@ async def run_async_fallback(
original_model_group=original_model_group,
kwargs=kwargs,
original_exception=original_exception,
host=litellm_routing_ctx.host(),
)
if (
failure_callbacks_blocked_logging_obj := _failure_callbacks_blocked_logging_object(
@ -925,7 +942,9 @@ async def run_async_fallback(
raise error_from_fallbacks
async def log_success_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception):
async def log_success_fallback_event(
original_model_group: str, kwargs: dict, original_exception: Exception, host: "PythonHost | None" = None
):
"""
Log a successful fallback event to all registered callbacks.
@ -940,7 +959,9 @@ async def log_success_fallback_event(original_model_group: str, kwargs: dict, or
Errors during logging are caught and reported but do not interrupt the process.
"""
# Get deduplicated CustomLogger instances from all callback lists
custom_loggers: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger)
from litellm.router_python_host import python_host
custom_loggers: Final = (python_host if host is None else host).callbacks.get_custom_loggers_for_type(CustomLogger)
for _callback_custom_logger in custom_loggers:
try:
@ -953,7 +974,9 @@ async def log_success_fallback_event(original_model_group: str, kwargs: dict, or
verbose_router_logger.error("Error in log_success_fallback_event: %s", e)
async def log_failure_fallback_event(original_model_group: str, kwargs: dict, original_exception: Exception):
async def log_failure_fallback_event(
original_model_group: str, kwargs: dict, original_exception: Exception, host: "PythonHost | None" = None
):
"""
Log a failed fallback event to all registered callbacks.
@ -968,7 +991,9 @@ async def log_failure_fallback_event(original_model_group: str, kwargs: dict, or
Errors during logging are caught and reported but do not interrupt the process.
"""
# Get deduplicated CustomLogger instances from all callback lists
custom_loggers: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(CustomLogger)
from litellm.router_python_host import python_host
custom_loggers: Final = (python_host if host is None else host).callbacks.get_custom_loggers_for_type(CustomLogger)
for _callback_custom_logger in custom_loggers:
try:

View file

@ -7,11 +7,10 @@ from dataclasses import dataclass
from itertools import accumulate
from os.path import commonprefix
from types import MappingProxyType
from typing import Final, TypeVar
from typing import TYPE_CHECKING, Final, TypeVar
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.litellm_core_utils.routing_common_utils import MID_STREAM_CONTINUATION_KWARG
from litellm.litellm_core_utils.rules import Rules
@ -19,6 +18,9 @@ from litellm.types.llms.openai import AllMessageValues, OutputTextDeltaEvent
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
from litellm.utils import supports_assistant_prefill
if TYPE_CHECKING:
from litellm.router_python_host import PythonHost
class _ContinuationMarker:
"""Only the router constructs one, so a JSON request body cannot forge the continuation flag."""
@ -219,7 +221,7 @@ class ContinuationOutput:
"""Splices a fallback's continuation onto the stream the client already received. Drops a leading echo of the
prefill, since some providers (Mistral) return the prefix inside the completion, and the prefill's trailing
whitespace sent again by a provider that strips it (Anthropic), keeps the primary's chunk id, created time and single role delta, and
runs litellm.post_call_rules over the joined text, since each stream wrapper only sees its own fragment. A
runs the process post_call_rules (read through the router core's host) over the joined text, since each stream wrapper only sees its own fragment. A
continuation nested in another leaves the rules to the outermost one, the only one holding the whole text."""
def __init__(
@ -229,7 +231,11 @@ class ContinuationOutput:
model: str,
primary_response: object,
checks_post_call_rules: bool = True,
host: "PythonHost | None" = None,
) -> None:
from litellm.router_python_host import python_host
self._host: Final = python_host if host is None else host
self._text = emitted_text
self._checks_post_call_rules: Final = checks_post_call_rules
self._prefill_text: Final = _prefill_text(prefill_message)
@ -283,7 +289,7 @@ class ContinuationOutput:
return _without_leading_text(chunk, matched)
def _observed(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]:
if not self._checks_post_call_rules or not litellm.post_call_rules:
if not self._checks_post_call_rules or not self._host.dynamic().post_call_rules:
return chunks
for chunk in chunks:
self._text += _delta_text(chunk)

View file

@ -44,6 +44,8 @@ from litellm.utils import get_utc_datetime
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.router_python_host import PythonHost
Span = _Span
else:
Span = Any
@ -179,11 +181,11 @@ def _estimate_input_tokens(request_kwargs: dict[str, Any] | None, model: str = "
return 0
def _model_max_output_tokens(model_name: str) -> int | None:
# litellm.get_model_info raises a bare Exception for an unrecognized model;
# this lookup is a fallback default and must never fail the request.
def _model_max_output_tokens(model_name: str, host: PythonHost) -> int | None:
# get_model_info (through the router core's host) raises a bare Exception for an
# unrecognized model; this lookup is a fallback default and must never fail the request.
with contextlib.suppress(Exception):
info: Final = litellm.get_model_info(model=model_name)
info: Final = host.invoke("get_model_info")(model=model_name) # pyright: ignore[reportUnknownMemberType] # the host op table
model_max: Final = info.get("max_output_tokens")
if model_max is not None:
return max(0, int(model_max))
@ -227,8 +229,8 @@ def _deployment_output_cap(request_kwargs: Mapping[str, object] | None, deployme
)
def _default_otpm_reservation(model_name: str, otpm_limit: int) -> int:
model_max: Final = _model_max_output_tokens(model_name) if model_name else None
def _default_otpm_reservation(model_name: str, otpm_limit: int, host: PythonHost) -> int:
model_max: Final = _model_max_output_tokens(model_name, host) if model_name else None
share_of_limit: Final = max(1, int(otpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION))
return min(4096 if model_max is None else model_max, share_of_limit)
@ -238,11 +240,12 @@ def _otpm_reservation(
deployment: Mapping[str, object],
model_name: str,
otpm_limit: int,
host: PythonHost,
) -> int:
from_request: Final = _explicit_max_tokens(request_kwargs)
if from_request is not None:
return from_request
default: Final = _default_otpm_reservation(model_name, otpm_limit)
default: Final = _default_otpm_reservation(model_name, otpm_limit, host)
from_deployment: Final = _deployment_output_cap(request_kwargs, deployment)
return default if from_deployment is None else min(from_deployment, default)
@ -515,7 +518,11 @@ async def _increment_with_rollback(
def io_token_pre_call_check(
dual_cache: DualCache,
deployment: dict,
host: PythonHost | None = None,
) -> dict | None:
"""``host`` is the router core's host (the process host when none is given)."""
from litellm.router_python_host import python_host
itpm_limit, otpm_limit = get_deployment_io_token_limits(deployment)
if itpm_limit is None and otpm_limit is None:
return deployment
@ -525,7 +532,9 @@ def io_token_pre_call_check(
_model, served_model = _tokenizer_and_served_models(deployment)
estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model)
otpm_to_reserve: Final = (
_otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0
_otpm_reservation(request_kwargs, deployment, served_model, otpm_limit, python_host if host is None else host)
if otpm_limit is not None
else 0
)
dt: Final = get_utc_datetime()
@ -584,7 +593,11 @@ async def async_io_token_pre_call_check(
dual_cache: DualCache,
deployment: dict,
parent_otel_span: Span | None = None,
host: PythonHost | None = None,
) -> dict | None:
"""``host`` is the router core's host (the process host when none is given)."""
from litellm.router_python_host import python_host
itpm_limit, otpm_limit = get_deployment_io_token_limits(deployment)
if itpm_limit is None and otpm_limit is None:
return deployment
@ -600,7 +613,9 @@ async def async_io_token_pre_call_check(
_model, served_model = _tokenizer_and_served_models(deployment)
estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model)
otpm_to_reserve: Final = (
_otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0
_otpm_reservation(request_kwargs, deployment, served_model, otpm_limit, python_host if host is None else host)
if otpm_limit is not None
else 0
)
dt: Final = get_utc_datetime()

View file

@ -44,6 +44,8 @@ from litellm.utils import get_utc_datetime
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.router_python_host import PythonHost
Span = _Span
else:
Span = Any
@ -134,7 +136,14 @@ class ModelRateLimitingCheck(CustomLogger):
this check actively enforces those limits across ALL routing strategies.
"""
def __init__(self, dual_cache: DualCache, clock: Callable[[], datetime] = get_utc_datetime):
def __init__(
self,
dual_cache: DualCache,
clock: Callable[[], datetime] = get_utc_datetime,
host: "PythonHost | None" = None,
):
# The router core's host: the io-token checks read model info through it.
self._host: PythonHost | None = host
self.dual_cache = dual_cache
self.clock = clock
# model_ids already warned about itpm/otpm + tpm/rpm on the same deployment,
@ -312,6 +321,7 @@ class ModelRateLimitingCheck(CustomLogger):
io_token_pre_call_check(
self.dual_cache,
deployment,
host=self._host,
)
io_reservation_made = True
@ -397,6 +407,7 @@ class ModelRateLimitingCheck(CustomLogger):
self.dual_cache,
deployment,
parent_otel_span=parent_otel_span,
host=self._host,
)
io_reservation_made = True

View file

@ -1,12 +1,15 @@
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass
from typing import Final
from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
if TYPE_CHECKING:
from litellm.router_python_host import PythonHost
@dataclass(frozen=True, slots=True)
class InvalidPriority:
@ -84,9 +87,15 @@ def request_drops_params(
kwargs: Mapping[str, object],
router_defaults: Mapping[str, object],
deployment_params: Iterable[Mapping[str, object]],
host: "PythonHost | None" = None,
) -> bool:
"""``host`` is the router core's host, whose process ``drop_params`` counts (the process host when none is given)."""
from litellm.router_python_host import python_host
deployments: Final = tuple(deployment_params)
if litellm.drop_params is True or any(_drops_priority_by_name(s) for s in (kwargs, router_defaults, *deployments)):
if (python_host if host is None else host).dynamic().drop_params is True or any(
_drops_priority_by_name(s) for s in (kwargs, router_defaults, *deployments)
):
return True
request_flag: Final = _explicit_drop_params(kwargs)
if request_flag is not None:

View file

@ -874,7 +874,7 @@ def test_jev_classifier_cost_uses_registry_pricing(monkeypatch: pytest.MonkeyPat
answers={"tier": _answer()},
usage=JevUsage(input_tokens=3, output_tokens=4),
)
assert jev_classifier_cost(response, "jev-latest") == pytest.approx(0.0011)
assert jev_classifier_cost(response, "jev-latest", cost_map=litellm.model_cost) == pytest.approx(0.0011)
def test_jev_classifier_cost_is_none_without_registry_pricing() -> None:
@ -883,7 +883,7 @@ def test_jev_classifier_cost_is_none_without_registry_pricing() -> None:
answers={"tier": _answer()},
usage=JevUsage(input_tokens=3, output_tokens=4),
)
assert jev_classifier_cost(response, "jev-unpriced") is None
assert jev_classifier_cost(response, "jev-unpriced", cost_map=litellm.model_cost) is None
@pytest.mark.asyncio

View file

@ -75,7 +75,7 @@ async def test_get_llm_provider_for_deployment_dict_view_supports_mapping_and_at
return model, "openai", None, None
monkeypatch.setattr(
"litellm.router_strategy.budget_limiter.litellm.get_llm_provider",
"litellm.get_llm_provider",
_future_style_get_llm_provider,
)

View file

@ -425,10 +425,10 @@ class FallbackEvents(CustomLogger):
@pytest.mark.asyncio
async def test_fallback_events_reach_the_process_custom_loggers_innermost_hop_first(
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
make_core: MakeCore, host: ScriptedHost
) -> None:
events = FallbackEvents()
monkeypatch.setattr(litellm, "callbacks", [events])
host.callbacks.add_litellm_callback(events)
core = build(make_core, fallbacks=[{"a": ["b", "c"]}])
host.answer("openai/a", rate_limit())
host.answer("openai/b", server_error())
@ -443,10 +443,10 @@ async def test_fallback_events_reach_the_process_custom_loggers_innermost_hop_fi
@pytest.mark.asyncio
async def test_a_fallback_that_fails_everywhere_logs_failure_events(
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
make_core: MakeCore, host: ScriptedHost
) -> None:
events = FallbackEvents()
monkeypatch.setattr(litellm, "callbacks", [events])
host.callbacks.add_litellm_callback(events)
core = build(make_core, fallbacks=[{"a": ["b"]}])
host.answer("*", rate_limit())

View file

@ -267,10 +267,10 @@ def _stale(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok
@pytest.mark.asyncio
async def test_a_stale_routing_group_at_a_fallback_level_is_that_levels_error(
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
make_core: MakeCore, host: ScriptedHost
) -> None:
events: Final = FallbackEvents()
monkeypatch.setattr(litellm, "callbacks", [events])
host.callbacks.add_litellm_callback(events)
core: Final = _stale(make_core)
host.answer("*", rate_limit())

View file

@ -531,6 +531,9 @@ async def test_a_deployment_cools_down_once_its_failures_pass_allowed_fails(
@pytest.mark.asyncio
async def test_process_allowed_fails_is_the_default(make_core: MakeCore, host: ScriptedHost) -> None:
"""The host's process ``allowed_fails`` is the router's default, and a router left on the default
does not count it as set on the router: one failure cools nothing down, as in a process whose
``litellm.allowed_fails`` is 0."""
host.set_dynamic(allowed_fails=0)
core: Final = make_core(model_list=two_deployments(), num_retries=0)
host.answer("*", fresh(server_error))
@ -538,6 +541,20 @@ async def test_process_allowed_fails_is_the_default(make_core: MakeCore, host: S
await attempt(core)
assert core.settings().allowed_fails == 0
assert list(cooled_down(core, ["a", "b"])) == []
@pytest.mark.asyncio
async def test_an_allowed_fails_set_on_the_router_cools_down_after_that_many_failures(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""An ``allowed_fails`` passed to the router counts as set on it, even when it equals the host's default."""
host.set_dynamic(allowed_fails=0)
core: Final = make_core(model_list=two_deployments(), num_retries=0, allowed_fails=0)
host.answer("*", fresh(server_error))
await attempt(core)
assert list(cooled_down(core, ["a", "b"])) == host.deployments_called("acompletion")

View file

@ -0,0 +1,54 @@
"""
Both router cores and the helpers they share reach the process only through their host: no router
module reads a ``litellm.<setting>`` or calls a ``litellm.<function>`` directly. Exception classes,
types and modules (``litellm.RateLimitError``, ``litellm.Usage``, ``litellm.utils``) are code, not
process state, and stay importable.
"""
import ast
import inspect
from pathlib import Path
from typing import Final
import litellm
_PACKAGE: Final = Path(litellm.__file__).parent
# router_config.py is the host's own view of the process settings; router_python_host.py is the host.
_HOST_MODULES: Final = frozenset({"router_config.py", "router_python_host.py"})
def _router_modules() -> tuple[Path, ...]:
return tuple(
sorted(
path
for path in (
*_PACKAGE.glob("router_*.py"),
*(_PACKAGE / "router_utils").rglob("*.py"),
*(_PACKAGE / "router_strategy").rglob("*.py"),
)
if path.name not in _HOST_MODULES
)
)
def _is_code(name: str) -> bool:
value: Final[object] = getattr(litellm, name, None)
return inspect.isclass(value) or inspect.ismodule(value)
def _process_reads(path: Path) -> tuple[str, ...]:
tree: Final = ast.parse(path.read_text(encoding="utf-8"))
return tuple(
f"{path.relative_to(_PACKAGE.parent)}:{node.lineno} litellm.{node.attr}"
for node in ast.walk(tree)
if isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "litellm"
and not _is_code(node.attr)
)
def test_router_code_reaches_litellm_settings_and_functions_only_through_the_host() -> None:
modules: Final = _router_modules()
assert len(modules) > 50
assert [read for path in modules for read in _process_reads(path)] == []