This commit is contained in:
Nate Armstrong 2026-11-02 14:53:25 +00:00 • committed by GitHub
commit d8d61e2fef
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
30 changed files with 807 additions and 413 deletions

View file

@ -81,7 +81,11 @@ def can_block_current_thread() -> bool:
return threading.current_thread() is threading.main_thread() and not is_event_loop_running()
def run_async_function(async_function, *args, **kwargs):
def run_async_function(
async_function: Callable[..., Awaitable[T_Retval]],
*args: object,
**kwargs: object, # kwargs-ok: forwarded to async_function as given
) -> T_Retval:
"""
Helper utility to run an async function in a sync context.
Handles the case where there is an existing event loop running.

View file

@ -26,8 +26,14 @@ def get_configured_request_timeout() -> float | None:
"""Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
import litellm
timeout: Final = float(litellm.request_timeout)
if litellm.request_timeout_explicitly_set:
return configured_request_timeout(litellm.request_timeout, litellm.request_timeout_explicitly_set)
def configured_request_timeout(request_timeout: float, explicitly_set: bool) -> float | None:
"""``get_configured_request_timeout`` for a process ``request_timeout`` read elsewhere (a router
host's process settings): the value when it was configured, else ``None``."""
timeout: Final = float(request_timeout)
if explicitly_set:
return timeout
if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
return timeout

View file

@ -58,6 +58,7 @@ class RouterDynamicConfig:
input_callback: Sequence[object]
callback_manager: "LoggingCallbackManager"
cache: "Cache | None"
request_timeout_explicitly_set: bool = False
_versions: Final = itertools.count(1)
@ -110,6 +111,7 @@ def current_dynamic_config() -> RouterDynamicConfig:
input_callback=litellm.input_callback,
callback_manager=litellm.logging_callback_manager,
cache=litellm.cache,
request_timeout_explicitly_set=litellm.request_timeout_explicitly_set,
)
@ -120,6 +122,7 @@ _LIVE_SOURCES: Final = MappingProxyType(
"router_max_fallbacks": "ROUTER_MAX_FALLBACKS",
"allowed_fails": "allowed_fails",
"request_timeout": "request_timeout",
"request_timeout_explicitly_set": "request_timeout_explicitly_set",
"fallbacks": "fallbacks",
"default_fallbacks": "default_fallbacks",
"context_window_fallbacks": "context_window_fallbacks",

View file

@ -98,6 +98,16 @@ def _as_router(obj: "DeprecatedRouterMembers") -> "Router":
raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only")
def _registry_source(obj: "DeprecatedRouterMembers") -> "Router":
"""What ``registry()`` reads: the Router on the Python core, the host registry on the Rust core.
A deprecated member whose ``registry()`` query checks for a name reads it directly, as the
member did before ``registry()`` existed, so a ``None`` name gets the old result on both cores
instead of ``registry()``'s ``TypeError``.
"""
return _as_router(obj).registry()._router # pyright: ignore[reportPrivateUsage] # the shim is part of the router
def _public_attribute_error(obj: object, core_name: str, public_name: str, error: AttributeError) -> AttributeError:
"""
``error`` renamed to the public attribute when it is the router's own ``core_name`` that is
@ -507,8 +517,8 @@ class DeprecatedRouterMembers:
self, model_name: str, model_alias: str | None = None, team_id: str | None = None
) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged
_note("get_all_deployments", "registry().deployments(model_name, source='all')")
return (
_as_router(self).registry().deployments(model_name, source="all", model_alias=model_alias, team_id=team_id)
return _registry_source(self)._core_get_all_deployments(
model_name=model_name, model_alias=model_alias, team_id=team_id
)
_get_all_deployments = _private_alias("get_all_deployments", get_all_deployments)
@ -519,7 +529,7 @@ class DeprecatedRouterMembers:
def deployments_named(self, model_name: str) -> "tuple[DeploymentTypedDict, ...] | None":
_note("deployments_named", "registry().deployments(model_name, source='named')")
return _as_router(self).registry().deployments(model_name, source="named")
return _registry_source(self)._core_deployments_named(model_name)
def get_model_list_from_model_alias(
self, model_name: str | None = None
@ -535,11 +545,11 @@ class DeprecatedRouterMembers:
self, model_group: str
) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged
_note("get_model_list_of_routed_group", "registry().deployments(model_group, source='routed_group')")
return _as_router(self).registry().deployments(model_group, source="routed_group")
return _registry_source(self)._core_get_model_list_of_routed_group(model_group)
def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]:
_note("wildcard_route_deployments", "registry().deployments(model, source='wildcard')")
return _as_router(self).registry().deployments(model, source="wildcard", team_id=team_id)
return _registry_source(self)._core_wildcard_route_deployments(model, team_id)
def get_routable_deployments(
self,
@ -550,21 +560,17 @@ class DeprecatedRouterMembers:
_note(
"get_routable_deployments", "registry().deployments(model, source='routable', request=..., call_type=...)"
)
return (
_as_router(self)
.registry()
.deployments(model, source="routable", request=request_kwargs, call_type=call_type)
)
return _registry_source(self)._core_get_routable_deployments(model, request_kwargs, call_type)
def get_deployment_by_model_group_name(self, model_group_name: str) -> "Deployment | None":
_note("get_deployment_by_model_group_name", "registry().deployments(name, source='first_in_group')")
return _as_router(self).registry().deployments(model_group_name, source="first_in_group")
return _registry_source(self)._core_get_deployment_by_model_group_name(model_group_name=model_group_name)
def deployments_for_request(
self, model: str, request_kwargs: Mapping[str, object]
) -> "Sequence[DeploymentTypedDict]":
_note("deployments_for_request", "registry().deployments(model, source='for_request', request=...)")
return _as_router(self).registry().deployments(model, source="for_request", request=request_kwargs)
return _registry_source(self)._core_deployments_for_request(model, request_kwargs)
def should_include_deployment(
self,
@ -599,34 +605,32 @@ class DeprecatedRouterMembers:
self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False
) -> tuple[str, ...]:
_note("resolved_litellm_models", "registry().names(model_name, kind='litellm_models' or 'litellm_base_models')")
return (
_as_router(self)
.registry()
.names(model_name, kind="litellm_base_models" if use_base_model else "litellm_models", team_id=team_id)
return _registry_source(self)._core_resolved_litellm_models(
model_name, team_id=team_id, use_base_model=use_base_model
)
def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]:
_note("wildcard_routed_models", "registry().names(model, kind='wildcard')")
return _as_router(self).registry().names(model, kind="wildcard", team_id=team_id)
return _registry_source(self)._core_wildcard_routed_models(model, team_id)
def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]:
_note("get_candidate_model_ids_for_route", "registry().names(model, kind='route_candidates')")
return _as_router(self).registry().names(model, kind="route_candidates", team_id=team_id)
return _registry_source(self)._core_get_candidate_model_ids_for_route(model, team_id=team_id)
# -- registry: resolving names ---------------------------------------------------------------
def routable_model_group(self, model_name: str) -> str:
_note("routable_model_group", "registry().resolve(name)")
return _as_router(self).registry().resolve(model_name)
return _registry_source(self)._core_routable_model_group(model_name)
def get_model_from_alias(self, model: str) -> str | None:
_note("get_model_from_alias", "registry().resolve(name, kind='alias')")
return _as_router(self).registry().resolve(model, kind="alias")
return _registry_source(self)._core_get_model_from_alias(model=model)
_get_model_from_alias = _private_alias("get_model_from_alias", get_model_from_alias)
def resolve_served_model_name(self, model_name: str, served_model: str) -> str:
_note("resolve_served_model_name", "registry().resolve(name, kind='served', served_model=...)")
return _as_router(self).registry().resolve(model_name, kind="served", served_model=served_model)
return _registry_source(self)._core_resolve_served_model_name(model_name, served_model)
def resolve_model_name_from_model_id(
self, model_id: str | None, custom_llm_provider: str | None = None
@ -646,23 +650,25 @@ class DeprecatedRouterMembers:
def is_recognized_model(self, model: str) -> bool:
_note("is_recognized_model", "registry().resolve(name, kind='recognized')")
return _as_router(self).registry().resolve(model, kind="recognized")
return _registry_source(self)._core_is_recognized_model(model)
def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None:
_note("get_routable_upstream_model", "registry().resolve(name, kind='upstream')")
return _as_router(self).registry().resolve(model_name, kind="upstream", team_id=team_id)
return _registry_source(self)._core_get_routable_upstream_model(model_name, team_id=team_id)
def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]:
_note("get_search_tool_support", "registry().resolve(name, kind='search_tool_support')")
return _as_router(self).registry().resolve(model_name, kind="search_tool_support", team_id=team_id)
return _registry_source(self)._core_get_search_tool_support(model_name, team_id=team_id)
def get_model_group(self, id: str) -> list | None: # mutable-ok: the old return type, unchanged
_note("get_model_group", "registry().resolve(id, kind='group_of_id')")
return _as_router(self).registry().resolve(id, kind="group_of_id")
return _registry_source(self)._core_get_model_group(id=id)
def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool:
_note("is_model_access_group_for_wildcard_route", "registry().resolve(group, kind='wildcard_access_group')")
return _as_router(self).registry().resolve(model_access_group, kind="wildcard_access_group")
return _registry_source(self)._core_is_model_access_group_for_wildcard_route(
model_access_group=model_access_group
)
_is_model_access_group_for_wildcard_route = _private_alias(
"is_model_access_group_for_wildcard_route", is_model_access_group_for_wildcard_route
@ -670,7 +676,7 @@ class DeprecatedRouterMembers:
def team_model_has_alternatives(self, deployment_id: str) -> bool:
_note("team_model_has_alternatives", "registry().resolve(id, kind='team_model_alternatives')")
return _as_router(self).registry().resolve(deployment_id, kind="team_model_alternatives")
return _registry_source(self)._core_team_model_has_alternatives(deployment_id)
def routing_group_has_alternatives(self, model_group: str | None) -> bool:
_note("routing_group_has_alternatives", "registry().resolve(group, kind='routing_group_alternatives')")

View file

@ -180,7 +180,7 @@ from litellm.litellm_core_utils.reasoning_effort_capability import (
resolve_supported_reasoning_efforts,
)
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
configured_request_timeout,
get_service_tier_timeout,
)
from litellm.litellm_core_utils.routing_common_utils import (
@ -366,6 +366,7 @@ from litellm.router_utils.fallback_event_handlers import (
mid_stream_fallback_hop_kwargs,
mid_stream_fallback_snapshot_kwargs,
mid_stream_retry_kwargs,
non_stream_fallback_answer_as_chunk,
per_request_fallback_controls,
record_disable_fallbacks,
record_pre_routing_selection,
@ -699,6 +700,12 @@ _NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
# The router strategies whose pick runs the pre-routing hook (`_async_get_available_deployment`).
_PRE_ROUTING_PICK_STRATEGIES: Final = frozenset(
{"usage-based-routing-v2", "simple-shuffle", "cost-based-routing", "latency-based-routing", "least-busy"}
)
# The generic calls (sync and async) resolve their timeout with the LLM pass-through resolver.
_GENERIC_CALL_FUNCTION_NAMES: Final = frozenset({"_ageneric_api_call_with_fallbacks", "generic_api_call"})
_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
_RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
@ -2083,7 +2090,7 @@ class PythonCore(DeprecatedRouterMembers):
# Per-attempt request_timeout, independent of router_settings.timeout.
# Only stored when a router timeout is also set, since otherwise
# request_timeout already flows through self.timeout above.
self.request_timeout = get_configured_request_timeout() if timeout is not None else None
self.request_timeout = self._process_request_timeout() if timeout is not None else None
self.stream_timeout = stream_timeout
self.retry_after = retry_after
@ -2972,16 +2979,13 @@ class PythonCore(DeprecatedRouterMembers):
if selector is None:
return None
# `cost-based-routing` is intentionally omitted —
# `LowestCostLoggingHandler` only implements
# `async_get_available_deployments`
match strategy:
case "least-busy":
return selector.get_available_deployments(
model_group=model,
healthy_deployments=healthy_deployments,
)
case "usage-based-routing" | "usage-based-routing-v2":
case "usage-based-routing" | "usage-based-routing-v2" | "cost-based-routing":
return selector.get_available_deployments(
model_group=model,
healthy_deployments=healthy_deployments,
@ -3411,6 +3415,7 @@ class PythonCore(DeprecatedRouterMembers):
model_client = None
else:
model_client = potential_model_client
self.total_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit)
## only run if model group given, not model id
@ -3427,6 +3432,7 @@ class PythonCore(DeprecatedRouterMembers):
}
)
response: Final = self._host.invoke("completion")(**input_kwargs)
self.success_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
## CHECK CONTENT FILTER ERROR ##
@ -3459,6 +3465,8 @@ class PythonCore(DeprecatedRouterMembers):
return response
except Exception as e:
verbose_router_logger.info("litellm.completion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
if model_name is not None:
self.fail_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
# Set per-deployment num_retries on exception for retry logic
if deployment is not None:
self._set_deployment_num_retries_on_exception(e, deployment)
@ -3733,6 +3741,17 @@ class PythonCore(DeprecatedRouterMembers):
adopting_wrapper.adopt_fallback_response_headers(fallback_response, prepared)
return prepared
@staticmethod
def _non_stream_fallback_item(
wrapper_ref: "weakref.ref[FallbackAwareStreamWrapper]", fallback_response: object
) -> object:
"""The one item a stream yields for a fallback that answered without a stream: its
response as a stream chunk, carrying the fallback's hidden params and headers."""
prepared: Final = PythonCore._adopt_fallback_response_headers(wrapper_ref, fallback_response)
item: Final = non_stream_fallback_answer_as_chunk(fallback_response)
PythonCore._apply_fallback_hidden_params_to_item(item, prepared)
return item
@staticmethod
def _apply_fallback_hidden_params_to_item(
fallback_item: object,
@ -3949,8 +3968,7 @@ class PythonCore(DeprecatedRouterMembers):
)
yield fallback_item
else:
# If fallback returns a non-streaming response, yield None
yield None
yield PythonCore._non_stream_fallback_item(wrapper_ref, fallback_response) # pyright: ignore[reportAny] # the legacy untyped router core
except Exception as fallback_error:
# If fallback also fails, log and re-raise original error
@ -4710,7 +4728,7 @@ class PythonCore(DeprecatedRouterMembers):
initial_kwargs["messages"] = messages
router_self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs)
fallback_response = run_async_function(
router_self.async_function_with_fallbacks_common_utils,
router_self.async_function_with_fallbacks_common_utils, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
e=e.original_exception
if isinstance(e.original_exception, litellm_exceptions.ContextWindowExceededError)
else e,
@ -4724,7 +4742,7 @@ class PythonCore(DeprecatedRouterMembers):
include_fallback_errors=initial_kwargs.get("include_fallback_errors", False) is True,
)
if hasattr(fallback_response, "__iter__"):
if hasattr(fallback_response, "__iter__") and not isinstance(fallback_response, ModelResponse): # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
prepared_fallback_hidden_params = PythonCore._adopt_fallback_response_headers(
wrapper_ref, fallback_response
)
@ -4747,7 +4765,7 @@ class PythonCore(DeprecatedRouterMembers):
router_self._combine_fallback_usage(fallback_item, None)
yield fallback_item
else:
yield None
yield PythonCore._non_stream_fallback_item(wrapper_ref, fallback_response) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
except Exception as fallback_error:
verbose_router_logger.error("Fallback also failed: %s", fallback_error)
@ -4766,9 +4784,9 @@ class PythonCore(DeprecatedRouterMembers):
"stream_with_fallbacks: error closing model_response: %s",
close_err,
)
if fallback_response is not None and hasattr(fallback_response, "close"):
if fallback_response is not None and hasattr(fallback_response, "close"): # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
try:
fallback_response.close()
fallback_response.close() # pyright: ignore[reportAttributeAccessIssue] # a stream wrapper, closed when it has close()
except BaseException as close_err:
verbose_router_logger.debug(
"stream_with_fallbacks: error closing fallback_response: %s",
@ -5299,17 +5317,19 @@ class PythonCore(DeprecatedRouterMembers):
kwargs["model_info"] = model_info
if function_name == "_ageneric_api_call_with_fallbacks":
if function_name in _GENERIC_CALL_FUNCTION_NAMES:
from litellm.passthrough.timeout_utils import (
resolve_llm_passthrough_timeout,
)
# The process request_timeout is read through the host; it ranks right after the
# router timeout, as resolve_llm_passthrough_timeout ranks it.
_router_timeout: Final = (
self.request_timeout
if self.request_timeout is not None
else float(self._explicit_timeout)
if isinstance(self._explicit_timeout, (int, float))
else None
else self._process_request_timeout()
)
_router_stream_timeout: Final = (
self.stream_timeout
@ -5381,6 +5401,11 @@ class PythonCore(DeprecatedRouterMembers):
hop_kwargs, _FALLBACK_HOP_ADAPTER.validate_python(deployment)
)
def _process_request_timeout(self) -> float | None:
"""The host's explicitly configured process ``request_timeout``, else None."""
dynamic: Final = self._host.dynamic()
return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set)
def _get_async_openai_model_client(self, deployment: dict, kwargs: dict):
"""
Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment
@ -5926,7 +5951,7 @@ class PythonCore(DeprecatedRouterMembers):
raise e
def _image_generation(self, prompt: str, model: str, **kwargs):
model_name: Final = ""
model_name = model # rebind-ok: the group until a deployment is picked, as in _aimage_generation
try:
verbose_router_logger.debug("Inside _image_generation()- model: %s", model)
deployment: Final = self.get_available_deployment(
@ -5937,6 +5962,7 @@ class PythonCore(DeprecatedRouterMembers):
)
self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs)
data: Final = deployment["litellm_params"].copy()
model_name = data["model"] # pyright: ignore[reportUnknownVariableType] # rebind-ok: the picked deployment's model, as in _aimage_generation
model_client: Final = self._get_async_openai_model_client(
deployment=deployment,
@ -5958,10 +5984,10 @@ class PythonCore(DeprecatedRouterMembers):
}
)
self.success_calls[model_name] += 1
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
return response
except Exception as e:
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
if model_name is not None:
self.fail_calls[model_name] += 1
self._stamp_retry_skip_deployment_id(e, kwargs)
@ -6300,37 +6326,64 @@ class PythonCore(DeprecatedRouterMembers):
is_async: bool | None = False,
**kwargs,
):
messages: Final = [{"role": "user", "content": prompt}]
try:
kwargs["model"] = model
kwargs["prompt"] = prompt
kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries)
kwargs.setdefault("metadata", {}).update({"model_group": model})
"""The sync twin of `atext_completion`: picks, shapes, retries and falls back the same way."""
kwargs["model"] = model # rebind-ok: the request kwargs, shaped in place as atext_completion shapes them
kwargs["prompt"] = prompt # rebind-ok: same
kwargs["original_function"] = self._text_completion # rebind-ok: same
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
return self.function_with_fallbacks(**kwargs) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # the legacy untyped router core
# pick the one that is available (lowest TPM/RPM)
with endpoint_selection("text_completion", model=model):
deployment: Final = self.get_available_deployment(
model=model,
messages=messages,
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
def _text_completion(
self,
model: str,
prompt: str,
**kwargs: object, # kwargs-ok: the request kwargs, sent on to the deployment
) -> object:
try:
verbose_router_logger.debug("Inside _text_completion()- model: %s", model)
deployment: Final = self.get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy untyped router core
model=model,
messages=[{"role": "user", "content": prompt}],
specific_deployment=kwargs.pop("specific_deployment", None), # pyright: ignore[reportArgumentType] # the legacy untyped router core
request_kwargs=kwargs,
)
self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, splits_stream_timeout=True) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
data: Final = deployment["litellm_params"].copy()
for k, v in self._core_default_litellm_params.items():
if k not in kwargs: # prioritize model-specific params > default router params
kwargs[k] = v
elif k == "metadata":
kwargs[k].update(v)
pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data))
if pinned_model_id is not None:
kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults
model_name: Final = data["model"] # pyright: ignore[reportUnknownVariableType] # the legacy untyped router core
# call via litellm.completion()
return self._host.invoke("text_completion")(
**{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs}
potential_model_client: Final = self._get_client(deployment=deployment, kwargs=kwargs, client_type="sync") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy untyped router core
dynamic_api_key: Final = kwargs.get("api_key", None)
model_client: Final = ( # pyright: ignore[reportUnknownVariableType] # the legacy untyped router core
None
if dynamic_api_key is not None
and potential_model_client is not None
and dynamic_api_key != potential_model_client.api_key # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
else potential_model_client
)
self.total_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit)
self.routing_strategy_pre_call_checks(deployment=deployment) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
response: Final = self._host.invoke("text_completion")( # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
**{
**data,
"prompt": prompt,
"caching": self._core_cache_responses,
"client": model_client,
**kwargs,
}
)
self.success_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
return response
except Exception as e:
verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model, e)
if model is not None: # pyright: ignore[reportUnnecessaryComparison] # the legacy untyped router core
self.fail_calls[model] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
self._stamp_retry_skip_deployment_id(e, kwargs)
raise e
async def atext_completion(
@ -7476,8 +7529,11 @@ class PythonCore(DeprecatedRouterMembers):
# For passthrough routes, use the actual model from deployment
# and swap model name in endpoint if present
if "endpoint" in kwargs and kwargs["endpoint"]:
kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name)
self._add_deployment_model_to_endpoint_for_llm_passthrough_route(
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
model=model,
model_name=model_name, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
)
kwargs["model"] = model_name
# Perform pre-call checks for routing strategy
@ -9473,7 +9529,7 @@ class PythonCore(DeprecatedRouterMembers):
Wrapped to reduce code duplication and prevent bugs.
"""
return run_async_function(self.async_function_with_fallbacks, *args, **kwargs)
return run_async_function(self.async_function_with_fallbacks, *args, **kwargs) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
def _get_fallback_model_group_for_lookup_groups(
self,
@ -10232,11 +10288,22 @@ class PythonCore(DeprecatedRouterMembers):
- None
Raises:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits; the deployment is
cooled down first, as on the async path
"""
for _callback in self._routing_callbacks():
if isinstance(_callback, CustomLogger):
_callback.pre_call_check(deployment)
try:
_callback.pre_call_check(deployment) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
except litellm_exceptions.RateLimitError as e:
set_cooldown_deployments(
ctx=self._core_ctx,
exception_status=e.status_code,
original_exception=e,
deployment=deployment["model_info"]["id"], # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
time_to_cooldown=self._core_cooldown_time,
)
raise
async def async_routing_strategy_pre_call_checks(
self,
@ -16234,6 +16301,18 @@ class PythonCore(DeprecatedRouterMembers):
)
return registered_model_name
def _core_sync_pick_pre_routes(self) -> bool:
"""Whether the sync pick runs the pre-routing hook: a strategy router is registered and the
router strategy is one whose async pick runs the hook too (see `_async_get_available_deployment`)."""
return self._core_routing_strategy in _PRE_ROUTING_PICK_STRATEGIES and any(
(
self._core_auto_routers,
self._core_complexity_routers,
self._core_adaptive_routers,
self._core_quality_routers,
)
)
async def _core_async_pre_routing_hook(
self,
model: str,
@ -16646,6 +16725,33 @@ class PythonCore(DeprecatedRouterMembers):
"usage-based-routing-v2, cost-based-routing, latency-based-routing, least-busy), "
"or remove `plugins` from the Router config."
)
if self._core_sync_pick_pre_routes():
# The async pick's pre-routing hook (auto/complexity/adaptive/quality routers), run to
# completion from the sync pick, so a strategy router serves sync calls too.
hook_kwargs: Final[dict[str, object]] = ( # pyright: ignore[reportUnknownVariableType] # mutable-ok: the request kwargs the hook stamps
request_kwargs if request_kwargs is not None else {}
)
responses_call: Final = input is not None and messages is None
pre_routed: Final = run_async_function(
partial(
self._core_async_pre_routing_hook, # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
model=model,
request_kwargs=hook_kwargs,
messages=messages,
input=input,
specific_deployment=specific_deployment,
)
)
if pre_routed is not None:
model = pre_routed.model # rebind-ok: the tier group the strategy picked, as the async pick rebinds it
messages = pre_routed.messages # rebind-ok: same
record_pre_routing_selection(hook_kwargs, model)
self._pin_tier_params_onto_request( # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
model=model,
tier_litellm_params=pre_routed.litellm_params if pre_routed is not None else None,
request_kwargs=hook_kwargs,
responses_call=responses_call,
)
# users need to explicitly call a specific deployment, by setting `specific_deployment = True` as completion()/embedding() kwarg
# When this was no explicit we had several issues with fallbacks timing out
@ -16926,7 +17032,7 @@ class PythonCore(DeprecatedRouterMembers):
if len(pass_through_deployments) == 0:
# No deployments support pass-through
raise litellm_exceptions.BadRequestError(
message=f"Model {model} has no deployment configured with use_in_pass_through=True. Please add use_in_pass_through: true in the deployment configuration",
message=f"Model {model} has no deployments configured with use_in_pass_through=True. Please add use_in_pass_through: true to the deployment configuration",
model=model,
llm_provider="",
)

View file

@ -424,6 +424,19 @@ def failure_kind(error: BaseException) -> str:
return next((name for name in names if name in _FAILURE_KINDS), "APIError")
def _crate_int(value: object) -> int | None:
"""A count setting kept as given, as the crate reads it: an int, or a number spelled another way
(``1.5``, ``"3"``, ``"1e-05"``) truncated to one; None (the crate's default) when it is no number."""
if value is None or isinstance(value, bool):
return None if value is None else int(value)
if isinstance(value, int):
return value
try:
return int(float(str(value)))
except (ValueError, OverflowError):
return None
def _retry_after_seconds(error: BaseException) -> float | None:
response: Final = getattr(error, "response", None)
headers: Final[object] = getattr(error, "litellm_response_headers", None) or getattr(response, "headers", None)
@ -490,15 +503,18 @@ class Resolved:
routing_strategy: str | None
routing_strategy_args: Mapping[str, object]
num_retries: int
max_fallbacks: int
allowed_fails: int
explicit_allowed_fails: int | None
# `num_retries`, `max_fallbacks` and `allowed_fails` as the Python core stores them: as given
# (the crate reads them through `_crate_int`).
num_retries: object
max_fallbacks: object
allowed_fails: object
explicit_allowed_fails: object
cooldown_time: float
disable_cooldowns: bool | None
allowed_fails_policy: object
retry_after: float
timeout: float | None
# `retry_after` and `timeout` as the Python core stores them: as given (an int stays an int).
retry_after: object
timeout: object
fallbacks: list[object] | None # mutable-ok: the setting as Python spells it
context_window_fallbacks: list[object] | None # mutable-ok: same
content_policy_fallbacks: list[object] | None # mutable-ok: same
@ -535,15 +551,19 @@ class Resolved:
list(_as_list(get("fallbacks"))) or (dynamic.fallbacks if dynamic.fallbacks else None) # pyright: ignore[reportAssignmentType] # the process list as given
)
refuse_malformed_fallbacks(configured)
default_fallbacks: Final = get("default_fallbacks") or dynamic.default_fallbacks
if isinstance(default_fallbacks, str):
# As the Python core: a given default_fallbacks, even an empty one, adds a `{"*": ...}` entry
# (`[]` falls back to the process default_fallbacks, so it may add `{"*": None}`).
given_default_fallbacks: Final = get("default_fallbacks")
has_default_fallbacks: Final = given_default_fallbacks is not None or dynamic.default_fallbacks is not None
default_fallbacks: Final = given_default_fallbacks or dynamic.default_fallbacks
if has_default_fallbacks and isinstance(default_fallbacks, str):
raise TypeError(
f"default_fallbacks must be a list of model names, got '{default_fallbacks}'. Use ['{default_fallbacks}']"
)
fallbacks: Final[list[object] | None] = ( # mutable-ok: see above
configured
if default_fallbacks is None
else [*(configured if configured is not None else ()), {"*": default_fallbacks}]
[*(configured if configured is not None else ()), {"*": default_fallbacks}]
if has_default_fallbacks
else configured
)
content_policy: Final = list(_as_list(get("content_policy_fallbacks"))) or (
list(dynamic.content_policy_fallbacks) if dynamic.content_policy_fallbacks else None
@ -561,23 +581,23 @@ class Resolved:
if strategy is None and "routing_strategy" in config.arguments
else normalize_strategy(strategy) or "simple-shuffle",
routing_strategy_args=MappingProxyType(dict(as_str_mapping(get("routing_strategy_args")) or {})),
num_retries=int(str(num_retries))
num_retries=num_retries
if num_retries is not None
else dynamic.num_retries
if dynamic.num_retries is not None
else 2,
max_fallbacks=int(str(max_fallbacks))
max_fallbacks=max_fallbacks
if max_fallbacks is not None
else dynamic.max_fallbacks
if dynamic.max_fallbacks is not None
else dynamic.router_max_fallbacks,
allowed_fails=int(str(allowed_fails)) if allowed_fails is not None else dynamic.allowed_fails,
explicit_allowed_fails=int(str(allowed_fails)) if allowed_fails is not None else None,
allowed_fails=allowed_fails if allowed_fails is not None else dynamic.allowed_fails,
explicit_allowed_fails=allowed_fails,
cooldown_time=float(str(get("cooldown_time") or _DEFAULT_COOLDOWN_SECONDS)),
disable_cooldowns=_optional_bool(get("disable_cooldowns")),
allowed_fails_policy=resolved_allowed_fails_policy(get("allowed_fails_policy")),
retry_after=float(str(get("retry_after") or 0)),
timeout=float(str(timeout)) if timeout else dynamic.request_timeout,
retry_after=get("retry_after") if "retry_after" in config.arguments else 0,
timeout=timeout if timeout else dynamic.request_timeout,
fallbacks=fallbacks,
context_window_fallbacks=list(_as_list(get("context_window_fallbacks")))
or (list(dynamic.context_window_fallbacks) if dynamic.context_window_fallbacks else None),
@ -628,7 +648,7 @@ class Resolved:
if name in {"model_group_alias", "default_litellm_params"}:
updates[name] = MappingProxyType(dict(as_str_mapping(value) or {}))
elif name in _NUMBER_SETTINGS:
updates[name] = self._number_setting(name, value)
updates[name] = self._number_setting(name, value, updated=name in updated)
if name == "allowed_fails":
updates["explicit_allowed_fails"] = updates[name]
elif name in _SELECTION_SETTINGS:
@ -648,16 +668,18 @@ class Resolved:
updates["routing_group_members"] = None
return dataclasses.replace(self, **updates, assigned=self.assigned | frozenset(values)) # pyright: ignore[reportArgumentType] # each value converted to its field's type above
def _number_setting(self, name: str, value: object) -> object:
"""A numeric setting converted to its field's type."""
if name in ("num_retries", "max_fallbacks", "allowed_fails"):
return int(str(value))
def _number_setting(self, name: str, value: object, *, updated: bool = False) -> object:
"""A numeric setting converted to its field's type. ``retry_after`` and ``timeout`` are kept
as the Python core keeps them: ``update_settings`` stores ``int(value)``, an attribute write
the value as given."""
if name in ("retry_after", "timeout", "num_retries", "allowed_fails"):
return int(value) if updated else value # pyright: ignore[reportArgumentType] # int() of the value as given, as Python casts it
if name == "max_fallbacks":
return value
if name == "cooldown_time":
# Stored as given (`_core_cooldown_time`), so a failure cools down for that long:
# 0 for no time at all; None leaves the cooldown cache's own default, set by the caller.
return float(str(value)) if value is not None else self.cooldown_time
if name == "retry_after":
return float(str(value or 0))
return float(str(value)) if value is not None else None
def _selection_setting(self, name: str, value: object, *, validate: bool) -> object:
@ -681,8 +703,8 @@ class Resolved:
"cooldown_time": self.cooldown_time,
"disable_cooldowns": self.disable_cooldowns,
"allowed_fails_policy": _plain(self.allowed_fails_policy),
"retry_after": self.retry_after,
"timeout": self.timeout,
"retry_after": float(str(self.retry_after or 0)),
"timeout": float(str(self.timeout)) if self.timeout is not None else None,
"routing_groups": group_documents(self.routing_groups) if self.routing_groups else None,
"enable_pre_call_checks": True if self.enable_pre_call_checks else None,
"routing_plugins": True if self.routing_plugins else None,
@ -706,16 +728,16 @@ class Resolved:
"fallbacks": _plain(self.fallbacks),
"context_window_fallbacks": _plain(self.context_window_fallbacks),
"content_policy_fallbacks": _plain(self.content_policy_fallbacks),
"num_retries": self.num_retries,
"max_fallbacks": self.max_fallbacks,
"allowed_fails": self._router_allowed_fails(),
"num_retries": _crate_int(self.num_retries),
"max_fallbacks": _crate_int(self.max_fallbacks),
"allowed_fails": _crate_int(self._router_allowed_fails()),
"fallback_keys": self._fallback_keys(model_names),
"default_deployment": _plain(served_default_deployment(self.default_deployment)[0]),
"default_deployment_raises": True if served_default_deployment(self.default_deployment)[1] else None,
}
return {name: value for name, value in values.items() if value is not None}
def _router_allowed_fails(self) -> int | None:
def _router_allowed_fails(self) -> object:
"""``allowed_fails`` when it counts as set on the router (``_is_allowed_fails_set_on_router``):
passed explicitly, or different from the process default."""
import litellm
@ -1033,7 +1055,7 @@ class RustCore(
# Settings the Python core keeps as plain attributes, and settings deleted with `del`.
self._plain: Final = initial_plain(config)
self._deleted: frozenset[str] = frozenset()
self._request_timeout: Final = request_timeout(config)
self._request_timeout: Final = request_timeout(config, host)
legacy: Final[dict[str, object]] = {"deployment_stats": {}} # mutable-ok: the core's own legacy attributes
self._legacy: Final = legacy
self._health_view: RustHealth | None = None
@ -1390,7 +1412,7 @@ class RustCore(
else default_litellm_params(self._config),
default_max_parallel_requests=self._endpoints.default_max_parallel_requests, # pyright: ignore[reportArgumentType] # the setting as given
cooldown_time=self._cooldown_given, # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it
allowed_fails=r.allowed_fails,
allowed_fails=r.allowed_fails, # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it
allowed_fails_policy=r.allowed_fails_policy, # pyright: ignore[reportArgumentType] # the policy as passed
disable_cooldowns=r.disable_cooldowns,
enable_pre_call_checks=r.enable_pre_call_checks,
@ -2018,8 +2040,8 @@ class RustCore(
**self._routing_fields(prepared),
**pinned_video_deployment(plan, request),
**unrouted_fields(plan),
# The sync path never runs pre-routing; a custom strategy picks on both.
**({} if plan.sync and self._custom_strategy is None else {"pre_routes": True}),
# A custom strategy picks on both paths; the sync path pre-routes when it can.
**({"pre_routes": True} if not plan.sync or self._sync_pick_pre_routes() else {}),
**({"mock_failure": mock} if mock is not None else {}),
# Every target of the walk is refused.
**({"fallbacks": []} if rejected_fallback_target(request) is not None else {}),
@ -2389,6 +2411,14 @@ class RustCore(
messages,
)
def _sync_pick_pre_routes(self) -> bool:
"""Whether a sync pick asks for the ``pre_route`` step: always for a custom strategy, else
when a strategy router is registered. A router with routing plugins never pre-routes a sync
pick, so router-core refuses it (the pipeline only runs on the async path)."""
if self._custom_strategy is not None:
return True
return not self._resolved.routing_plugins and self._prerouter.has_strategy_routers()
def _custom_pick_sync(
self,
step: Mapping[str, object],
@ -2397,10 +2427,15 @@ class RustCore(
input: object = _NO_INPUT,
specific_deployment: object = _NO_INPUT,
) -> object:
"""A custom strategy's sync pick (``get_available_deployment``); without one, no pre-routing."""
"""A custom strategy's sync pick (``get_available_deployment``); without one, the async
pre-routing hook run to completion (``_pre_route``)."""
custom: Final = self._custom_strategy
if custom is None:
return None
from litellm.litellm_core_utils import asyncify
return asyncify.run_async_function( # pyright: ignore[reportUnknownMemberType] # the untyped sync bridge
partial(self._pre_route, step, request, messages, input, specific_deployment)
)
picked: Final[object] = custom.get_available_deployment( # pyright: ignore[reportUnknownMemberType] # the strategy's untyped pick
model=str(step.get("model")),
messages=request.get("messages") if messages is _NO_INPUT else messages, # pyright: ignore[reportArgumentType] # see _pre_route
@ -2665,9 +2700,6 @@ class RustCore(
message: Final = (
f"Model {model} has no deployments configured with use_in_pass_through=True. "
"Please add use_in_pass_through: true to the deployment configuration"
if pass_through == "async_pass_through"
else f"Model {model} has no deployment configured with use_in_pass_through=True. "
"Please add use_in_pass_through: true in the deployment configuration"
)
return litellm_exceptions.BadRequestError(message=message, model=model, llm_provider="")
if kind == "context_window_exceeded":
@ -2805,7 +2837,7 @@ class _Selection:
fields["specific_deployment"] = True
if probe:
fields["health_check_probe"] = True
if kind == "pick" or (kind == "sync_pick" and core._custom_strategy is not None): # pyright: ignore[reportPrivateUsage] # the selection serves its own core
if kind == "pick" or (kind == "sync_pick" and core._sync_pick_pre_routes()): # pyright: ignore[reportPrivateUsage] # the selection serves its own core
fields["pre_routes"] = True
self._specific_deployment: Final = specific_deployment
# The error a callback filter raised, which a failed selection raises.
@ -3262,15 +3294,35 @@ class _Driver:
"""A routing pre-call check refused the attempt (``async_routing_strategy_pre_call_checks``):
a rate limit cools the deployment down (``set_cooldown_deployments``); the attempt's failure
is recorded by nothing else."""
report: Final = self._refusal_report(deployment, error)
if report is not None:
await self._core._drive_cache_steps(report) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
return {**self._failed(step, deployment, call, error), "recorded": True}
def _refused_sync(
self,
step: Mapping[str, object],
deployment: Mapping[str, object],
call: "AttemptCall",
error: BaseException,
) -> Mapping[str, object]:
"""``_refused`` on the sync path (``routing_strategy_pre_call_checks``): a rate limit cools
the deployment down here too."""
report: Final = self._refusal_report(deployment, error)
if report is not None:
self._core._drive_sync_steps(report) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
return {**self._failed(step, deployment, call, error), "recorded": True}
def _refusal_report(self, deployment: Mapping[str, object], error: BaseException) -> "RouterCall | None":
"""The uncounted failure report that cools down a deployment a pre-call check rate-limited."""
import litellm.exceptions as litellm_exceptions
core: Final = self._core
if isinstance(error, litellm_exceptions.RateLimitError):
deployment_id: Final = (as_str_mapping(deployment.get("model_info")) or {}).get("id")
if isinstance(deployment_id, str):
report: Final = core._handle.report_failure(deployment_id, dict(attempt_failure(error)), uncounted=True) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
await core._drive_cache_steps(report) # pyright: ignore[reportPrivateUsage] # see above
return {**self._failed(step, deployment, call, error), "recorded": True}
if not isinstance(error, litellm_exceptions.RateLimitError):
return None
deployment_id: Final = (as_str_mapping(deployment.get("model_info")) or {}).get("id")
if not isinstance(deployment_id, str):
return None
return self._core._handle.report_failure(deployment_id, dict(attempt_failure(error)), uncounted=True) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
def _pick_kwargs(self, step: Mapping[str, object]) -> Mapping[str, object]:
"""The request kwargs the pick sees: the request with its level's retry count."""
@ -3365,8 +3417,7 @@ class _Driver:
try:
self._core._endpoints.sync_pre_call_checks(self._plan, deployment, self._request) # pyright: ignore[reportPrivateUsage] # see above
except Exception as refused: # noqa: BLE001 # handed to the Rust call loop like any attempt failure
# The sync path records no outcome for a refused pre-call check: no cooldown.
return {**self._failed(step, deployment, call, refused), "recorded": True}
return self._refused_sync(step, deployment, call, refused)
answered: Final = self._core._host.invoke(call.host_op)(**call.kwargs) # pyright: ignore[reportPrivateUsage] # a sync endpoint
if inspect.isawaitable(answered):
raise _NotSynchronous(f"the host's {call.host_op} endpoint is not synchronous")

View file

@ -70,7 +70,7 @@ Kind = Literal[ # rebind-ok: rebound as the Python router rebinds it
"search",
]
Stamp = Literal["deployment", "retry_skip", "none"] # rebind-ok: rebound as the Python router rebinds it
CountKey = Literal["deployment", "group", "empty"] # rebind-ok: rebound as the Python router rebinds it
CountKey = Literal["deployment", "group"] # rebind-ok: rebound as the Python router rebinds it
# Request keys the router reads to route the call; the deployment call never receives them.
ROUTING_KEYS: Final = frozenset(
@ -174,9 +174,6 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType(
sync=True,
splits_stream_timeout=True,
stamp="deployment",
total_key=None,
success_key=None,
fail_key=None,
named=("messages",),
),
"atext_completion": Plan(
@ -187,6 +184,15 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType(
fail_key="group",
named=("prompt",),
),
"text_completion": Plan(
"text_completion",
"text_completion",
"text",
sync=True,
splits_stream_timeout=True,
fail_key="group",
named=("prompt",),
),
# `_asearch_with_fallbacks`: the search tool name routes as the model, with no deployment
# picked; its helper picks among the router's search tools.
"asearch": Plan(
@ -208,9 +214,6 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType(
"image",
sync=True,
prepare_field=None,
total_key="empty",
success_key="empty",
fail_key="empty",
named=("prompt",),
),
"atranscription": Plan(
@ -379,7 +382,7 @@ class Counters:
@staticmethod
def _key(which: CountKey, model_name: str, group: str) -> str:
return model_name if which == "deployment" else group if which == "group" else ""
return model_name if which == "deployment" else group
def started(self, plan: Plan, call: AttemptCall) -> None:
if plan.total_key is not None:
@ -492,9 +495,7 @@ def attempt_call(
function_name="generic_api_call",
splits_stream_timeout=plan.splits_stream_timeout,
)
endpoint = kwargs.get("endpoint") # rebind-ok: rebound as the Python router rebinds it
if endpoint and isinstance(endpoint, str):
kwargs["endpoint"] = endpoint.replace(requested, model_name)
rewrite_endpoint(core, kwargs, requested, model_name)
kwargs["model"] = model_name
final = { # rebind-ok: rebound as the Python router rebinds it
**data,
@ -516,6 +517,33 @@ def attempt_call(
return AttemptCall(plan.host_op, kwargs, kwargs, requested, group)
def rewrite_endpoint(
core: "RustCore",
kwargs: dict[str, object], # mutable-ok: the attempt's own kwargs, shaped in place
requested: str,
model_name: str,
) -> None:
"""`_add_deployment_model_to_endpoint_for_llm_passthrough_route`: the requested name, as a whole
path segment of the request's endpoint, becomes the deployment's model without its provider."""
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
endpoint: Final = kwargs.get("endpoint")
if not endpoint:
return
try:
stripped, _, _, _ = core._host.invoke("get_llm_provider")( # pyright: ignore[reportPrivateUsage] # the core's host
model=model_name,
custom_llm_provider=kwargs.get("custom_llm_provider"),
api_base=kwargs.get("api_base"),
)
replacement = str(stripped) # rebind-ok: rebound as the Python router rebinds it
except Exception: # noqa: BLE001 # as the Python router: an unresolvable model is used as is
replacement = model_name # rebind-ok: rebound as the Python router rebinds it
kwargs["endpoint"] = replace_path_segment( # rebind-ok: the attempt's kwargs, shaped in place
str(endpoint), requested, replacement
)
def _generic_attempt(
core: "RustCore",
plan: Plan,
@ -526,7 +554,6 @@ def _generic_attempt(
group: str,
) -> AttemptCall:
"""`_ageneric_api_call_with_fallbacks_helper` for one deployment."""
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
from litellm.router_rust_websocket import turn_admission
socket: Final = plan.call_type == "_aresponses_websocket"
@ -540,20 +567,7 @@ def _generic_attempt(
opens_responses_socket=socket,
)
model_name: Final = str(data.get("model"))
endpoint: Final = kwargs.get("endpoint")
if endpoint:
try:
stripped, _, _, _ = core._host.invoke("get_llm_provider")( # pyright: ignore[reportPrivateUsage] # the core's host
model=model_name,
custom_llm_provider=kwargs.get("custom_llm_provider"),
api_base=kwargs.get("api_base"),
)
replacement = str(stripped) # rebind-ok: rebound as the Python router rebinds it
except Exception: # noqa: BLE001 # as the Python router: an unresolvable model is used as is
replacement = model_name # rebind-ok: rebound as the Python router rebinds it
kwargs["endpoint"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
replace_path_segment(str(endpoint), requested, replacement)
)
rewrite_endpoint(core, kwargs, requested, model_name)
provider: Final = _generic_provider(data)
final: Final = {
**without_retrieve_stream_defaults(data, kwargs),
@ -604,14 +618,14 @@ def model_client(
core: "RustCore", plan: Plan, deployment: Mapping[str, object], kwargs: Mapping[str, object]
) -> object:
"""The SDK client cached for the deployment (`_get_async_openai_model_client`; `_get_client`
with the sync keys for sync completion and embedding), dropped when the request carries
with the sync keys for sync completion, text completion and embedding), dropped when the request carries
another api key."""
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
if as_dict(deployment.get("litellm_params")).get("kubernetes_pod_discovery") is True:
return None
model_id: Final = as_dict(deployment.get("model_info")).get("id")
sync_keys: Final = plan.sync and plan.kind in ("chat", "embedding")
sync_keys: Final = plan.sync and plan.kind in ("chat", "embedding", "text")
stream: Final = "stream_" if kwargs.get("stream") is True else ""
key: Final = f"{model_id}_{stream}client" if sync_keys else f"{model_id}_{stream}async_client"
span: Final = get_parent_otel_span_from_kwargs(dict(kwargs))
@ -832,14 +846,13 @@ class RustEndpoints:
"""The public endpoint methods of one Rust core, beyond the routed call loop the core drives."""
def __init__(self, core: "RustCore", config: "RouterConfig") -> None:
from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
from litellm.router_utils.request_priority import InvalidPriority, parse_default_priority
self._core: Final = core
self.counters: Final = Counters()
explicit: Final = config.get("timeout")
self._explicit_timeout: Final = explicit
self._request_timeout: Final = get_configured_request_timeout() if explicit is not None else None
self._request_timeout: Final = self.process_request_timeout() if explicit is not None else None
self._stream_timeout: Final = config.get("stream_timeout")
# apply() flips it after construction, as on the Python core
self._cache_responses: bool = bool(config.get("cache_responses"))
@ -927,8 +940,16 @@ class RustEndpoints:
request_timeout=self._request_timeout,
stream_timeout=self._stream_timeout,
cache_responses=self._cache_responses,
process_request_timeout=self.process_request_timeout,
)
def process_request_timeout(self) -> float | None:
"""The host's explicitly configured process ``request_timeout``, else None."""
from litellm.litellm_core_utils.request_timeout_resolver import configured_request_timeout
dynamic: Final = self._core._host.dynamic() # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core
return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set)
def default_litellm_params(self, resolved: "Resolved") -> Mapping[str, object]:
"""The defaults the Python constructor completes before any call: the router ``timeout``
argument, no client retries, and the caching groups in the default metadata."""
@ -1495,30 +1516,12 @@ class RustEndpoints:
return await self._direct("amoderation", prepared)
def text_completion(self, request: Mapping[str, object]) -> object:
"""The sync `text_completion`: one pick, no retries, the group name as the model."""
from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin
kwargs: Final = {k: v for k, v in request.items() if k not in ("is_retry", "is_fallback", "is_async")}
model: Final = kwargs.get("model")
prompt: Final = kwargs.get("prompt")
"""The sync `text_completion`: routed, retried and fallen back as `atext_completion` is."""
model: Final = request.get("model")
if not isinstance(model, str):
raise TypeError("text_completion() missing 1 required positional argument: 'model'")
kwargs["num_retries"] = kwargs.get("num_retries", self._core._resolved.num_retries) # pyright: ignore[reportPrivateUsage] # see above
metadata_bucket(kwargs, "metadata").update({"model_group": model})
specific: Final = kwargs.pop("specific_deployment", None)
deployment: Final = self.select_sync(model, kwargs, specific=bool(specific))
data: Final = as_dict(deployment.get("litellm_params"))
for key, value in self._core._resolved.default_litellm_params.items(): # pyright: ignore[reportPrivateUsage] # see above
if key not in kwargs:
kwargs[key] = value
elif key == "metadata":
bucket = kwargs[key]
if isinstance(bucket, dict):
bucket.update(as_dict(value)) # pyright: ignore[reportUnknownMemberType] # the request's metadata dict
pinned: Final = deployment_model_id_pin(data)
if pinned is not None:
kwargs["model_id"] = pinned
return self._invoke("text_completion")(**{**data, "prompt": prompt, "caching": self._cache_responses, **kwargs})
kwargs: Final = {k: v for k, v in request.items() if k not in ("is_retry", "is_fallback", "is_async")}
return self._core._call_sync(PLANS["text_completion"], kwargs) # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core
def select_sync(self, model: str, request: Mapping[str, object], *, specific: bool = False) -> Mapping[str, object]:
"""Pick a deployment for ``model`` on the sync path (`get_available_deployment`)."""

View file

@ -705,6 +705,10 @@ class PreRouter:
# -- the step --------------------------------------------------------------------------------
def has_strategy_routers(self) -> bool:
"""Whether any auto, complexity, adaptive or quality router is registered."""
return self._strategies.any_registered()
async def pre_route(
self,
model: str,

View file

@ -669,7 +669,7 @@ class HostRegistry(HostModelInfo):
info_dict: UntypedEntry = model_info if isinstance(model_info, dict) else {} # pyright: ignore[reportAssignmentType] # a configured model_info
declared_id = None if info_dict.get("id") is None else str(info_dict["id"])
if "id" not in info_dict:
info_dict["id"] = deployment_id(str(model_name), litellm_params)
info_dict["id"] = deployment_id(model_name, litellm_params) # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it
organization = litellm_params.get("organization")
if isinstance(organization, list):
# Each organization's deployment has its own id (``organization_deployment_id``).
@ -677,7 +677,7 @@ class HostRegistry(HostModelInfo):
litellm_params["organization"] = org
self._create_deployment(
model,
str(model_name),
model_name, # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it
litellm_params,
{
**info_dict,
@ -685,7 +685,7 @@ class HostRegistry(HostModelInfo):
declared_id,
position,
org,
lambda: deployment_id(str(model_name), litellm_params),
lambda: deployment_id(model_name, litellm_params), # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it
),
},
declared_id=declared_id,
@ -694,12 +694,14 @@ class HostRegistry(HostModelInfo):
else:
self._create_deployment(
model,
str(model_name),
model_name, # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it
litellm_params,
info_dict,
declared_id=declared_id,
duplicate_ids=duplicate_ids,
)
# Python logs the group names here, which reads every alias entry (a malformed one raises).
verbose_router_logger.debug("\nInitialized Model List %s", self._core_get_model_names())
self._core_model_names = {str(_LITELLM_PARAMS_ADAPTER.validate_python(m).get("model_name")) for m in model_list}
if self.strategies is not None:
self.strategies.finalize()

View file

@ -281,13 +281,14 @@ def pick_sync(
input: object,
request_kwargs: dict[str, object] | None, # mutable-ok: see pick_async
) -> object:
"""``_select_deployment_sync``: cost-based routing has no sync pick."""
"""``_select_deployment_sync``."""
match selector:
case LeastBusyLoggingHandler() if strategy == "least-busy":
return selector.get_available_deployments(model_group=model, healthy_deployments=deployments) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts
case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() if strategy in (
case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() | LowestCostLoggingHandler() if strategy in (
"usage-based-routing",
"usage-based-routing-v2",
"cost-based-routing",
):
return selector.get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts
model_group=model,

View file

@ -184,11 +184,15 @@ def _as_dict(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, dict) else None # pyright: ignore[reportUnknownVariableType] # the argument dict, as passed
def request_timeout(config: "RouterConfig") -> float | None:
"""The per-attempt ``request_timeout``: only stored when a router timeout is also set."""
from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
def request_timeout(config: "RouterConfig", host: "PythonHost") -> float | None:
"""The per-attempt ``request_timeout``: the host's configured process ``request_timeout``, only
stored when a router timeout is also set."""
from litellm.litellm_core_utils.request_timeout_resolver import configured_request_timeout
return get_configured_request_timeout() if config.get("timeout") is not None else None
if config.get("timeout") is None:
return None
dynamic: Final = host.dynamic()
return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set)
def apply_construction_effects(config: "RouterConfig", host: "PythonHost") -> None:

View file

@ -8,7 +8,7 @@ the same precedence and the same oddities. Nothing here reads a Python router.
"""
import dataclasses
from collections.abc import Mapping, MutableMapping, Sequence
from collections.abc import Callable, Mapping, MutableMapping, Sequence
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
@ -62,6 +62,8 @@ _FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt", "anthro
_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params"
# Generic handlers whose stream timeout is split from the request timeout.
STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
# The generic calls (sync and async) resolve their timeout with the LLM pass-through resolver.
GENERIC_CALL_FUNCTION_NAMES: Final = frozenset({"_ageneric_api_call_with_fallbacks", "generic_api_call"})
@dataclasses.dataclass(frozen=True, slots=True)
@ -70,7 +72,8 @@ class Shaping:
``timeout`` is the router's ``timeout`` (its own setting, else the process ``request_timeout``),
``request_timeout`` the explicitly configured process ``request_timeout``, kept only when the
router has a timeout of its own, as the Python router keeps them."""
router has a timeout of its own, as the Python router keeps them; ``process_request_timeout``
reads the host's explicitly configured process ``request_timeout`` at call time."""
default_litellm_params: Mapping[str, object]
timeout: object
@ -78,6 +81,7 @@ class Shaping:
request_timeout: float | None
stream_timeout: object
cache_responses: bool
process_request_timeout: Callable[[], float | None] = lambda: None
def as_dict(value: object) -> dict[str, object]: # mutable-ok: a fresh dict of a request mapping
@ -532,15 +536,17 @@ def _set_timeouts(
function_name: str | None,
splits_stream_timeout: bool,
) -> None:
if function_name == "_ageneric_api_call_with_fallbacks":
if function_name in GENERIC_CALL_FUNCTION_NAMES:
from litellm.passthrough.timeout_utils import resolve_llm_passthrough_timeout
# The process request_timeout, read through the host, ranks right after the router timeout,
# as resolve_llm_passthrough_timeout ranks it.
router_timeout: Final = (
shaping.request_timeout
if shaping.request_timeout is not None
else float(shaping.explicit_timeout)
if isinstance(shaping.explicit_timeout, (int, float))
else None
else shaping.process_request_timeout()
)
router_stream_timeout: Final = (
shaping.stream_timeout

View file

@ -55,7 +55,8 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, set_hidden_params
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.types.utils import ModelResponseStream, Usage
from litellm.router_utils.fallback_event_handlers import non_stream_fallback_answer_as_chunk
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
if TYPE_CHECKING:
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
@ -617,6 +618,17 @@ def _adopt_fallback_response_headers(
return prepared
def _non_stream_fallback_item(
wrapper_ref: "weakref.ref[FallbackAwareStreamWrapper]", fallback_response: object
) -> object:
"""The one item a stream yields for a fallback that answered without a stream: its response as
a stream chunk, carrying the fallback's hidden params and headers."""
prepared: Final = _adopt_fallback_response_headers(wrapper_ref, fallback_response)
item: Final = non_stream_fallback_answer_as_chunk(fallback_response)
_apply_fallback_hidden_params_to_item(item, prepared)
return item
def _chat_wrapper_init(
wrapper: FallbackAwareStreamWrapper, stream: object, model_response: CustomStreamWrapper
) -> None:
@ -748,8 +760,7 @@ def wrap_chat_stream( # noqa: C901 # one function per Python wrapper it ports
)
yield fallback_item
else:
# A fallback that answers without a stream yields None (an oddity kept as is).
yield None
yield _non_stream_fallback_item(wrapper_ref, fallback_response)
except Exception as fallback_error:
verbose_router_logger.error("Fallback also failed: %s", fallback_error)
if isinstance(trigger, MidStreamFallbackError) and trigger.original_exception is not None:
@ -801,7 +812,7 @@ def wrap_chat_stream_sync(
trigger: Final = _fallback_trigger(e, (litellm_exceptions.ContextWindowExceededError,))
try:
fallback_response = attempt.fallback_sync(trigger) # rebind-ok: rebound as the stream advances
if hasattr(fallback_response, "__iter__"):
if hasattr(fallback_response, "__iter__") and not isinstance(fallback_response, ModelResponse):
prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) # rebind-ok: re-adopted
settled = False # rebind-ok: rebound as the stream advances
for fallback_item in cast("Iterable[object]", fallback_response): # cast-ok: checked
@ -817,7 +828,7 @@ def wrap_chat_stream_sync(
_combine_fallback_usage(fallback_item, None)
yield fallback_item
else:
yield None
yield _non_stream_fallback_item(wrapper_ref, fallback_response)
except Exception as fallback_error:
verbose_router_logger.error("Fallback also failed: %s", fallback_error)
if isinstance(trigger, MidStreamFallbackError) and trigger.original_exception is not None:

View file

@ -209,15 +209,32 @@ class LowestCostLoggingHandler(CustomLogger):
"""
Returns a deployment with the lowest cost
"""
cost_key: Final = f"cost_map:{model_group}"
request_count_dict: Final = await self.router_cache.async_get_cache(key=f"cost_map:{model_group}") or {} # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
return self._lowest_cost_deployment(request_count_dict, healthy_deployments, messages, input) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
request_count_dict: Final = await self.router_cache.async_get_cache(key=cost_key) or {}
# -----------------------
# Find lowest used model
# ----------------------
float("inf")
@with_service_target("router_usage")
def get_available_deployments( # pyright: ignore[reportUnknownParameterType] # noqa: ANN201 # the selector's untyped deployment dict, as the async twin returns it
self,
model_group: str,
healthy_deployments: list, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the router's list of deployment dicts, as the async twin takes it
messages: list[dict[str, str]] | None = None, # mutable-ok: the request's messages, as given
input: str | list | None = None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the request's input, as given
request_kwargs: dict | None = None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: unused, the same signature as the async twin
):
"""
The sync twin of `async_get_available_deployments`: the deployment with the lowest cost
"""
request_count_dict: Final = self.router_cache.get_cache(key=f"cost_map:{model_group}") or {} # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
return self._lowest_cost_deployment(request_count_dict, healthy_deployments, messages, input) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
def _lowest_cost_deployment( # pyright: ignore[reportUnknownParameterType] # noqa: ANN202 # the selector's untyped deployment dict
self,
request_count_dict: dict | None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the untyped cached usage map, filled in place as before
healthy_deployments: list, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the router's list of deployment dicts
messages: list[dict[str, str]] | None, # mutable-ok: the request's messages, as given
input: str | list | None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the request's input, as given
):
"""The cheapest healthy deployment still under its tpm/rpm limits this minute, else None."""
current_date: Final = datetime.now().strftime("%Y-%m-%d")
current_hour: Final = datetime.now().strftime("%H")
current_minute: Final = datetime.now().strftime("%M")
@ -296,8 +313,8 @@ class LowestCostLoggingHandler(CustomLogger):
verbose_router_logger.debug(
"item_cost: %s, item_tpm: %s, item_rpm: %s, model_id: %s",
item_cost,
item_tpm,
item_rpm,
item_tpm, # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage map
item_rpm, # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage map
_deployment.get("model_info", {}).get("id"),
)

View file

@ -469,6 +469,68 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
[f"{prefix}:rpm:{current_minute}" for prefix in prefixes],
)
def _no_deployments_available_error(
self,
model_group: str,
healthy_deployments: Sequence[object],
tpm_values: Sequence[object] | None,
rpm_values: Sequence[object] | None,
) -> litellm.RateLimitError:
"""The error both the sync and the async pick raise when no deployment is under its limits."""
### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ###
deployment_dict: Final = {}
for index, _deployment in enumerate(healthy_deployments):
if isinstance(_deployment, dict):
id = _deployment.get("model_info", {}).get("id")
### GET DEPLOYMENT TPM LIMIT ###
_deployment_tpm = None
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("model_info", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = float("inf")
### GET CURRENT TPM ###
current_tpm = tpm_values[index] if tpm_values else 0
### GET DEPLOYMENT TPM LIMIT ###
_deployment_rpm = None
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("model_info", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = float("inf")
### GET CURRENT RPM ###
current_rpm = rpm_values[index] if rpm_values else 0
deployment_dict[id] = {
"current_tpm": current_tpm,
"tpm_limit": _deployment_tpm,
"current_rpm": current_rpm,
"rpm_limit": _deployment_rpm,
}
return litellm.RateLimitError(
message=f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}",
llm_provider="",
model=model_group,
response=httpx.Response(
status_code=429,
content="",
headers={"retry-after": str(60)},
request=httpx.Request(
method="tpm_rpm_limits",
url="https://github.com/BerriAI/litellm",
),
),
)
@with_service_target("router_usage")
async def async_get_available_deployments(
self,
@ -520,63 +582,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
input=input,
)
try:
assert deployment is not None
return deployment
except Exception:
### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ###
deployment_dict: Final = {}
for index, _deployment in enumerate(healthy_deployments):
if isinstance(_deployment, dict):
id = _deployment.get("model_info", {}).get("id")
### GET DEPLOYMENT TPM LIMIT ###
_deployment_tpm = None
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("model_info", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = float("inf")
### GET CURRENT TPM ###
current_tpm = tpm_values[index] if tpm_values else 0
### GET DEPLOYMENT TPM LIMIT ###
_deployment_rpm = None
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("model_info", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = float("inf")
### GET CURRENT RPM ###
current_rpm = rpm_values[index] if rpm_values else 0
deployment_dict[id] = {
"current_tpm": current_tpm,
"tpm_limit": _deployment_tpm,
"current_rpm": current_rpm,
"rpm_limit": _deployment_rpm,
}
raise litellm.RateLimitError(
message=f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}",
llm_provider="",
model=model_group,
response=httpx.Response(
status_code=429,
content="",
headers={"retry-after": str(60)},
request=httpx.Request(
method="tpm_rpm_limits",
url="https://github.com/BerriAI/litellm",
),
),
)
if deployment is None:
raise self._no_deployments_available_error(model_group, healthy_deployments, tpm_values, rpm_values) # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage lists
return deployment
@with_service_target("router_usage")
def get_available_deployments(
@ -631,49 +639,6 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
input=input,
)
try:
assert deployment is not None
return deployment
except Exception:
### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ###
deployment_dict: Final = {}
for index, _deployment in enumerate(healthy_deployments):
if isinstance(_deployment, dict):
id = _deployment.get("model_info", {}).get("id")
### GET DEPLOYMENT TPM LIMIT ###
_deployment_tpm = None
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("model_info", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = float("inf")
### GET CURRENT TPM ###
current_tpm = tpm_values[index] if tpm_values else 0
### GET DEPLOYMENT TPM LIMIT ###
_deployment_rpm = None
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("model_info", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = float("inf")
### GET CURRENT RPM ###
current_rpm = rpm_values[index] if rpm_values else 0
deployment_dict[id] = {
"current_tpm": current_tpm,
"tpm_limit": _deployment_tpm,
"current_rpm": current_rpm,
"rpm_limit": _deployment_rpm,
}
raise ValueError(
f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}"
)
if deployment is None:
raise self._no_deployments_available_error(model_group, healthy_deployments, tpm_values, rpm_values) # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage lists
return deployment

View file

@ -17,6 +17,7 @@ CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
"_acompletion": "/v1/chat/completions",
"text_completion": "/v1/completions",
"atext_completion": "/v1/completions",
"_text_completion": "/v1/completions",
"_atext_completion": "/v1/completions",
"anthropic_messages": "/v1/messages",
"aresponses": "/v1/responses",

View file

@ -1013,3 +1013,15 @@ _check_non_standard_fallback_format = check_non_standard_fallback_format
def run_non_standard_fallback_format(fallbacks: Sequence[str] | Sequence[Mapping[str, object]], model_group: str):
pass
def non_stream_fallback_answer_as_chunk(fallback_response: object) -> object:
"""A mid-stream fallback that answered without a stream: the whole response as the one stream
chunk it is, as a mocked stream sends a full response (`convert_model_response_to_streaming`).
Anything that is not a chat response is yielded as it is."""
from litellm.llms.base_llm.base_model_iterator import convert_model_response_to_streaming
from litellm.types.utils import ModelResponse
if isinstance(fallback_response, ModelResponse):
return convert_model_response_to_streaming(fallback_response)
return fallback_response

View file

@ -632,11 +632,10 @@ def test_select_deployment_sync_returns_none_without_selector():
assert result is None
def test_select_deployment_sync_does_not_dispatch_cost_based_routing():
def test_select_deployment_sync_dispatches_cost_based_routing():
"""
`LowestCostLoggingHandler` only implements `async_get_available_deployments`,
so the sync dispatch must fall through to None for `cost-based-routing`
instead of raising AttributeError.
`LowestCostLoggingHandler` has a sync `get_available_deployments` as well, so the sync dispatch
picks with it for `cost-based-routing` as the async dispatch does.
"""
router = _build_router(
routing_strategy="simple-shuffle",
@ -649,21 +648,19 @@ def test_select_deployment_sync_does_not_dispatch_cost_based_routing():
],
)
cost_selector = router._group_selectors["cheap"]["cost-based-routing"]
assert not hasattr(cost_selector, "get_available_deployments"), (
"test premise broken: LowestCostLoggingHandler unexpectedly grew a "
"sync method — revisit the sync dispatch arm."
)
healthy = router._core_get_model_list(model_name="filtered-model") # pyright: ignore[reportPrivateUsage] # pytest and the cores' untyped members
assert healthy
result = router._select_deployment_sync(
strategy="cost-based-routing",
selector=cost_selector,
model="filtered-model",
healthy_deployments=[],
healthy_deployments=healthy,
messages=None,
input=None,
request_kwargs=None,
)
assert result is None
assert result in healthy
@pytest.mark.asyncio
@ -1036,7 +1033,7 @@ def test_sync_usage_based_v2_override_stays_scoped_to_the_request_that_asked_for
plain = router.completion(model="other-model", messages=messages, mock_response="ok")
assert plain.choices[0].message.content == "ok"
with pytest.raises(ValueError, match="No deployments available"):
with pytest.raises(litellm.RateLimitError, match="No deployments available"): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
router.completion(
model="other-model", messages=messages, mock_response="ok", routing_strategy="usage-based-routing-v2"
)

View file

@ -3448,7 +3448,7 @@ WILDCARD_COOLED: Final = (FIRST_WILDCARD, SECOND_WILDCARD)
("simple-shuffle", EVERY_DEPLOYMENT, "async", "refused", None, ALL_COOLED),
("cost-based-routing", STRATEGY_COOLED, "sync", "refused", "team-a", None),
("cost-based-routing", STRATEGY_COOLED, "async", "refused", "team-a", None),
("cost-based-routing", STRATEGY_COOLED, "pass-through", "refused", "team-a", None),
("cost-based-routing", STRATEGY_COOLED, "pass-through-chat", "refused", "team-a", None),
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED),
("simple-shuffle", EVERY_DEPLOYMENT, "async", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED),
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", (ALIAS_TARGET,)),

View file

@ -263,24 +263,32 @@ def test_request_timeout_is_unset_without_a_router_timeout(
assert _one(make_core).request_timeout is None # pyright: ignore[reportAttributeAccessIssue]
def test_request_timeout_reads_litellm_request_timeout_directly_when_a_router_timeout_is_set(
def test_request_timeout_reads_the_host_request_timeout_when_a_router_timeout_is_set(
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Oddity: this one setting is read from ``litellm.request_timeout`` itself, not from the host's
process settings (which say 33 here)."""
"""Read from the host's process settings (33 here), not from ``litellm.request_timeout`` itself."""
host.set_dynamic(request_timeout=33.0)
monkeypatch.setattr(litellm, "request_timeout", 77.0)
assert _one(make_core, timeout=10).request_timeout == 77.0 # pyright: ignore[reportAttributeAccessIssue]
assert _one(make_core, timeout=10).request_timeout == 33.0 # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
def test_request_timeout_is_unset_when_litellm_request_timeout_is_its_untouched_default(
make_core: MakeCore, monkeypatch: pytest.MonkeyPatch
def test_request_timeout_is_unset_when_the_host_request_timeout_is_its_untouched_default(
make_core: MakeCore,
host: ScriptedHost,
monkeypatch: pytest.MonkeyPatch, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members
) -> None:
monkeypatch.setattr(litellm, "request_timeout", 6000)
monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False, raising=False)
host.set_dynamic(request_timeout=6000.0, request_timeout_explicitly_set=False)
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
assert _one(make_core, timeout=10).request_timeout is None # pyright: ignore[reportAttributeAccessIssue]
def test_request_timeout_keeps_the_default_value_when_the_host_marks_it_explicitly_set(
make_core: MakeCore, host: ScriptedHost
) -> None:
host.set_dynamic(request_timeout=6000.0, request_timeout_explicitly_set=True)
assert _one(make_core, timeout=10).request_timeout == 6000.0 # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
def test_stream_timeout_is_kept_as_given(make_core: MakeCore) -> None:
assert _one(make_core, stream_timeout=3).stream_timeout == 3 # pyright: ignore[reportAttributeAccessIssue]

View file

@ -288,9 +288,9 @@ async def test_generic_call_sends_the_declared_custom_llm_provider(make_core: Ma
@pytest.mark.parametrize(
("endpoint", "expected"),
[("aocr", 600.0), ("aresponses", 600.0), ("ocr", 6000.0), ("generate_content", 6000.0)],
[("aocr", 600.0), ("aresponses", 600.0), ("ocr", 600.0), ("generate_content", 600.0), ("responses", 600.0)],
)
async def test_generic_default_timeout_is_the_passthrough_default_for_async_and_the_process_setting_for_sync(
async def test_generic_default_timeout_is_the_passthrough_default_for_sync_and_async(
make_core: MakeCore, host: EndpointHost, endpoint: str, expected: float
):
core: Final = make_core(model_list=[deployment("g", "d1")])
@ -306,6 +306,20 @@ async def test_generic_timeout_takes_the_router_timeout(make_core: MakeCore, hos
assert host.calls[0].kwargs["timeout"] == 42
@pytest.mark.parametrize("endpoint", ["aocr", "ocr"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
async def test_generic_timeout_without_a_router_timeout_takes_the_host_request_timeout(
make_core: MakeCore,
host: EndpointHost,
endpoint: str,
monkeypatch: pytest.MonkeyPatch, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members
):
host.set_dynamic(request_timeout=33.0)
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
core: Final = make_core(model_list=[deployment("g", "d1")])
await _call(core, endpoint, {"model": "g", "document": {}})
assert host.calls[0].kwargs["timeout"] == 33.0
@pytest.mark.parametrize(
("endpoint", "request_body", "expected"),
[

View file

@ -171,25 +171,17 @@ async def test_cache_responses_turned_on_after_construction_reaches_the_host(
# --- sync text completion and image generation ----------------------------------------------
async def test_sync_text_completion_sends_the_router_num_retries_and_no_routing_fields(
make_core: MakeCore, host: EndpointHost
):
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
async def test_sync_text_completion_shapes_the_request_like_atext_completion(make_core: MakeCore, host: EndpointHost):
core: Final = make_core(model_list=[deployment("g", "d1", timeout=7)], num_retries=4)
core.text_completion(model="g", prompt="p")
sent: Final = _sent(host)
assert sent["num_retries"] == 4
assert "model_info" not in sent
assert "litellm_trace_id" not in sent
assert sent.get("timeout") is None
assert "num_retries" not in sent
assert sent["model_info"]["id"] == "d1" # pyright: ignore[reportIndexIssue]
trace_id: Final = sent["litellm_trace_id"]
assert isinstance(trace_id, str) and len(trace_id) == 36
assert sent["timeout"] == 7
metadata: Final = _bucket(sent)
assert metadata["model_group"] == "g"
assert "deployment" not in metadata
async def test_sync_text_completion_request_num_retries_wins(make_core: MakeCore, host: EndpointHost):
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
core.text_completion(model="g", prompt="p", num_retries=1)
assert _sent(host)["num_retries"] == 1
assert (metadata["model_group"], metadata["deployment"]) == ("g", "openai/d1")
async def test_sync_text_completion_merges_default_metadata_into_request_metadata(
@ -204,23 +196,36 @@ async def test_sync_text_completion_merges_default_metadata_into_request_metadat
assert (metadata["x"], metadata["y"], metadata["model_group"], sent["user"]) == (1, 2, "g", "u")
async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model(
make_core: MakeCore, host: EndpointHost
):
"""The request's model="g" is spread after the deployment params, so it replaces "openai/d1"."""
async def test_sync_text_completion_sends_the_deployment_model(make_core: MakeCore, host: EndpointHost):
core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
core.text_completion(model="g", prompt="p")
call: Final = host.calls[0]
assert (call.model, call.kwargs["api_base"]) == ("g", "https://d1.example")
assert (call.model, call.kwargs["api_base"]) == ("openai/d1", "https://d1.example")
async def test_sync_text_completion_is_not_retried(make_core: MakeCore, host: EndpointHost):
host.answer("g", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
async def test_sync_text_completion_is_retried_with_the_router_num_retries(make_core: MakeCore, host: EndpointHost):
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3)
with pytest.raises(litellm.InternalServerError):
core.text_completion(model="g", prompt="p")
assert len(host.calls) == 1
assert host.sleeps == []
assert len(host.calls) == 4
async def test_sync_text_completion_request_num_retries_wins(make_core: MakeCore, host: EndpointHost):
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
with pytest.raises(litellm.InternalServerError): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
core.text_completion(model="g", prompt="p", num_retries=1) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # pytest and the cores' untyped members
assert len(host.calls) == 2
async def test_sync_text_completion_falls_back_like_atext_completion(make_core: MakeCore, host: EndpointHost):
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
core: Final = make_core(
model_list=[deployment("g", "d1"), deployment("h", "d2")], num_retries=0, fallbacks=[{"g": ["h"]}]
)
core.text_completion(model="g", prompt="p") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # pytest and the cores' untyped members
assert host.deployments_called() == ["d1", "d2"]
async def test_sync_image_generation_sends_routing_metadata_without_a_trace_id(make_core: MakeCore, host: EndpointHost):

View file

@ -364,11 +364,13 @@ async def test_an_override_matching_a_routing_group_strategy_is_not_bound_to_the
def test_a_sync_usage_based_v2_override_counts_rpm_before_the_call(make_core: MakeCore, host: ScriptedHost) -> None:
"""The override selector is on no callback list, so the sync pick runs its pre-call check itself:
the first call is counted against ``rpm=1`` and the next pick finds no deployment. Oddity: the sync
path raises a bare ``ValueError`` where the async one raises ``RateLimitError``."""
the first call is counted against ``rpm=1`` and the next pick finds no deployment, raising the same
``RateLimitError`` as the async path."""
core: Final = make_core(model_list=[deployment("g", "a", rpm=1)], num_retries=0)
core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue]
with pytest.raises(ValueError, match=re.escape("No deployments available for selected model. Passed model=g.")):
with pytest.raises( # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
litellm.RateLimitError, match=re.escape("No deployments available for selected model. Passed model=g.")
):
core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue]
assert host.deployments_called() == ["a"]
@ -424,22 +426,31 @@ def test_sync_completion_under_latency_based_routing_reaches_the_group(make_core
assert len(host.calls) == 10
def test_sync_completion_under_cost_based_routing_finds_no_deployment(make_core: MakeCore, host: ScriptedHost) -> None:
"""Oddity: the cost-based selector has no sync pick, so every sync pick under cost-based routing
rate-limits without calling a provider."""
def test_sync_completion_under_cost_based_routing_picks_the_cheapest_deployment(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""The cost-based selector picks on the sync path as it does on the async one."""
core: Final = make_core(
model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="cost-based-routing", num_retries=0
model_list=[
deployment("g", "a", input_cost_per_token=0.5, output_cost_per_token=0.5),
deployment("g", "b", input_cost_per_token=0.1, output_cost_per_token=0.1),
],
routing_strategy="cost-based-routing",
num_retries=0,
)
with pytest.raises(RouterRateLimitError) as raised:
for _ in range(3):
core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g")
assert host.calls == []
assert host.deployments_called() == ["b", "b", "b"]
@pytest.mark.asyncio
async def test_pass_through_pick_under_cost_based_routing_finds_no_deployment(make_core: MakeCore) -> None:
async def test_pass_through_pick_under_cost_based_routing_picks_the_cheapest_deployment(make_core: MakeCore) -> None:
core: Final = make_core(
model_list=[deployment("g", "a", use_in_pass_through=True)], routing_strategy="cost-based-routing"
model_list=[
deployment("g", "a", use_in_pass_through=True, input_cost_per_token=0.5, output_cost_per_token=0.5),
deployment("g", "b", use_in_pass_through=True, input_cost_per_token=0.1, output_cost_per_token=0.1),
],
routing_strategy="cost-based-routing",
)
with pytest.raises(RouterRateLimitError):
await core.select("g", kind="pass_through")
picked: Final = await core.select("g", kind="pass_through") # pyright: ignore[reportCallIssue, reportUnknownVariableType] # pytest and the cores' untyped members
assert picked["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue, reportCallIssue, reportArgumentType]

View file

@ -328,9 +328,10 @@ async def test_a_per_user_github_copilot_deployment_rejects_another_credential_n
async def test_the_process_request_timeout_outranks_the_router_timeout_once_a_router_timeout_is_set(
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Oddity: this per-attempt timeout is read from ``litellm.request_timeout`` itself (not the host's
process settings), and only when the router has a ``timeout`` of its own."""
monkeypatch.setattr(litellm, "request_timeout", 33.0)
"""This per-attempt timeout is read from the host's process settings (not ``litellm.request_timeout``
itself), and only when the router has a ``timeout`` of its own."""
host.set_dynamic(request_timeout=33.0)
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42)
await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
assert host.calls[0].kwargs["timeout"] == 33.0
@ -341,7 +342,8 @@ async def test_the_process_request_timeout_is_the_stream_timeout_without_a_strea
) -> None:
from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts
monkeypatch.setattr(litellm, "request_timeout", 33.0)
host.set_dynamic(request_timeout=33.0)
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
host.answer("openai/d1", chat_stream("A"))
core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42)
await collect_texts(await core.acompletion(model="g", messages=MESSAGES, stream=True)) # pyright: ignore[reportAttributeAccessIssue]
@ -424,8 +426,18 @@ async def test_acompletion_counts_total_success_and_failed_calls_per_deployment_
assert counts == (2, 1, 1)
def test_sync_completion_does_not_count_calls(make_core: MakeCore, host: ScriptedHost) -> None:
"""Oddity: only the async path updates ``total_calls``/``success_calls``/``fail_calls``."""
core: Final = make_core(model_list=[deployment("g", "d1")])
def test_completion_counts_total_success_and_failed_calls_per_deployment_model(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""The sync path counts calls as ``acompletion`` does."""
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0)
core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
assert core.total_calls["openai/d1"] == 0 # pyright: ignore[reportAttributeAccessIssue]
host.answer("openai/d1", rate_limit())
with pytest.raises(litellm.RateLimitError): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
counts: Final = ( # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members
core.total_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
core.success_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
core.fail_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
)
assert counts == (2, 1, 1)

View file

@ -61,33 +61,34 @@ def stream_with_usage(sync: bool = False) -> Callable[..., object]:
# --- a fallback that answers without a stream --------------------------------------------------
async def test_a_mid_stream_fallback_that_answers_without_a_stream_yields_one_none_item(
async def test_a_mid_stream_fallback_that_answers_without_a_stream_yields_its_response_as_one_chunk(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Oddity: the fallback's whole response is dropped and the stream yields ``None`` in its place."""
host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True))
host.answer("openai/b", completion_response("not streamed", model="b"))
core: Final = build(make_core)
stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue]
seen: list[object] = []
await drain_into(stream, seen)
assert seen[-1] is None
last: Final = seen[-1]
assert isinstance(last, ModelResponseStream)
assert (last.model, last.choices[0].delta.content) == ("b", "not streamed") # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
assert host.deployments_called() == ["a", "b"]
def test_a_sync_mid_stream_fallback_that_answers_without_a_stream_yields_the_response_fields(
def test_a_sync_mid_stream_fallback_that_answers_without_a_stream_yields_its_response_as_one_chunk(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Oddity: a ``ModelResponse`` is iterable (pydantic yields ``(field, value)`` pairs), so the sync
wrapper streams the fallback response's fields instead of chunks."""
"""The sync wrapper yields the same one chunk as the async one, not the response's pydantic fields."""
host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True, sync=True))
host.answer("openai/b", completion_response("not streamed", model="b"))
core: Final = build(make_core)
stream: Final = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue]
items: Final = list(stream) # pyright: ignore[reportArgumentType]
fields: Final = [item for item in items if isinstance(item, tuple)]
assert ("model", "b") in fields
assert {name for name, _ in fields} >= {"id", "choices", "usage"}
assert not any(isinstance(item, tuple) for item in items) # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members
last: Final = items[-1] # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members
assert isinstance(last, ModelResponseStream)
assert (last.model, last.choices[0].delta.content) == ("b", "not streamed") # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
assert host.deployments_called() == ["a", "b"]

View file

@ -8,6 +8,7 @@ Python does them today and called out in the test's docstring.
"""
import datetime
import inspect
import io
from collections.abc import Callable, Mapping
from types import SimpleNamespace
@ -73,6 +74,10 @@ ROUTED_ENDPOINTS: Final[dict[str, dict[str, object]]] = {
"acancel_batch": {"batch_id": "b1"},
}
SYNC_COUNTED: Final[dict[str, dict[str, object]]] = {
"completion": {"messages": MESSAGES},
"embedding": {"input": ["a"]},
}
# --- batch completions (5565-5790) ----------------------------------------------------------
@ -320,17 +325,21 @@ async def test_image_generation_for_an_unknown_group_raises_without_calling_the_
("aembedding", "openai/d1", "openai/d1"),
("acreate_batch", "openai/d1", "g"),
("acancel_batch", "openai/d1", "g"),
("image_generation", "", ""),
("image_generation", "openai/d1", "openai/d1"),
("text_completion", "openai/d1", "g"),
("completion", "openai/d1", "openai/d1"),
("embedding", "openai/d1", "openai/d1"),
],
)
async def test_call_counters_are_keyed_differently_per_endpoint(
async def test_call_counters_are_keyed_per_endpoint_and_the_same_for_sync_and_async(
make_core: MakeCore, host: EndpointHost, endpoint: str, total_key: str, fail_key: str
):
"""total_calls/fail_calls are keyed by the deployment model for some endpoints and by the model group
for others (fail_calls even differs from total_calls for some); sync image_generation counts under ""."""
for others (fail_calls even differs from total_calls for some); a sync endpoint counts as its async
twin does."""
host.answer("*", rate_limit())
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=1)
request: Final = ROUTED_ENDPOINTS.get(endpoint, {"prompt": "p"})
request: Final = SYNC_COUNTED.get(endpoint) or ROUTED_ENDPOINTS.get(endpoint, {"prompt": "p"})
with pytest.raises(litellm.RateLimitError):
await core.call(endpoint, {"model": "g", **request})
assert dict(core.total_calls) == {total_key: 2} # pyright: ignore[reportAttributeAccessIssue] # a public counter
@ -399,16 +408,22 @@ async def test_async_passthrough_rewrites_only_whole_path_segments_naming_the_gr
assert host.calls[0].kwargs["endpoint"] == "/model/us.anthropic.claude/invoke/gg/us.anthropic.claude:stream"
async def test_sync_generic_call_replaces_every_substring_of_the_group_name_in_the_endpoint(
make_core: MakeCore, monkeypatch: pytest.MonkeyPatch, host: EndpointHost
@pytest.mark.parametrize("endpoint", ["aresponses", "responses"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
async def test_generic_call_rewrites_only_whole_path_segments_naming_the_group(
make_core: MakeCore,
monkeypatch: pytest.MonkeyPatch,
host: EndpointHost,
endpoint: str, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members
):
"""The sync path uses str.replace with the full deployment model (provider prefix kept), so a group
name inside other path segments is rewritten too, unlike the async path's segment-bounded rewrite."""
"""Sync and async both swap the group name for the provider-stripped deployment model one whole path
segment at a time, so a segment that merely contains the group name ("gg") is left alone."""
patch_direct_endpoints(monkeypatch, host)
core: Final = make_core(model_list=[deployment("g", "d1")])
core.responses(model="g", input="hi", endpoint="/gg/g")
sent: Final = getattr(core, endpoint)(model="g", input="hi", endpoint="/gg/g") # pyright: ignore[reportAny] # pytest and the cores' untyped members
if inspect.isawaitable(sent): # pyright: ignore[reportAny] # pytest and the cores' untyped members
await sent
call: Final = host.calls[0]
assert (call.model, call.kwargs["endpoint"]) == ("openai/d1", "/openai/d1openai/d1/openai/d1")
assert (call.model, call.kwargs["endpoint"]) == ("openai/d1", "/gg/d1")
# --- batches (8112-8362) --------------------------------------------------------------------

View file

@ -100,10 +100,10 @@ async def test_a_passing_pre_call_check_sees_the_deployment_that_is_then_called(
assert gate.seen == [("async", host.deployments_called("acompletion")[0])]
def test_a_rate_limit_from_a_sync_pre_call_check_is_raised_without_a_cooldown(
def test_a_rate_limit_from_a_sync_pre_call_check_cools_the_deployment_down_and_is_raised(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Oddity: the sync twin raises the gate's error but, unlike the async path, sets no cooldown."""
"""The sync twin cools the refused deployment down as the async path does."""
core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33)
gate: Final = gated(host, rate_limit(message="gate says no"))
@ -111,7 +111,23 @@ def test_a_rate_limit_from_a_sync_pre_call_check_is_raised_without_a_cooldown(
assert isinstance(error, litellm.RateLimitError)
assert host.calls == []
assert [kind for kind, _ in gate.seen] == ["sync"]
((kind, checked),) = gate.seen
assert kind == "sync"
cooled: Final = cooled_down(core, ["a", "b"])
assert list(cooled) == [checked]
assert cooled[checked]["cooldown_time"] == 33
def test_any_other_error_from_a_sync_pre_call_check_is_raised_without_a_cooldown(
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33)
gated(host, ValueError("gate broke"))
error: Final = attempt_sync(core)
assert type(error) is ValueError
assert host.calls == []
assert cooled_down(core, ["a", "b"]) == {}

View file

@ -0,0 +1,105 @@
"""
Settings and configs both cores must treat as ``PythonCore`` does (oddities included): count and
timeout settings kept as given, ``default_fallbacks=[]``, malformed model lists that fail at
construction, and deprecated registry members called with a ``None`` name.
"""
import warnings
from typing import Final
import pytest
from pydantic import ValidationError
from litellm.router import Router
from litellm.router_core import RouterCore
from tests.unit.test_router.core.retries_health_helpers import MakeCore
from tests.unit.test_router.core.scripted_host import deployment
def as_router(core: RouterCore) -> Router:
router: Final = Router.__new__(Router)
router._attach(core) # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue, reportUnknownMemberType] # the facade's own attach, as from_config calls it
return router
def read(router: Router, name: str) -> object:
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
return getattr(router, name) # pyright: ignore[reportAny] # a deprecated attribute, as given
# --- settings kept as given -------------------------------------------------------------------------
def test_retry_after_and_an_int_timeout_stay_ints(make_core: MakeCore) -> None:
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], timeout=1))
assert (repr(read(router, "retry_after")), repr(read(router, "timeout"))) == ("0", "1")
settings: Final = router.get_settings()
assert (repr(settings["retry_after"]), repr(settings["timeout"])) == ("0", "1")
def test_update_settings_casts_timeout_and_retry_after_to_int(make_core: MakeCore) -> None:
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
router.update_settings(timeout=3.7, retry_after=2.2)
assert (repr(read(router, "timeout")), repr(read(router, "retry_after"))) == ("3", "2")
@pytest.mark.parametrize("name", ["num_retries", "allowed_fails", "max_fallbacks"])
def test_a_count_setting_that_is_no_int_is_kept_as_given(make_core: MakeCore, name: str) -> None:
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], **{name: "1e-05"}))
assert read(router, name) == "1e-05"
# --- default_fallbacks ------------------------------------------------------------------------------
def test_an_empty_default_fallbacks_adds_a_star_entry_with_no_models(make_core: MakeCore) -> None:
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], default_fallbacks=[]))
assert read(router, "fallbacks") == [{"*": None}]
# --- malformed model lists --------------------------------------------------------------------------
@pytest.mark.parametrize("model_name", [None, 5])
def test_a_model_name_that_is_no_string_fails_at_construction(make_core: MakeCore, model_name: object) -> None:
entry: Final = {"model_name": model_name, "litellm_params": {"model": "openai/gpt-4o", "api_key": "k"}}
with pytest.raises(TypeError, match="expected str instance"):
make_core(model_list=[entry])
with pytest.raises(ValidationError, match="model_name"):
make_core(model_list=[{**entry, "model_info": {"id": "x"}}])
def test_an_alias_entry_without_hidden_fails_at_construction(make_core: MakeCore) -> None:
with pytest.raises(KeyError, match="hidden"):
make_core(model_list=[deployment("gpt", "a")], model_group_alias={"al": {"model": "gpt"}})
# --- deprecated registry members with a None name -----------------------------------------------------
@pytest.mark.parametrize(
("member", "arguments", "expected"),
[
("get_model_group", (None,), None),
("resolve_served_model_name", (None, {}), None),
("get_model_from_alias", (None,), None),
("get_routable_upstream_model", (None,), None),
("get_candidate_model_ids_for_route", (None,), frozenset()),
],
)
def test_a_deprecated_member_called_with_no_name_returns_the_old_empty_result(
make_core: MakeCore, member: str, arguments: tuple[object, ...], expected: object
) -> None:
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
assert getattr(router, member)(*arguments) == expected # pyright: ignore[reportAny] # the deprecated member
@pytest.mark.asyncio
async def test_remaining_usage_of_no_group_is_empty(make_core: MakeCore) -> None:
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
assert await router.get_remaining_model_group_usage(None) == {} # pyright: ignore[reportArgumentType] # a None group, as the fuzzer passes it

View file

@ -535,13 +535,13 @@ async def test_pass_through_selection_without_pass_through_deployments_raises_ba
with pytest.raises(litellm.BadRequestError) as raised:
await core.select("g", kind="pass_through")
assert str(raised.value) == (
"litellm.BadRequestError: Model g has no deployment configured with use_in_pass_through=True. "
"Please add use_in_pass_through: true in the deployment configuration"
"litellm.BadRequestError: Model g has no deployments configured with use_in_pass_through=True. "
"Please add use_in_pass_through: true to the deployment configuration"
)
@pytest.mark.asyncio
async def test_async_pass_through_selection_words_the_missing_deployment_error_differently(
async def test_async_pass_through_selection_words_the_missing_deployment_error_the_same_way(
make_core: MakeCore,
) -> None:
core: Final = make_core(model_list=[deployment("g", "a")])

View file

@ -195,15 +195,23 @@ async def test_selecting_a_marker_by_its_id_refuses_it_naming_its_litellm_model(
assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="auto_router/complexity_router")
@pytest.mark.parametrize("entry", ["get_available_deployment", "completion"])
def test_the_sync_path_never_runs_pre_routing_and_refuses_the_marker(
make_core: MakeCore, host: ScriptedHost, entry: str
@pytest.mark.parametrize("model", ["smart", "al"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
def test_sync_get_available_deployment_runs_pre_routing_like_the_async_pick(make_core: MakeCore, model: str) -> None:
core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"})
request: Final[dict[str, object]] = {}
chosen: Final = core.get_available_deployment(model, messages=MESSAGES, request_kwargs=request) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
assert chosen["model_info"]["id"] == "s"
assert _metadata(request)["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue]
@pytest.mark.parametrize(("model", "messages", "expected"), [("smart", MESSAGES, "s"), ("al", HARD_MESSAGES, "b")]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
def test_sync_completion_routes_through_a_complexity_router_like_acompletion(
make_core: MakeCore, host: ScriptedHost, model: str, messages: list[dict[str, str]], expected: str
) -> None:
core: Final = make_core(model_list=_tiered())
with pytest.raises(litellm.BadRequestError) as raised:
_sync_entry(core, entry, "smart")
assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="smart")
assert host.calls == []
core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"})
core.completion(model=model, messages=messages) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
assert host.deployments_called() == [expected]
assert host.calls[-1].model_group == model
# --- choosing among tag-scoped markers -----------------------------------------------------------------