fix(router): make sync router paths behave like their async twins in both cores

- sync text_completion picks, shapes, counts and retries like atext_completion
- sync generic calls rewrite the endpoint path with replace_path_segment
- a non-stream fallback mid-stream yields one converted chunk on sync and async
- generic calls default to the 600s passthrough timeout on sync too; the
  process request_timeout is read through the host
- a sync pre-call RateLimitError cools the deployment down
- the sync usage-v2 rpm override raises RateLimitError
- sync calls are counted with the async twins' keys
- one pass-through no-deployments message
- cost-based routing gets a sync pick
- sync pre-routing (complexity router aliases) works like async
This commit is contained in:
narmstrong-berry 2026-11-02 02:52:28 +00:00
parent afd1e98692
commit 1412e84490
27 changed files with 610 additions and 351 deletions

View file

@ -81,7 +81,11 @@ def can_block_current_thread() -> bool:
return threading.current_thread() is threading.main_thread() and not is_event_loop_running()
def run_async_function(async_function, *args, **kwargs):
def run_async_function(
async_function: Callable[..., Awaitable[T_Retval]],
*args: object,
**kwargs: object, # kwargs-ok: forwarded to async_function as given
) -> T_Retval:
"""
Helper utility to run an async function in a sync context.
Handles the case where there is an existing event loop running.

View file

@ -26,8 +26,14 @@ def get_configured_request_timeout() -> float | None:
"""Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
import litellm
timeout: Final = float(litellm.request_timeout)
if litellm.request_timeout_explicitly_set:
return configured_request_timeout(litellm.request_timeout, litellm.request_timeout_explicitly_set)
def configured_request_timeout(request_timeout: float, explicitly_set: bool) -> float | None:
"""``get_configured_request_timeout`` for a process ``request_timeout`` read elsewhere (a router
host's process settings): the value when it was configured, else ``None``."""
timeout: Final = float(request_timeout)
if explicitly_set:
return timeout
if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
return timeout

View file

@ -58,6 +58,7 @@ class RouterDynamicConfig:
input_callback: Sequence[object]
callback_manager: "LoggingCallbackManager"
cache: "Cache | None"
request_timeout_explicitly_set: bool = False
_versions: Final = itertools.count(1)
@ -110,6 +111,7 @@ def current_dynamic_config() -> RouterDynamicConfig:
input_callback=litellm.input_callback,
callback_manager=litellm.logging_callback_manager,
cache=litellm.cache,
request_timeout_explicitly_set=litellm.request_timeout_explicitly_set,
)
@ -120,6 +122,7 @@ _LIVE_SOURCES: Final = MappingProxyType(
"router_max_fallbacks": "ROUTER_MAX_FALLBACKS",
"allowed_fails": "allowed_fails",
"request_timeout": "request_timeout",
"request_timeout_explicitly_set": "request_timeout_explicitly_set",
"fallbacks": "fallbacks",
"default_fallbacks": "default_fallbacks",
"context_window_fallbacks": "context_window_fallbacks",

View file

@ -180,7 +180,7 @@ from litellm.litellm_core_utils.reasoning_effort_capability import (
resolve_supported_reasoning_efforts,
)
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
configured_request_timeout,
get_service_tier_timeout,
)
from litellm.litellm_core_utils.routing_common_utils import (
@ -366,6 +366,7 @@ from litellm.router_utils.fallback_event_handlers import (
mid_stream_fallback_hop_kwargs,
mid_stream_fallback_snapshot_kwargs,
mid_stream_retry_kwargs,
non_stream_fallback_answer_as_chunk,
per_request_fallback_controls,
record_disable_fallbacks,
record_pre_routing_selection,
@ -699,6 +700,12 @@ _NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
# The router strategies whose pick runs the pre-routing hook (`_async_get_available_deployment`).
_PRE_ROUTING_PICK_STRATEGIES: Final = frozenset(
{"usage-based-routing-v2", "simple-shuffle", "cost-based-routing", "latency-based-routing", "least-busy"}
)
# The generic calls (sync and async) resolve their timeout with the LLM pass-through resolver.
_GENERIC_CALL_FUNCTION_NAMES: Final = frozenset({"_ageneric_api_call_with_fallbacks", "generic_api_call"})
_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
_RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
@ -2083,7 +2090,7 @@ class PythonCore(DeprecatedRouterMembers):
# Per-attempt request_timeout, independent of router_settings.timeout.
# Only stored when a router timeout is also set, since otherwise
# request_timeout already flows through self.timeout above.
self.request_timeout = get_configured_request_timeout() if timeout is not None else None
self.request_timeout = self._process_request_timeout() if timeout is not None else None
self.stream_timeout = stream_timeout
self.retry_after = retry_after
@ -2972,16 +2979,13 @@ class PythonCore(DeprecatedRouterMembers):
if selector is None:
return None
# `cost-based-routing` is intentionally omitted —
# `LowestCostLoggingHandler` only implements
# `async_get_available_deployments`
match strategy:
case "least-busy":
return selector.get_available_deployments(
model_group=model,
healthy_deployments=healthy_deployments,
)
case "usage-based-routing" | "usage-based-routing-v2":
case "usage-based-routing" | "usage-based-routing-v2" | "cost-based-routing":
return selector.get_available_deployments(
model_group=model,
healthy_deployments=healthy_deployments,
@ -3411,6 +3415,7 @@ class PythonCore(DeprecatedRouterMembers):
model_client = None
else:
model_client = potential_model_client
self.total_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit)
## only run if model group given, not model id
@ -3427,6 +3432,7 @@ class PythonCore(DeprecatedRouterMembers):
}
)
response: Final = self._host.invoke("completion")(**input_kwargs)
self.success_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
## CHECK CONTENT FILTER ERROR ##
@ -3459,6 +3465,8 @@ class PythonCore(DeprecatedRouterMembers):
return response
except Exception as e:
verbose_router_logger.info("litellm.completion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
if model_name is not None:
self.fail_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
# Set per-deployment num_retries on exception for retry logic
if deployment is not None:
self._set_deployment_num_retries_on_exception(e, deployment)
@ -3733,6 +3741,17 @@ class PythonCore(DeprecatedRouterMembers):
adopting_wrapper.adopt_fallback_response_headers(fallback_response, prepared)
return prepared
@staticmethod
def _non_stream_fallback_item(
wrapper_ref: "weakref.ref[FallbackAwareStreamWrapper]", fallback_response: object
) -> object:
"""The one item a stream yields for a fallback that answered without a stream: its
response as a stream chunk, carrying the fallback's hidden params and headers."""
prepared: Final = PythonCore._adopt_fallback_response_headers(wrapper_ref, fallback_response)
item: Final = non_stream_fallback_answer_as_chunk(fallback_response)
PythonCore._apply_fallback_hidden_params_to_item(item, prepared)
return item
@staticmethod
def _apply_fallback_hidden_params_to_item(
fallback_item: object,
@ -3949,8 +3968,7 @@ class PythonCore(DeprecatedRouterMembers):
)
yield fallback_item
else:
# If fallback returns a non-streaming response, yield None
yield None
yield PythonCore._non_stream_fallback_item(wrapper_ref, fallback_response) # pyright: ignore[reportAny] # the legacy untyped router core
except Exception as fallback_error:
# If fallback also fails, log and re-raise original error
@ -4710,7 +4728,7 @@ class PythonCore(DeprecatedRouterMembers):
initial_kwargs["messages"] = messages
router_self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs)
fallback_response = run_async_function(
router_self.async_function_with_fallbacks_common_utils,
router_self.async_function_with_fallbacks_common_utils, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
e=e.original_exception
if isinstance(e.original_exception, litellm_exceptions.ContextWindowExceededError)
else e,
@ -4724,7 +4742,7 @@ class PythonCore(DeprecatedRouterMembers):
include_fallback_errors=initial_kwargs.get("include_fallback_errors", False) is True,
)
if hasattr(fallback_response, "__iter__"):
if hasattr(fallback_response, "__iter__") and not isinstance(fallback_response, ModelResponse): # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
prepared_fallback_hidden_params = PythonCore._adopt_fallback_response_headers(
wrapper_ref, fallback_response
)
@ -4747,7 +4765,7 @@ class PythonCore(DeprecatedRouterMembers):
router_self._combine_fallback_usage(fallback_item, None)
yield fallback_item
else:
yield None
yield PythonCore._non_stream_fallback_item(wrapper_ref, fallback_response) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
except Exception as fallback_error:
verbose_router_logger.error("Fallback also failed: %s", fallback_error)
@ -4766,9 +4784,9 @@ class PythonCore(DeprecatedRouterMembers):
"stream_with_fallbacks: error closing model_response: %s",
close_err,
)
if fallback_response is not None and hasattr(fallback_response, "close"):
if fallback_response is not None and hasattr(fallback_response, "close"): # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
try:
fallback_response.close()
fallback_response.close() # pyright: ignore[reportAttributeAccessIssue] # a stream wrapper, closed when it has close()
except BaseException as close_err:
verbose_router_logger.debug(
"stream_with_fallbacks: error closing fallback_response: %s",
@ -5299,17 +5317,19 @@ class PythonCore(DeprecatedRouterMembers):
kwargs["model_info"] = model_info
if function_name == "_ageneric_api_call_with_fallbacks":
if function_name in _GENERIC_CALL_FUNCTION_NAMES:
from litellm.passthrough.timeout_utils import (
resolve_llm_passthrough_timeout,
)
# The process request_timeout is read through the host; it ranks right after the
# router timeout, as resolve_llm_passthrough_timeout ranks it.
_router_timeout: Final = (
self.request_timeout
if self.request_timeout is not None
else float(self._explicit_timeout)
if isinstance(self._explicit_timeout, (int, float))
else None
else self._process_request_timeout()
)
_router_stream_timeout: Final = (
self.stream_timeout
@ -5381,6 +5401,11 @@ class PythonCore(DeprecatedRouterMembers):
hop_kwargs, _FALLBACK_HOP_ADAPTER.validate_python(deployment)
)
def _process_request_timeout(self) -> float | None:
"""The host's explicitly configured process ``request_timeout``, else None."""
dynamic: Final = self._host.dynamic()
return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set)
def _get_async_openai_model_client(self, deployment: dict, kwargs: dict):
"""
Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment
@ -5926,7 +5951,7 @@ class PythonCore(DeprecatedRouterMembers):
raise e
def _image_generation(self, prompt: str, model: str, **kwargs):
model_name: Final = ""
model_name = model # rebind-ok: the group until a deployment is picked, as in _aimage_generation
try:
verbose_router_logger.debug("Inside _image_generation()- model: %s", model)
deployment: Final = self.get_available_deployment(
@ -5937,6 +5962,7 @@ class PythonCore(DeprecatedRouterMembers):
)
self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs)
data: Final = deployment["litellm_params"].copy()
model_name = data["model"] # pyright: ignore[reportUnknownVariableType] # rebind-ok: the picked deployment's model, as in _aimage_generation
model_client: Final = self._get_async_openai_model_client(
deployment=deployment,
@ -5958,10 +5984,10 @@ class PythonCore(DeprecatedRouterMembers):
}
)
self.success_calls[model_name] += 1
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
return response
except Exception as e:
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
if model_name is not None:
self.fail_calls[model_name] += 1
self._stamp_retry_skip_deployment_id(e, kwargs)
@ -6300,37 +6326,64 @@ class PythonCore(DeprecatedRouterMembers):
is_async: bool | None = False,
**kwargs,
):
messages: Final = [{"role": "user", "content": prompt}]
try:
kwargs["model"] = model
kwargs["prompt"] = prompt
kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries)
kwargs.setdefault("metadata", {}).update({"model_group": model})
"""The sync twin of `atext_completion`: picks, shapes, retries and falls back the same way."""
kwargs["model"] = model # rebind-ok: the request kwargs, shaped in place as atext_completion shapes them
kwargs["prompt"] = prompt # rebind-ok: same
kwargs["original_function"] = self._text_completion # rebind-ok: same
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
return self.function_with_fallbacks(**kwargs) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # the legacy untyped router core
# pick the one that is available (lowest TPM/RPM)
with endpoint_selection("text_completion", model=model):
deployment: Final = self.get_available_deployment(
model=model,
messages=messages,
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
def _text_completion(
self,
model: str,
prompt: str,
**kwargs: object, # kwargs-ok: the request kwargs, sent on to the deployment
) -> object:
try:
verbose_router_logger.debug("Inside _text_completion()- model: %s", model)
deployment: Final = self.get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy untyped router core
model=model,
messages=[{"role": "user", "content": prompt}],
specific_deployment=kwargs.pop("specific_deployment", None), # pyright: ignore[reportArgumentType] # the legacy untyped router core
request_kwargs=kwargs,
)
self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, splits_stream_timeout=True) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
data: Final = deployment["litellm_params"].copy()
for k, v in self._core_default_litellm_params.items():
if k not in kwargs: # prioritize model-specific params > default router params
kwargs[k] = v
elif k == "metadata":
kwargs[k].update(v)
pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data))
if pinned_model_id is not None:
kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults
model_name: Final = data["model"] # pyright: ignore[reportUnknownVariableType] # the legacy untyped router core
# call via litellm.completion()
return self._host.invoke("text_completion")(
**{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs}
potential_model_client: Final = self._get_client(deployment=deployment, kwargs=kwargs, client_type="sync") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy untyped router core
dynamic_api_key: Final = kwargs.get("api_key", None)
model_client: Final = ( # pyright: ignore[reportUnknownVariableType] # the legacy untyped router core
None
if dynamic_api_key is not None
and potential_model_client is not None
and dynamic_api_key != potential_model_client.api_key # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
else potential_model_client
)
self.total_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit)
self.routing_strategy_pre_call_checks(deployment=deployment) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
response: Final = self._host.invoke("text_completion")( # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
**{
**data,
"prompt": prompt,
"caching": self._core_cache_responses,
"client": model_client,
**kwargs,
}
)
self.success_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
return response
except Exception as e:
verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model, e)
if model is not None: # pyright: ignore[reportUnnecessaryComparison] # the legacy untyped router core
self.fail_calls[model] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
self._stamp_retry_skip_deployment_id(e, kwargs)
raise e
async def atext_completion(
@ -7476,8 +7529,11 @@ class PythonCore(DeprecatedRouterMembers):
# For passthrough routes, use the actual model from deployment
# and swap model name in endpoint if present
if "endpoint" in kwargs and kwargs["endpoint"]:
kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name)
self._add_deployment_model_to_endpoint_for_llm_passthrough_route(
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
model=model,
model_name=model_name, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
)
kwargs["model"] = model_name
# Perform pre-call checks for routing strategy
@ -9473,7 +9529,7 @@ class PythonCore(DeprecatedRouterMembers):
Wrapped to reduce code duplication and prevent bugs.
"""
return run_async_function(self.async_function_with_fallbacks, *args, **kwargs)
return run_async_function(self.async_function_with_fallbacks, *args, **kwargs) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
def _get_fallback_model_group_for_lookup_groups(
self,
@ -10232,11 +10288,22 @@ class PythonCore(DeprecatedRouterMembers):
- None
Raises:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits; the deployment is
cooled down first, as on the async path
"""
for _callback in self._routing_callbacks():
if isinstance(_callback, CustomLogger):
_callback.pre_call_check(deployment)
try:
_callback.pre_call_check(deployment) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
except litellm_exceptions.RateLimitError as e:
set_cooldown_deployments(
ctx=self._core_ctx,
exception_status=e.status_code,
original_exception=e,
deployment=deployment["model_info"]["id"], # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
time_to_cooldown=self._core_cooldown_time,
)
raise
async def async_routing_strategy_pre_call_checks(
self,
@ -16234,6 +16301,18 @@ class PythonCore(DeprecatedRouterMembers):
)
return registered_model_name
def _core_sync_pick_pre_routes(self) -> bool:
"""Whether the sync pick runs the pre-routing hook: a strategy router is registered and the
router strategy is one whose async pick runs the hook too (see `_async_get_available_deployment`)."""
return self._core_routing_strategy in _PRE_ROUTING_PICK_STRATEGIES and any(
(
self._core_auto_routers,
self._core_complexity_routers,
self._core_adaptive_routers,
self._core_quality_routers,
)
)
async def _core_async_pre_routing_hook(
self,
model: str,
@ -16646,6 +16725,33 @@ class PythonCore(DeprecatedRouterMembers):
"usage-based-routing-v2, cost-based-routing, latency-based-routing, least-busy), "
"or remove `plugins` from the Router config."
)
if self._core_sync_pick_pre_routes():
# The async pick's pre-routing hook (auto/complexity/adaptive/quality routers), run to
# completion from the sync pick, so a strategy router serves sync calls too.
hook_kwargs: Final[dict[str, object]] = ( # pyright: ignore[reportUnknownVariableType] # mutable-ok: the request kwargs the hook stamps
request_kwargs if request_kwargs is not None else {}
)
responses_call: Final = input is not None and messages is None
pre_routed: Final = run_async_function(
partial(
self._core_async_pre_routing_hook, # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
model=model,
request_kwargs=hook_kwargs,
messages=messages,
input=input,
specific_deployment=specific_deployment,
)
)
if pre_routed is not None:
model = pre_routed.model # rebind-ok: the tier group the strategy picked, as the async pick rebinds it
messages = pre_routed.messages # rebind-ok: same
record_pre_routing_selection(hook_kwargs, model)
self._pin_tier_params_onto_request( # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
model=model,
tier_litellm_params=pre_routed.litellm_params if pre_routed is not None else None,
request_kwargs=hook_kwargs,
responses_call=responses_call,
)
# users need to explicitly call a specific deployment, by setting `specific_deployment = True` as completion()/embedding() kwarg
# When this was no explicit we had several issues with fallbacks timing out
@ -16926,7 +17032,7 @@ class PythonCore(DeprecatedRouterMembers):
if len(pass_through_deployments) == 0:
# No deployments support pass-through
raise litellm_exceptions.BadRequestError(
message=f"Model {model} has no deployment configured with use_in_pass_through=True. Please add use_in_pass_through: true in the deployment configuration",
message=f"Model {model} has no deployments configured with use_in_pass_through=True. Please add use_in_pass_through: true to the deployment configuration",
model=model,
llm_provider="",
)

View file

@ -1033,7 +1033,7 @@ class RustCore(
# Settings the Python core keeps as plain attributes, and settings deleted with `del`.
self._plain: Final = initial_plain(config)
self._deleted: frozenset[str] = frozenset()
self._request_timeout: Final = request_timeout(config)
self._request_timeout: Final = request_timeout(config, host)
legacy: Final[dict[str, object]] = {"deployment_stats": {}} # mutable-ok: the core's own legacy attributes
self._legacy: Final = legacy
self._health_view: RustHealth | None = None
@ -2018,8 +2018,8 @@ class RustCore(
**self._routing_fields(prepared),
**pinned_video_deployment(plan, request),
**unrouted_fields(plan),
# The sync path never runs pre-routing; a custom strategy picks on both.
**({} if plan.sync and self._custom_strategy is None else {"pre_routes": True}),
# A custom strategy picks on both paths; the sync path pre-routes when it can.
**({"pre_routes": True} if not plan.sync or self._sync_pick_pre_routes() else {}),
**({"mock_failure": mock} if mock is not None else {}),
# Every target of the walk is refused.
**({"fallbacks": []} if rejected_fallback_target(request) is not None else {}),
@ -2389,6 +2389,14 @@ class RustCore(
messages,
)
def _sync_pick_pre_routes(self) -> bool:
"""Whether a sync pick asks for the ``pre_route`` step: always for a custom strategy, else
when a strategy router is registered. A router with routing plugins never pre-routes a sync
pick, so router-core refuses it (the pipeline only runs on the async path)."""
if self._custom_strategy is not None:
return True
return not self._resolved.routing_plugins and self._prerouter.has_strategy_routers()
def _custom_pick_sync(
self,
step: Mapping[str, object],
@ -2397,10 +2405,15 @@ class RustCore(
input: object = _NO_INPUT,
specific_deployment: object = _NO_INPUT,
) -> object:
"""A custom strategy's sync pick (``get_available_deployment``); without one, no pre-routing."""
"""A custom strategy's sync pick (``get_available_deployment``); without one, the async
pre-routing hook run to completion (``_pre_route``)."""
custom: Final = self._custom_strategy
if custom is None:
return None
from litellm.litellm_core_utils import asyncify
return asyncify.run_async_function( # pyright: ignore[reportUnknownMemberType] # the untyped sync bridge
partial(self._pre_route, step, request, messages, input, specific_deployment)
)
picked: Final[object] = custom.get_available_deployment( # pyright: ignore[reportUnknownMemberType] # the strategy's untyped pick
model=str(step.get("model")),
messages=request.get("messages") if messages is _NO_INPUT else messages, # pyright: ignore[reportArgumentType] # see _pre_route
@ -2665,9 +2678,6 @@ class RustCore(
message: Final = (
f"Model {model} has no deployments configured with use_in_pass_through=True. "
"Please add use_in_pass_through: true to the deployment configuration"
if pass_through == "async_pass_through"
else f"Model {model} has no deployment configured with use_in_pass_through=True. "
"Please add use_in_pass_through: true in the deployment configuration"
)
return litellm_exceptions.BadRequestError(message=message, model=model, llm_provider="")
if kind == "context_window_exceeded":
@ -2805,7 +2815,7 @@ class _Selection:
fields["specific_deployment"] = True
if probe:
fields["health_check_probe"] = True
if kind == "pick" or (kind == "sync_pick" and core._custom_strategy is not None): # pyright: ignore[reportPrivateUsage] # the selection serves its own core
if kind == "pick" or (kind == "sync_pick" and core._sync_pick_pre_routes()): # pyright: ignore[reportPrivateUsage] # the selection serves its own core
fields["pre_routes"] = True
self._specific_deployment: Final = specific_deployment
# The error a callback filter raised, which a failed selection raises.
@ -3262,15 +3272,35 @@ class _Driver:
"""A routing pre-call check refused the attempt (``async_routing_strategy_pre_call_checks``):
a rate limit cools the deployment down (``set_cooldown_deployments``); the attempt's failure
is recorded by nothing else."""
report: Final = self._refusal_report(deployment, error)
if report is not None:
await self._core._drive_cache_steps(report) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
return {**self._failed(step, deployment, call, error), "recorded": True}
def _refused_sync(
self,
step: Mapping[str, object],
deployment: Mapping[str, object],
call: "AttemptCall",
error: BaseException,
) -> Mapping[str, object]:
"""``_refused`` on the sync path (``routing_strategy_pre_call_checks``): a rate limit cools
the deployment down here too."""
report: Final = self._refusal_report(deployment, error)
if report is not None:
self._core._drive_sync_steps(report) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
return {**self._failed(step, deployment, call, error), "recorded": True}
def _refusal_report(self, deployment: Mapping[str, object], error: BaseException) -> "RouterCall | None":
"""The uncounted failure report that cools down a deployment a pre-call check rate-limited."""
import litellm.exceptions as litellm_exceptions
core: Final = self._core
if isinstance(error, litellm_exceptions.RateLimitError):
deployment_id: Final = (as_str_mapping(deployment.get("model_info")) or {}).get("id")
if isinstance(deployment_id, str):
report: Final = core._handle.report_failure(deployment_id, dict(attempt_failure(error)), uncounted=True) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
await core._drive_cache_steps(report) # pyright: ignore[reportPrivateUsage] # see above
return {**self._failed(step, deployment, call, error), "recorded": True}
if not isinstance(error, litellm_exceptions.RateLimitError):
return None
deployment_id: Final = (as_str_mapping(deployment.get("model_info")) or {}).get("id")
if not isinstance(deployment_id, str):
return None
return self._core._handle.report_failure(deployment_id, dict(attempt_failure(error)), uncounted=True) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
def _pick_kwargs(self, step: Mapping[str, object]) -> Mapping[str, object]:
"""The request kwargs the pick sees: the request with its level's retry count."""
@ -3365,8 +3395,7 @@ class _Driver:
try:
self._core._endpoints.sync_pre_call_checks(self._plan, deployment, self._request) # pyright: ignore[reportPrivateUsage] # see above
except Exception as refused: # noqa: BLE001 # handed to the Rust call loop like any attempt failure
# The sync path records no outcome for a refused pre-call check: no cooldown.
return {**self._failed(step, deployment, call, refused), "recorded": True}
return self._refused_sync(step, deployment, call, refused)
answered: Final = self._core._host.invoke(call.host_op)(**call.kwargs) # pyright: ignore[reportPrivateUsage] # a sync endpoint
if inspect.isawaitable(answered):
raise _NotSynchronous(f"the host's {call.host_op} endpoint is not synchronous")

View file

@ -70,7 +70,7 @@ Kind = Literal[ # rebind-ok: rebound as the Python router rebinds it
"search",
]
Stamp = Literal["deployment", "retry_skip", "none"] # rebind-ok: rebound as the Python router rebinds it
CountKey = Literal["deployment", "group", "empty"] # rebind-ok: rebound as the Python router rebinds it
CountKey = Literal["deployment", "group"] # rebind-ok: rebound as the Python router rebinds it
# Request keys the router reads to route the call; the deployment call never receives them.
ROUTING_KEYS: Final = frozenset(
@ -174,9 +174,6 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType(
sync=True,
splits_stream_timeout=True,
stamp="deployment",
total_key=None,
success_key=None,
fail_key=None,
named=("messages",),
),
"atext_completion": Plan(
@ -187,6 +184,15 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType(
fail_key="group",
named=("prompt",),
),
"text_completion": Plan(
"text_completion",
"text_completion",
"text",
sync=True,
splits_stream_timeout=True,
fail_key="group",
named=("prompt",),
),
# `_asearch_with_fallbacks`: the search tool name routes as the model, with no deployment
# picked; its helper picks among the router's search tools.
"asearch": Plan(
@ -208,9 +214,6 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType(
"image",
sync=True,
prepare_field=None,
total_key="empty",
success_key="empty",
fail_key="empty",
named=("prompt",),
),
"atranscription": Plan(
@ -379,7 +382,7 @@ class Counters:
@staticmethod
def _key(which: CountKey, model_name: str, group: str) -> str:
return model_name if which == "deployment" else group if which == "group" else ""
return model_name if which == "deployment" else group
def started(self, plan: Plan, call: AttemptCall) -> None:
if plan.total_key is not None:
@ -492,9 +495,7 @@ def attempt_call(
function_name="generic_api_call",
splits_stream_timeout=plan.splits_stream_timeout,
)
endpoint = kwargs.get("endpoint") # rebind-ok: rebound as the Python router rebinds it
if endpoint and isinstance(endpoint, str):
kwargs["endpoint"] = endpoint.replace(requested, model_name)
rewrite_endpoint(core, kwargs, requested, model_name)
kwargs["model"] = model_name
final = { # rebind-ok: rebound as the Python router rebinds it
**data,
@ -516,6 +517,33 @@ def attempt_call(
return AttemptCall(plan.host_op, kwargs, kwargs, requested, group)
def rewrite_endpoint(
core: "RustCore",
kwargs: dict[str, object], # mutable-ok: the attempt's own kwargs, shaped in place
requested: str,
model_name: str,
) -> None:
"""`_add_deployment_model_to_endpoint_for_llm_passthrough_route`: the requested name, as a whole
path segment of the request's endpoint, becomes the deployment's model without its provider."""
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
endpoint: Final = kwargs.get("endpoint")
if not endpoint:
return
try:
stripped, _, _, _ = core._host.invoke("get_llm_provider")( # pyright: ignore[reportPrivateUsage] # the core's host
model=model_name,
custom_llm_provider=kwargs.get("custom_llm_provider"),
api_base=kwargs.get("api_base"),
)
replacement = str(stripped) # rebind-ok: rebound as the Python router rebinds it
except Exception: # noqa: BLE001 # as the Python router: an unresolvable model is used as is
replacement = model_name # rebind-ok: rebound as the Python router rebinds it
kwargs["endpoint"] = replace_path_segment( # rebind-ok: the attempt's kwargs, shaped in place
str(endpoint), requested, replacement
)
def _generic_attempt(
core: "RustCore",
plan: Plan,
@ -526,7 +554,6 @@ def _generic_attempt(
group: str,
) -> AttemptCall:
"""`_ageneric_api_call_with_fallbacks_helper` for one deployment."""
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
from litellm.router_rust_websocket import turn_admission
socket: Final = plan.call_type == "_aresponses_websocket"
@ -540,20 +567,7 @@ def _generic_attempt(
opens_responses_socket=socket,
)
model_name: Final = str(data.get("model"))
endpoint: Final = kwargs.get("endpoint")
if endpoint:
try:
stripped, _, _, _ = core._host.invoke("get_llm_provider")( # pyright: ignore[reportPrivateUsage] # the core's host
model=model_name,
custom_llm_provider=kwargs.get("custom_llm_provider"),
api_base=kwargs.get("api_base"),
)
replacement = str(stripped) # rebind-ok: rebound as the Python router rebinds it
except Exception: # noqa: BLE001 # as the Python router: an unresolvable model is used as is
replacement = model_name # rebind-ok: rebound as the Python router rebinds it
kwargs["endpoint"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
replace_path_segment(str(endpoint), requested, replacement)
)
rewrite_endpoint(core, kwargs, requested, model_name)
provider: Final = _generic_provider(data)
final: Final = {
**without_retrieve_stream_defaults(data, kwargs),
@ -604,14 +618,14 @@ def model_client(
core: "RustCore", plan: Plan, deployment: Mapping[str, object], kwargs: Mapping[str, object]
) -> object:
"""The SDK client cached for the deployment (`_get_async_openai_model_client`; `_get_client`
with the sync keys for sync completion and embedding), dropped when the request carries
with the sync keys for sync completion, text completion and embedding), dropped when the request carries
another api key."""
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
if as_dict(deployment.get("litellm_params")).get("kubernetes_pod_discovery") is True:
return None
model_id: Final = as_dict(deployment.get("model_info")).get("id")
sync_keys: Final = plan.sync and plan.kind in ("chat", "embedding")
sync_keys: Final = plan.sync and plan.kind in ("chat", "embedding", "text")
stream: Final = "stream_" if kwargs.get("stream") is True else ""
key: Final = f"{model_id}_{stream}client" if sync_keys else f"{model_id}_{stream}async_client"
span: Final = get_parent_otel_span_from_kwargs(dict(kwargs))
@ -832,14 +846,13 @@ class RustEndpoints:
"""The public endpoint methods of one Rust core, beyond the routed call loop the core drives."""
def __init__(self, core: "RustCore", config: "RouterConfig") -> None:
from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
from litellm.router_utils.request_priority import InvalidPriority, parse_default_priority
self._core: Final = core
self.counters: Final = Counters()
explicit: Final = config.get("timeout")
self._explicit_timeout: Final = explicit
self._request_timeout: Final = get_configured_request_timeout() if explicit is not None else None
self._request_timeout: Final = self.process_request_timeout() if explicit is not None else None
self._stream_timeout: Final = config.get("stream_timeout")
# apply() flips it after construction, as on the Python core
self._cache_responses: bool = bool(config.get("cache_responses"))
@ -927,8 +940,16 @@ class RustEndpoints:
request_timeout=self._request_timeout,
stream_timeout=self._stream_timeout,
cache_responses=self._cache_responses,
process_request_timeout=self.process_request_timeout,
)
def process_request_timeout(self) -> float | None:
"""The host's explicitly configured process ``request_timeout``, else None."""
from litellm.litellm_core_utils.request_timeout_resolver import configured_request_timeout
dynamic: Final = self._core._host.dynamic() # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core
return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set)
def default_litellm_params(self, resolved: "Resolved") -> Mapping[str, object]:
"""The defaults the Python constructor completes before any call: the router ``timeout``
argument, no client retries, and the caching groups in the default metadata."""
@ -1495,30 +1516,12 @@ class RustEndpoints:
return await self._direct("amoderation", prepared)
def text_completion(self, request: Mapping[str, object]) -> object:
"""The sync `text_completion`: one pick, no retries, the group name as the model."""
from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin
kwargs: Final = {k: v for k, v in request.items() if k not in ("is_retry", "is_fallback", "is_async")}
model: Final = kwargs.get("model")
prompt: Final = kwargs.get("prompt")
"""The sync `text_completion`: routed, retried and fallen back as `atext_completion` is."""
model: Final = request.get("model")
if not isinstance(model, str):
raise TypeError("text_completion() missing 1 required positional argument: 'model'")
kwargs["num_retries"] = kwargs.get("num_retries", self._core._resolved.num_retries) # pyright: ignore[reportPrivateUsage] # see above
metadata_bucket(kwargs, "metadata").update({"model_group": model})
specific: Final = kwargs.pop("specific_deployment", None)
deployment: Final = self.select_sync(model, kwargs, specific=bool(specific))
data: Final = as_dict(deployment.get("litellm_params"))
for key, value in self._core._resolved.default_litellm_params.items(): # pyright: ignore[reportPrivateUsage] # see above
if key not in kwargs:
kwargs[key] = value
elif key == "metadata":
bucket = kwargs[key]
if isinstance(bucket, dict):
bucket.update(as_dict(value)) # pyright: ignore[reportUnknownMemberType] # the request's metadata dict
pinned: Final = deployment_model_id_pin(data)
if pinned is not None:
kwargs["model_id"] = pinned
return self._invoke("text_completion")(**{**data, "prompt": prompt, "caching": self._cache_responses, **kwargs})
kwargs: Final = {k: v for k, v in request.items() if k not in ("is_retry", "is_fallback", "is_async")}
return self._core._call_sync(PLANS["text_completion"], kwargs) # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core
def select_sync(self, model: str, request: Mapping[str, object], *, specific: bool = False) -> Mapping[str, object]:
"""Pick a deployment for ``model`` on the sync path (`get_available_deployment`)."""

View file

@ -705,6 +705,10 @@ class PreRouter:
# -- the step --------------------------------------------------------------------------------
def has_strategy_routers(self) -> bool:
"""Whether any auto, complexity, adaptive or quality router is registered."""
return self._strategies.any_registered()
async def pre_route(
self,
model: str,

View file

@ -281,13 +281,14 @@ def pick_sync(
input: object,
request_kwargs: dict[str, object] | None, # mutable-ok: see pick_async
) -> object:
"""``_select_deployment_sync``: cost-based routing has no sync pick."""
"""``_select_deployment_sync``."""
match selector:
case LeastBusyLoggingHandler() if strategy == "least-busy":
return selector.get_available_deployments(model_group=model, healthy_deployments=deployments) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts
case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() if strategy in (
case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() | LowestCostLoggingHandler() if strategy in (
"usage-based-routing",
"usage-based-routing-v2",
"cost-based-routing",
):
return selector.get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts
model_group=model,

View file

@ -184,11 +184,15 @@ def _as_dict(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, dict) else None # pyright: ignore[reportUnknownVariableType] # the argument dict, as passed
def request_timeout(config: "RouterConfig") -> float | None:
"""The per-attempt ``request_timeout``: only stored when a router timeout is also set."""
from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
def request_timeout(config: "RouterConfig", host: "PythonHost") -> float | None:
"""The per-attempt ``request_timeout``: the host's configured process ``request_timeout``, only
stored when a router timeout is also set."""
from litellm.litellm_core_utils.request_timeout_resolver import configured_request_timeout
return get_configured_request_timeout() if config.get("timeout") is not None else None
if config.get("timeout") is None:
return None
dynamic: Final = host.dynamic()
return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set)
def apply_construction_effects(config: "RouterConfig", host: "PythonHost") -> None:

View file

@ -8,7 +8,7 @@ the same precedence and the same oddities. Nothing here reads a Python router.
"""
import dataclasses
from collections.abc import Mapping, MutableMapping, Sequence
from collections.abc import Callable, Mapping, MutableMapping, Sequence
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
@ -62,6 +62,8 @@ _FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt", "anthro
_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params"
# Generic handlers whose stream timeout is split from the request timeout.
STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
# The generic calls (sync and async) resolve their timeout with the LLM pass-through resolver.
GENERIC_CALL_FUNCTION_NAMES: Final = frozenset({"_ageneric_api_call_with_fallbacks", "generic_api_call"})
@dataclasses.dataclass(frozen=True, slots=True)
@ -70,7 +72,8 @@ class Shaping:
``timeout`` is the router's ``timeout`` (its own setting, else the process ``request_timeout``),
``request_timeout`` the explicitly configured process ``request_timeout``, kept only when the
router has a timeout of its own, as the Python router keeps them."""
router has a timeout of its own, as the Python router keeps them; ``process_request_timeout``
reads the host's explicitly configured process ``request_timeout`` at call time."""
default_litellm_params: Mapping[str, object]
timeout: object
@ -78,6 +81,7 @@ class Shaping:
request_timeout: float | None
stream_timeout: object
cache_responses: bool
process_request_timeout: Callable[[], float | None] = lambda: None
def as_dict(value: object) -> dict[str, object]: # mutable-ok: a fresh dict of a request mapping
@ -532,15 +536,17 @@ def _set_timeouts(
function_name: str | None,
splits_stream_timeout: bool,
) -> None:
if function_name == "_ageneric_api_call_with_fallbacks":
if function_name in GENERIC_CALL_FUNCTION_NAMES:
from litellm.passthrough.timeout_utils import resolve_llm_passthrough_timeout
# The process request_timeout, read through the host, ranks right after the router timeout,
# as resolve_llm_passthrough_timeout ranks it.
router_timeout: Final = (
shaping.request_timeout
if shaping.request_timeout is not None
else float(shaping.explicit_timeout)
if isinstance(shaping.explicit_timeout, (int, float))
else None
else shaping.process_request_timeout()
)
router_stream_timeout: Final = (
shaping.stream_timeout

View file

@ -55,7 +55,8 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, set_hidden_params
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.types.utils import ModelResponseStream, Usage
from litellm.router_utils.fallback_event_handlers import non_stream_fallback_answer_as_chunk
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
if TYPE_CHECKING:
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
@ -617,6 +618,17 @@ def _adopt_fallback_response_headers(
return prepared
def _non_stream_fallback_item(
wrapper_ref: "weakref.ref[FallbackAwareStreamWrapper]", fallback_response: object
) -> object:
"""The one item a stream yields for a fallback that answered without a stream: its response as
a stream chunk, carrying the fallback's hidden params and headers."""
prepared: Final = _adopt_fallback_response_headers(wrapper_ref, fallback_response)
item: Final = non_stream_fallback_answer_as_chunk(fallback_response)
_apply_fallback_hidden_params_to_item(item, prepared)
return item
def _chat_wrapper_init(
wrapper: FallbackAwareStreamWrapper, stream: object, model_response: CustomStreamWrapper
) -> None:
@ -748,8 +760,7 @@ def wrap_chat_stream( # noqa: C901 # one function per Python wrapper it ports
)
yield fallback_item
else:
# A fallback that answers without a stream yields None (an oddity kept as is).
yield None
yield _non_stream_fallback_item(wrapper_ref, fallback_response)
except Exception as fallback_error:
verbose_router_logger.error("Fallback also failed: %s", fallback_error)
if isinstance(trigger, MidStreamFallbackError) and trigger.original_exception is not None:
@ -801,7 +812,7 @@ def wrap_chat_stream_sync(
trigger: Final = _fallback_trigger(e, (litellm_exceptions.ContextWindowExceededError,))
try:
fallback_response = attempt.fallback_sync(trigger) # rebind-ok: rebound as the stream advances
if hasattr(fallback_response, "__iter__"):
if hasattr(fallback_response, "__iter__") and not isinstance(fallback_response, ModelResponse):
prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) # rebind-ok: re-adopted
settled = False # rebind-ok: rebound as the stream advances
for fallback_item in cast("Iterable[object]", fallback_response): # cast-ok: checked
@ -817,7 +828,7 @@ def wrap_chat_stream_sync(
_combine_fallback_usage(fallback_item, None)
yield fallback_item
else:
yield None
yield _non_stream_fallback_item(wrapper_ref, fallback_response)
except Exception as fallback_error:
verbose_router_logger.error("Fallback also failed: %s", fallback_error)
if isinstance(trigger, MidStreamFallbackError) and trigger.original_exception is not None:

View file

@ -209,15 +209,32 @@ class LowestCostLoggingHandler(CustomLogger):
"""
Returns a deployment with the lowest cost
"""
cost_key: Final = f"cost_map:{model_group}"
request_count_dict: Final = await self.router_cache.async_get_cache(key=f"cost_map:{model_group}") or {} # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
return self._lowest_cost_deployment(request_count_dict, healthy_deployments, messages, input) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
request_count_dict: Final = await self.router_cache.async_get_cache(key=cost_key) or {}
# -----------------------
# Find lowest used model
# ----------------------
float("inf")
@with_service_target("router_usage")
def get_available_deployments( # pyright: ignore[reportUnknownParameterType] # noqa: ANN201 # the selector's untyped deployment dict, as the async twin returns it
self,
model_group: str,
healthy_deployments: list, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the router's list of deployment dicts, as the async twin takes it
messages: list[dict[str, str]] | None = None, # mutable-ok: the request's messages, as given
input: str | list | None = None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the request's input, as given
request_kwargs: dict | None = None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: unused, the same signature as the async twin
):
"""
The sync twin of `async_get_available_deployments`: the deployment with the lowest cost
"""
request_count_dict: Final = self.router_cache.get_cache(key=f"cost_map:{model_group}") or {} # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
return self._lowest_cost_deployment(request_count_dict, healthy_deployments, messages, input) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
def _lowest_cost_deployment( # pyright: ignore[reportUnknownParameterType] # noqa: ANN202 # the selector's untyped deployment dict
self,
request_count_dict: dict | None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the untyped cached usage map, filled in place as before
healthy_deployments: list, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the router's list of deployment dicts
messages: list[dict[str, str]] | None, # mutable-ok: the request's messages, as given
input: str | list | None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the request's input, as given
):
"""The cheapest healthy deployment still under its tpm/rpm limits this minute, else None."""
current_date: Final = datetime.now().strftime("%Y-%m-%d")
current_hour: Final = datetime.now().strftime("%H")
current_minute: Final = datetime.now().strftime("%M")
@ -296,8 +313,8 @@ class LowestCostLoggingHandler(CustomLogger):
verbose_router_logger.debug(
"item_cost: %s, item_tpm: %s, item_rpm: %s, model_id: %s",
item_cost,
item_tpm,
item_rpm,
item_tpm, # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage map
item_rpm, # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage map
_deployment.get("model_info", {}).get("id"),
)

View file

@ -469,6 +469,68 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
[f"{prefix}:rpm:{current_minute}" for prefix in prefixes],
)
def _no_deployments_available_error(
self,
model_group: str,
healthy_deployments: Sequence[object],
tpm_values: Sequence[object] | None,
rpm_values: Sequence[object] | None,
) -> litellm.RateLimitError:
"""The error both the sync and the async pick raise when no deployment is under its limits."""
### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ###
deployment_dict: Final = {}
for index, _deployment in enumerate(healthy_deployments):
if isinstance(_deployment, dict):
id = _deployment.get("model_info", {}).get("id")
### GET DEPLOYMENT TPM LIMIT ###
_deployment_tpm = None
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("model_info", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = float("inf")
### GET CURRENT TPM ###
current_tpm = tpm_values[index] if tpm_values else 0
### GET DEPLOYMENT TPM LIMIT ###
_deployment_rpm = None
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("model_info", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = float("inf")
### GET CURRENT RPM ###
current_rpm = rpm_values[index] if rpm_values else 0
deployment_dict[id] = {
"current_tpm": current_tpm,
"tpm_limit": _deployment_tpm,
"current_rpm": current_rpm,
"rpm_limit": _deployment_rpm,
}
return litellm.RateLimitError(
message=f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}",
llm_provider="",
model=model_group,
response=httpx.Response(
status_code=429,
content="",
headers={"retry-after": str(60)},
request=httpx.Request(
method="tpm_rpm_limits",
url="https://github.com/BerriAI/litellm",
),
),
)
@with_service_target("router_usage")
async def async_get_available_deployments(
self,
@ -520,63 +582,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
input=input,
)
try:
assert deployment is not None
return deployment
except Exception:
### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ###
deployment_dict: Final = {}
for index, _deployment in enumerate(healthy_deployments):
if isinstance(_deployment, dict):
id = _deployment.get("model_info", {}).get("id")
### GET DEPLOYMENT TPM LIMIT ###
_deployment_tpm = None
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("model_info", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = float("inf")
### GET CURRENT TPM ###
current_tpm = tpm_values[index] if tpm_values else 0
### GET DEPLOYMENT TPM LIMIT ###
_deployment_rpm = None
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("model_info", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = float("inf")
### GET CURRENT RPM ###
current_rpm = rpm_values[index] if rpm_values else 0
deployment_dict[id] = {
"current_tpm": current_tpm,
"tpm_limit": _deployment_tpm,
"current_rpm": current_rpm,
"rpm_limit": _deployment_rpm,
}
raise litellm.RateLimitError(
message=f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}",
llm_provider="",
model=model_group,
response=httpx.Response(
status_code=429,
content="",
headers={"retry-after": str(60)},
request=httpx.Request(
method="tpm_rpm_limits",
url="https://github.com/BerriAI/litellm",
),
),
)
if deployment is None:
raise self._no_deployments_available_error(model_group, healthy_deployments, tpm_values, rpm_values) # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage lists
return deployment
@with_service_target("router_usage")
def get_available_deployments(
@ -631,49 +639,6 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
input=input,
)
try:
assert deployment is not None
return deployment
except Exception:
### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ###
deployment_dict: Final = {}
for index, _deployment in enumerate(healthy_deployments):
if isinstance(_deployment, dict):
id = _deployment.get("model_info", {}).get("id")
### GET DEPLOYMENT TPM LIMIT ###
_deployment_tpm = None
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = _deployment.get("model_info", {}).get("tpm", None)
if _deployment_tpm is None:
_deployment_tpm = float("inf")
### GET CURRENT TPM ###
current_tpm = tpm_values[index] if tpm_values else 0
### GET DEPLOYMENT TPM LIMIT ###
_deployment_rpm = None
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = _deployment.get("model_info", {}).get("rpm", None)
if _deployment_rpm is None:
_deployment_rpm = float("inf")
### GET CURRENT RPM ###
current_rpm = rpm_values[index] if rpm_values else 0
deployment_dict[id] = {
"current_tpm": current_tpm,
"tpm_limit": _deployment_tpm,
"current_rpm": current_rpm,
"rpm_limit": _deployment_rpm,
}
raise ValueError(
f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}"
)
if deployment is None:
raise self._no_deployments_available_error(model_group, healthy_deployments, tpm_values, rpm_values) # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage lists
return deployment

View file

@ -17,6 +17,7 @@ CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
"_acompletion": "/v1/chat/completions",
"text_completion": "/v1/completions",
"atext_completion": "/v1/completions",
"_text_completion": "/v1/completions",
"_atext_completion": "/v1/completions",
"anthropic_messages": "/v1/messages",
"aresponses": "/v1/responses",

View file

@ -1013,3 +1013,15 @@ _check_non_standard_fallback_format = check_non_standard_fallback_format
def run_non_standard_fallback_format(fallbacks: Sequence[str] | Sequence[Mapping[str, object]], model_group: str):
pass
def non_stream_fallback_answer_as_chunk(fallback_response: object) -> object:
"""A mid-stream fallback that answered without a stream: the whole response as the one stream
chunk it is, as a mocked stream sends a full response (`convert_model_response_to_streaming`).
Anything that is not a chat response is yielded as it is."""
from litellm.llms.base_llm.base_model_iterator import convert_model_response_to_streaming
from litellm.types.utils import ModelResponse
if isinstance(fallback_response, ModelResponse):
return convert_model_response_to_streaming(fallback_response)
return fallback_response

View file

@ -632,11 +632,10 @@ def test_select_deployment_sync_returns_none_without_selector():
assert result is None
def test_select_deployment_sync_does_not_dispatch_cost_based_routing():
def test_select_deployment_sync_dispatches_cost_based_routing():
"""
`LowestCostLoggingHandler` only implements `async_get_available_deployments`,
so the sync dispatch must fall through to None for `cost-based-routing`
instead of raising AttributeError.
`LowestCostLoggingHandler` has a sync `get_available_deployments` as well, so the sync dispatch
picks with it for `cost-based-routing` as the async dispatch does.
"""
router = _build_router(
routing_strategy="simple-shuffle",
@ -649,21 +648,19 @@ def test_select_deployment_sync_does_not_dispatch_cost_based_routing():
],
)
cost_selector = router._group_selectors["cheap"]["cost-based-routing"]
assert not hasattr(cost_selector, "get_available_deployments"), (
"test premise broken: LowestCostLoggingHandler unexpectedly grew a "
"sync method — revisit the sync dispatch arm."
)
healthy = router._core_get_model_list(model_name="filtered-model") # pyright: ignore[reportPrivateUsage] # pytest and the cores' untyped members
assert healthy
result = router._select_deployment_sync(
strategy="cost-based-routing",
selector=cost_selector,
model="filtered-model",
healthy_deployments=[],
healthy_deployments=healthy,
messages=None,
input=None,
request_kwargs=None,
)
assert result is None
assert result in healthy
@pytest.mark.asyncio
@ -1036,7 +1033,7 @@ def test_sync_usage_based_v2_override_stays_scoped_to_the_request_that_asked_for
plain = router.completion(model="other-model", messages=messages, mock_response="ok")
assert plain.choices[0].message.content == "ok"
with pytest.raises(ValueError, match="No deployments available"):
with pytest.raises(litellm.RateLimitError, match="No deployments available"): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
router.completion(
model="other-model", messages=messages, mock_response="ok", routing_strategy="usage-based-routing-v2"
)

View file

@ -3448,7 +3448,7 @@ WILDCARD_COOLED: Final = (FIRST_WILDCARD, SECOND_WILDCARD)
("simple-shuffle", EVERY_DEPLOYMENT, "async", "refused", None, ALL_COOLED),
("cost-based-routing", STRATEGY_COOLED, "sync", "refused", "team-a", None),
("cost-based-routing", STRATEGY_COOLED, "async", "refused", "team-a", None),
("cost-based-routing", STRATEGY_COOLED, "pass-through", "refused", "team-a", None),
("cost-based-routing", STRATEGY_COOLED, "pass-through-chat", "refused", "team-a", None),
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED),
("simple-shuffle", EVERY_DEPLOYMENT, "async", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED),
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", (ALIAS_TARGET,)),

View file

@ -263,24 +263,32 @@ def test_request_timeout_is_unset_without_a_router_timeout(
assert _one(make_core).request_timeout is None # pyright: ignore[reportAttributeAccessIssue]
def test_request_timeout_reads_litellm_request_timeout_directly_when_a_router_timeout_is_set(
def test_request_timeout_reads_the_host_request_timeout_when_a_router_timeout_is_set(
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Oddity: this one setting is read from ``litellm.request_timeout`` itself, not from the host's
process settings (which say 33 here)."""
"""Read from the host's process settings (33 here), not from ``litellm.request_timeout`` itself."""
host.set_dynamic(request_timeout=33.0)
monkeypatch.setattr(litellm, "request_timeout", 77.0)
assert _one(make_core, timeout=10).request_timeout == 77.0 # pyright: ignore[reportAttributeAccessIssue]
assert _one(make_core, timeout=10).request_timeout == 33.0 # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
def test_request_timeout_is_unset_when_litellm_request_timeout_is_its_untouched_default(
make_core: MakeCore, monkeypatch: pytest.MonkeyPatch
def test_request_timeout_is_unset_when_the_host_request_timeout_is_its_untouched_default(
make_core: MakeCore,
host: ScriptedHost,
monkeypatch: pytest.MonkeyPatch, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members
) -> None:
monkeypatch.setattr(litellm, "request_timeout", 6000)
monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False, raising=False)
host.set_dynamic(request_timeout=6000.0, request_timeout_explicitly_set=False)
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
assert _one(make_core, timeout=10).request_timeout is None # pyright: ignore[reportAttributeAccessIssue]
def test_request_timeout_keeps_the_default_value_when_the_host_marks_it_explicitly_set(
make_core: MakeCore, host: ScriptedHost
) -> None:
host.set_dynamic(request_timeout=6000.0, request_timeout_explicitly_set=True)
assert _one(make_core, timeout=10).request_timeout == 6000.0 # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
def test_stream_timeout_is_kept_as_given(make_core: MakeCore) -> None:
assert _one(make_core, stream_timeout=3).stream_timeout == 3 # pyright: ignore[reportAttributeAccessIssue]

View file

@ -288,9 +288,9 @@ async def test_generic_call_sends_the_declared_custom_llm_provider(make_core: Ma
@pytest.mark.parametrize(
("endpoint", "expected"),
[("aocr", 600.0), ("aresponses", 600.0), ("ocr", 6000.0), ("generate_content", 6000.0)],
[("aocr", 600.0), ("aresponses", 600.0), ("ocr", 600.0), ("generate_content", 600.0), ("responses", 600.0)],
)
async def test_generic_default_timeout_is_the_passthrough_default_for_async_and_the_process_setting_for_sync(
async def test_generic_default_timeout_is_the_passthrough_default_for_sync_and_async(
make_core: MakeCore, host: EndpointHost, endpoint: str, expected: float
):
core: Final = make_core(model_list=[deployment("g", "d1")])
@ -306,6 +306,20 @@ async def test_generic_timeout_takes_the_router_timeout(make_core: MakeCore, hos
assert host.calls[0].kwargs["timeout"] == 42
@pytest.mark.parametrize("endpoint", ["aocr", "ocr"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
async def test_generic_timeout_without_a_router_timeout_takes_the_host_request_timeout(
make_core: MakeCore,
host: EndpointHost,
endpoint: str,
monkeypatch: pytest.MonkeyPatch, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members
):
host.set_dynamic(request_timeout=33.0)
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
core: Final = make_core(model_list=[deployment("g", "d1")])
await _call(core, endpoint, {"model": "g", "document": {}})
assert host.calls[0].kwargs["timeout"] == 33.0
@pytest.mark.parametrize(
("endpoint", "request_body", "expected"),
[

View file

@ -171,25 +171,17 @@ async def test_cache_responses_turned_on_after_construction_reaches_the_host(
# --- sync text completion and image generation ----------------------------------------------
async def test_sync_text_completion_sends_the_router_num_retries_and_no_routing_fields(
make_core: MakeCore, host: EndpointHost
):
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
async def test_sync_text_completion_shapes_the_request_like_atext_completion(make_core: MakeCore, host: EndpointHost):
core: Final = make_core(model_list=[deployment("g", "d1", timeout=7)], num_retries=4)
core.text_completion(model="g", prompt="p")
sent: Final = _sent(host)
assert sent["num_retries"] == 4
assert "model_info" not in sent
assert "litellm_trace_id" not in sent
assert sent.get("timeout") is None
assert "num_retries" not in sent
assert sent["model_info"]["id"] == "d1" # pyright: ignore[reportIndexIssue]
trace_id: Final = sent["litellm_trace_id"]
assert isinstance(trace_id, str) and len(trace_id) == 36
assert sent["timeout"] == 7
metadata: Final = _bucket(sent)
assert metadata["model_group"] == "g"
assert "deployment" not in metadata
async def test_sync_text_completion_request_num_retries_wins(make_core: MakeCore, host: EndpointHost):
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
core.text_completion(model="g", prompt="p", num_retries=1)
assert _sent(host)["num_retries"] == 1
assert (metadata["model_group"], metadata["deployment"]) == ("g", "openai/d1")
async def test_sync_text_completion_merges_default_metadata_into_request_metadata(
@ -204,23 +196,36 @@ async def test_sync_text_completion_merges_default_metadata_into_request_metadat
assert (metadata["x"], metadata["y"], metadata["model_group"], sent["user"]) == (1, 2, "g", "u")
async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model(
make_core: MakeCore, host: EndpointHost
):
"""The request's model="g" is spread after the deployment params, so it replaces "openai/d1"."""
async def test_sync_text_completion_sends_the_deployment_model(make_core: MakeCore, host: EndpointHost):
core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
core.text_completion(model="g", prompt="p")
call: Final = host.calls[0]
assert (call.model, call.kwargs["api_base"]) == ("g", "https://d1.example")
assert (call.model, call.kwargs["api_base"]) == ("openai/d1", "https://d1.example")
async def test_sync_text_completion_is_not_retried(make_core: MakeCore, host: EndpointHost):
host.answer("g", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
async def test_sync_text_completion_is_retried_with_the_router_num_retries(make_core: MakeCore, host: EndpointHost):
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3)
with pytest.raises(litellm.InternalServerError):
core.text_completion(model="g", prompt="p")
assert len(host.calls) == 1
assert host.sleeps == []
assert len(host.calls) == 4
async def test_sync_text_completion_request_num_retries_wins(make_core: MakeCore, host: EndpointHost):
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
with pytest.raises(litellm.InternalServerError): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
core.text_completion(model="g", prompt="p", num_retries=1) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # pytest and the cores' untyped members
assert len(host.calls) == 2
async def test_sync_text_completion_falls_back_like_atext_completion(make_core: MakeCore, host: EndpointHost):
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
core: Final = make_core(
model_list=[deployment("g", "d1"), deployment("h", "d2")], num_retries=0, fallbacks=[{"g": ["h"]}]
)
core.text_completion(model="g", prompt="p") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # pytest and the cores' untyped members
assert host.deployments_called() == ["d1", "d2"]
async def test_sync_image_generation_sends_routing_metadata_without_a_trace_id(make_core: MakeCore, host: EndpointHost):

View file

@ -364,11 +364,13 @@ async def test_an_override_matching_a_routing_group_strategy_is_not_bound_to_the
def test_a_sync_usage_based_v2_override_counts_rpm_before_the_call(make_core: MakeCore, host: ScriptedHost) -> None:
"""The override selector is on no callback list, so the sync pick runs its pre-call check itself:
the first call is counted against ``rpm=1`` and the next pick finds no deployment. Oddity: the sync
path raises a bare ``ValueError`` where the async one raises ``RateLimitError``."""
the first call is counted against ``rpm=1`` and the next pick finds no deployment, raising the same
``RateLimitError`` as the async path."""
core: Final = make_core(model_list=[deployment("g", "a", rpm=1)], num_retries=0)
core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue]
with pytest.raises(ValueError, match=re.escape("No deployments available for selected model. Passed model=g.")):
with pytest.raises( # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
litellm.RateLimitError, match=re.escape("No deployments available for selected model. Passed model=g.")
):
core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue]
assert host.deployments_called() == ["a"]
@ -424,22 +426,31 @@ def test_sync_completion_under_latency_based_routing_reaches_the_group(make_core
assert len(host.calls) == 10
def test_sync_completion_under_cost_based_routing_finds_no_deployment(make_core: MakeCore, host: ScriptedHost) -> None:
"""Oddity: the cost-based selector has no sync pick, so every sync pick under cost-based routing
rate-limits without calling a provider."""
def test_sync_completion_under_cost_based_routing_picks_the_cheapest_deployment(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""The cost-based selector picks on the sync path as it does on the async one."""
core: Final = make_core(
model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="cost-based-routing", num_retries=0
model_list=[
deployment("g", "a", input_cost_per_token=0.5, output_cost_per_token=0.5),
deployment("g", "b", input_cost_per_token=0.1, output_cost_per_token=0.1),
],
routing_strategy="cost-based-routing",
num_retries=0,
)
with pytest.raises(RouterRateLimitError) as raised:
for _ in range(3):
core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g")
assert host.calls == []
assert host.deployments_called() == ["b", "b", "b"]
@pytest.mark.asyncio
async def test_pass_through_pick_under_cost_based_routing_finds_no_deployment(make_core: MakeCore) -> None:
async def test_pass_through_pick_under_cost_based_routing_picks_the_cheapest_deployment(make_core: MakeCore) -> None:
core: Final = make_core(
model_list=[deployment("g", "a", use_in_pass_through=True)], routing_strategy="cost-based-routing"
model_list=[
deployment("g", "a", use_in_pass_through=True, input_cost_per_token=0.5, output_cost_per_token=0.5),
deployment("g", "b", use_in_pass_through=True, input_cost_per_token=0.1, output_cost_per_token=0.1),
],
routing_strategy="cost-based-routing",
)
with pytest.raises(RouterRateLimitError):
await core.select("g", kind="pass_through")
picked: Final = await core.select("g", kind="pass_through") # pyright: ignore[reportCallIssue, reportUnknownVariableType] # pytest and the cores' untyped members
assert picked["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue, reportCallIssue, reportArgumentType]

View file

@ -328,9 +328,10 @@ async def test_a_per_user_github_copilot_deployment_rejects_another_credential_n
async def test_the_process_request_timeout_outranks_the_router_timeout_once_a_router_timeout_is_set(
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Oddity: this per-attempt timeout is read from ``litellm.request_timeout`` itself (not the host's
process settings), and only when the router has a ``timeout`` of its own."""
monkeypatch.setattr(litellm, "request_timeout", 33.0)
"""This per-attempt timeout is read from the host's process settings (not ``litellm.request_timeout``
itself), and only when the router has a ``timeout`` of its own."""
host.set_dynamic(request_timeout=33.0)
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42)
await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
assert host.calls[0].kwargs["timeout"] == 33.0
@ -341,7 +342,8 @@ async def test_the_process_request_timeout_is_the_stream_timeout_without_a_strea
) -> None:
from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts
monkeypatch.setattr(litellm, "request_timeout", 33.0)
host.set_dynamic(request_timeout=33.0)
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
host.answer("openai/d1", chat_stream("A"))
core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42)
await collect_texts(await core.acompletion(model="g", messages=MESSAGES, stream=True)) # pyright: ignore[reportAttributeAccessIssue]
@ -424,8 +426,18 @@ async def test_acompletion_counts_total_success_and_failed_calls_per_deployment_
assert counts == (2, 1, 1)
def test_sync_completion_does_not_count_calls(make_core: MakeCore, host: ScriptedHost) -> None:
"""Oddity: only the async path updates ``total_calls``/``success_calls``/``fail_calls``."""
core: Final = make_core(model_list=[deployment("g", "d1")])
def test_completion_counts_total_success_and_failed_calls_per_deployment_model(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""The sync path counts calls as ``acompletion`` does."""
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0)
core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
assert core.total_calls["openai/d1"] == 0 # pyright: ignore[reportAttributeAccessIssue]
host.answer("openai/d1", rate_limit())
with pytest.raises(litellm.RateLimitError): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
counts: Final = ( # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members
core.total_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
core.success_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
core.fail_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
)
assert counts == (2, 1, 1)

View file

@ -61,33 +61,34 @@ def stream_with_usage(sync: bool = False) -> Callable[..., object]:
# --- a fallback that answers without a stream --------------------------------------------------
async def test_a_mid_stream_fallback_that_answers_without_a_stream_yields_one_none_item(
async def test_a_mid_stream_fallback_that_answers_without_a_stream_yields_its_response_as_one_chunk(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Oddity: the fallback's whole response is dropped and the stream yields ``None`` in its place."""
host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True))
host.answer("openai/b", completion_response("not streamed", model="b"))
core: Final = build(make_core)
stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue]
seen: list[object] = []
await drain_into(stream, seen)
assert seen[-1] is None
last: Final = seen[-1]
assert isinstance(last, ModelResponseStream)
assert (last.model, last.choices[0].delta.content) == ("b", "not streamed") # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
assert host.deployments_called() == ["a", "b"]
def test_a_sync_mid_stream_fallback_that_answers_without_a_stream_yields_the_response_fields(
def test_a_sync_mid_stream_fallback_that_answers_without_a_stream_yields_its_response_as_one_chunk(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Oddity: a ``ModelResponse`` is iterable (pydantic yields ``(field, value)`` pairs), so the sync
wrapper streams the fallback response's fields instead of chunks."""
"""The sync wrapper yields the same one chunk as the async one, not the response's pydantic fields."""
host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True, sync=True))
host.answer("openai/b", completion_response("not streamed", model="b"))
core: Final = build(make_core)
stream: Final = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue]
items: Final = list(stream) # pyright: ignore[reportArgumentType]
fields: Final = [item for item in items if isinstance(item, tuple)]
assert ("model", "b") in fields
assert {name for name, _ in fields} >= {"id", "choices", "usage"}
assert not any(isinstance(item, tuple) for item in items) # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members
last: Final = items[-1] # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members
assert isinstance(last, ModelResponseStream)
assert (last.model, last.choices[0].delta.content) == ("b", "not streamed") # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
assert host.deployments_called() == ["a", "b"]

View file

@ -8,6 +8,7 @@ Python does them today and called out in the test's docstring.
"""
import datetime
import inspect
import io
from collections.abc import Callable, Mapping
from types import SimpleNamespace
@ -73,6 +74,10 @@ ROUTED_ENDPOINTS: Final[dict[str, dict[str, object]]] = {
"acancel_batch": {"batch_id": "b1"},
}
SYNC_COUNTED: Final[dict[str, dict[str, object]]] = {
"completion": {"messages": MESSAGES},
"embedding": {"input": ["a"]},
}
# --- batch completions (5565-5790) ----------------------------------------------------------
@ -320,17 +325,21 @@ async def test_image_generation_for_an_unknown_group_raises_without_calling_the_
("aembedding", "openai/d1", "openai/d1"),
("acreate_batch", "openai/d1", "g"),
("acancel_batch", "openai/d1", "g"),
("image_generation", "", ""),
("image_generation", "openai/d1", "openai/d1"),
("text_completion", "openai/d1", "g"),
("completion", "openai/d1", "openai/d1"),
("embedding", "openai/d1", "openai/d1"),
],
)
async def test_call_counters_are_keyed_differently_per_endpoint(
async def test_call_counters_are_keyed_per_endpoint_and_the_same_for_sync_and_async(
make_core: MakeCore, host: EndpointHost, endpoint: str, total_key: str, fail_key: str
):
"""total_calls/fail_calls are keyed by the deployment model for some endpoints and by the model group
for others (fail_calls even differs from total_calls for some); sync image_generation counts under ""."""
for others (fail_calls even differs from total_calls for some); a sync endpoint counts as its async
twin does."""
host.answer("*", rate_limit())
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=1)
request: Final = ROUTED_ENDPOINTS.get(endpoint, {"prompt": "p"})
request: Final = SYNC_COUNTED.get(endpoint) or ROUTED_ENDPOINTS.get(endpoint, {"prompt": "p"})
with pytest.raises(litellm.RateLimitError):
await core.call(endpoint, {"model": "g", **request})
assert dict(core.total_calls) == {total_key: 2} # pyright: ignore[reportAttributeAccessIssue] # a public counter
@ -399,16 +408,22 @@ async def test_async_passthrough_rewrites_only_whole_path_segments_naming_the_gr
assert host.calls[0].kwargs["endpoint"] == "/model/us.anthropic.claude/invoke/gg/us.anthropic.claude:stream"
async def test_sync_generic_call_replaces_every_substring_of_the_group_name_in_the_endpoint(
make_core: MakeCore, monkeypatch: pytest.MonkeyPatch, host: EndpointHost
@pytest.mark.parametrize("endpoint", ["aresponses", "responses"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
async def test_generic_call_rewrites_only_whole_path_segments_naming_the_group(
make_core: MakeCore,
monkeypatch: pytest.MonkeyPatch,
host: EndpointHost,
endpoint: str, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members
):
"""The sync path uses str.replace with the full deployment model (provider prefix kept), so a group
name inside other path segments is rewritten too, unlike the async path's segment-bounded rewrite."""
"""Sync and async both swap the group name for the provider-stripped deployment model one whole path
segment at a time, so a segment that merely contains the group name ("gg") is left alone."""
patch_direct_endpoints(monkeypatch, host)
core: Final = make_core(model_list=[deployment("g", "d1")])
core.responses(model="g", input="hi", endpoint="/gg/g")
sent: Final = getattr(core, endpoint)(model="g", input="hi", endpoint="/gg/g") # pyright: ignore[reportAny] # pytest and the cores' untyped members
if inspect.isawaitable(sent): # pyright: ignore[reportAny] # pytest and the cores' untyped members
await sent
call: Final = host.calls[0]
assert (call.model, call.kwargs["endpoint"]) == ("openai/d1", "/openai/d1openai/d1/openai/d1")
assert (call.model, call.kwargs["endpoint"]) == ("openai/d1", "/gg/d1")
# --- batches (8112-8362) --------------------------------------------------------------------

View file

@ -100,10 +100,10 @@ async def test_a_passing_pre_call_check_sees_the_deployment_that_is_then_called(
assert gate.seen == [("async", host.deployments_called("acompletion")[0])]
def test_a_rate_limit_from_a_sync_pre_call_check_is_raised_without_a_cooldown(
def test_a_rate_limit_from_a_sync_pre_call_check_cools_the_deployment_down_and_is_raised(
make_core: MakeCore, host: ScriptedHost
) -> None:
"""Oddity: the sync twin raises the gate's error but, unlike the async path, sets no cooldown."""
"""The sync twin cools the refused deployment down as the async path does."""
core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33)
gate: Final = gated(host, rate_limit(message="gate says no"))
@ -111,7 +111,23 @@ def test_a_rate_limit_from_a_sync_pre_call_check_is_raised_without_a_cooldown(
assert isinstance(error, litellm.RateLimitError)
assert host.calls == []
assert [kind for kind, _ in gate.seen] == ["sync"]
((kind, checked),) = gate.seen
assert kind == "sync"
cooled: Final = cooled_down(core, ["a", "b"])
assert list(cooled) == [checked]
assert cooled[checked]["cooldown_time"] == 33
def test_any_other_error_from_a_sync_pre_call_check_is_raised_without_a_cooldown(
make_core: MakeCore, host: ScriptedHost
) -> None:
core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33)
gated(host, ValueError("gate broke"))
error: Final = attempt_sync(core)
assert type(error) is ValueError
assert host.calls == []
assert cooled_down(core, ["a", "b"]) == {}

View file

@ -535,13 +535,13 @@ async def test_pass_through_selection_without_pass_through_deployments_raises_ba
with pytest.raises(litellm.BadRequestError) as raised:
await core.select("g", kind="pass_through")
assert str(raised.value) == (
"litellm.BadRequestError: Model g has no deployment configured with use_in_pass_through=True. "
"Please add use_in_pass_through: true in the deployment configuration"
"litellm.BadRequestError: Model g has no deployments configured with use_in_pass_through=True. "
"Please add use_in_pass_through: true to the deployment configuration"
)
@pytest.mark.asyncio
async def test_async_pass_through_selection_words_the_missing_deployment_error_differently(
async def test_async_pass_through_selection_words_the_missing_deployment_error_the_same_way(
make_core: MakeCore,
) -> None:
core: Final = make_core(model_list=[deployment("g", "a")])

View file

@ -195,15 +195,23 @@ async def test_selecting_a_marker_by_its_id_refuses_it_naming_its_litellm_model(
assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="auto_router/complexity_router")
@pytest.mark.parametrize("entry", ["get_available_deployment", "completion"])
def test_the_sync_path_never_runs_pre_routing_and_refuses_the_marker(
make_core: MakeCore, host: ScriptedHost, entry: str
@pytest.mark.parametrize("model", ["smart", "al"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
def test_sync_get_available_deployment_runs_pre_routing_like_the_async_pick(make_core: MakeCore, model: str) -> None:
core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"})
request: Final[dict[str, object]] = {}
chosen: Final = core.get_available_deployment(model, messages=MESSAGES, request_kwargs=request) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
assert chosen["model_info"]["id"] == "s"
assert _metadata(request)["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue]
@pytest.mark.parametrize(("model", "messages", "expected"), [("smart", MESSAGES, "s"), ("al", HARD_MESSAGES, "b")]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
def test_sync_completion_routes_through_a_complexity_router_like_acompletion(
make_core: MakeCore, host: ScriptedHost, model: str, messages: list[dict[str, str]], expected: str
) -> None:
core: Final = make_core(model_list=_tiered())
with pytest.raises(litellm.BadRequestError) as raised:
_sync_entry(core, entry, "smart")
assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="smart")
assert host.calls == []
core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"})
core.completion(model=model, messages=messages) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
assert host.deployments_called() == [expected]
assert host.calls[-1].model_group == model
# --- choosing among tag-scoped markers -----------------------------------------------------------------