mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge 1a7deafe09 into 35adb08c3c
This commit is contained in:
commit
d8d61e2fef
30 changed files with 807 additions and 413 deletions
|
|
@ -81,7 +81,11 @@ def can_block_current_thread() -> bool:
|
|||
return threading.current_thread() is threading.main_thread() and not is_event_loop_running()
|
||||
|
||||
|
||||
def run_async_function(async_function, *args, **kwargs):
|
||||
def run_async_function(
|
||||
async_function: Callable[..., Awaitable[T_Retval]],
|
||||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: forwarded to async_function as given
|
||||
) -> T_Retval:
|
||||
"""
|
||||
Helper utility to run an async function in a sync context.
|
||||
Handles the case where there is an existing event loop running.
|
||||
|
|
|
|||
|
|
@ -26,8 +26,14 @@ def get_configured_request_timeout() -> float | None:
|
|||
"""Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
|
||||
import litellm
|
||||
|
||||
timeout: Final = float(litellm.request_timeout)
|
||||
if litellm.request_timeout_explicitly_set:
|
||||
return configured_request_timeout(litellm.request_timeout, litellm.request_timeout_explicitly_set)
|
||||
|
||||
|
||||
def configured_request_timeout(request_timeout: float, explicitly_set: bool) -> float | None:
|
||||
"""``get_configured_request_timeout`` for a process ``request_timeout`` read elsewhere (a router
|
||||
host's process settings): the value when it was configured, else ``None``."""
|
||||
timeout: Final = float(request_timeout)
|
||||
if explicitly_set:
|
||||
return timeout
|
||||
if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
|
||||
return timeout
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ class RouterDynamicConfig:
|
|||
input_callback: Sequence[object]
|
||||
callback_manager: "LoggingCallbackManager"
|
||||
cache: "Cache | None"
|
||||
request_timeout_explicitly_set: bool = False
|
||||
|
||||
|
||||
_versions: Final = itertools.count(1)
|
||||
|
|
@ -110,6 +111,7 @@ def current_dynamic_config() -> RouterDynamicConfig:
|
|||
input_callback=litellm.input_callback,
|
||||
callback_manager=litellm.logging_callback_manager,
|
||||
cache=litellm.cache,
|
||||
request_timeout_explicitly_set=litellm.request_timeout_explicitly_set,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -120,6 +122,7 @@ _LIVE_SOURCES: Final = MappingProxyType(
|
|||
"router_max_fallbacks": "ROUTER_MAX_FALLBACKS",
|
||||
"allowed_fails": "allowed_fails",
|
||||
"request_timeout": "request_timeout",
|
||||
"request_timeout_explicitly_set": "request_timeout_explicitly_set",
|
||||
"fallbacks": "fallbacks",
|
||||
"default_fallbacks": "default_fallbacks",
|
||||
"context_window_fallbacks": "context_window_fallbacks",
|
||||
|
|
|
|||
|
|
@ -98,6 +98,16 @@ def _as_router(obj: "DeprecatedRouterMembers") -> "Router":
|
|||
raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only")
|
||||
|
||||
|
||||
def _registry_source(obj: "DeprecatedRouterMembers") -> "Router":
|
||||
"""What ``registry()`` reads: the Router on the Python core, the host registry on the Rust core.
|
||||
|
||||
A deprecated member whose ``registry()`` query checks for a name reads it directly, as the
|
||||
member did before ``registry()`` existed, so a ``None`` name gets the old result on both cores
|
||||
instead of ``registry()``'s ``TypeError``.
|
||||
"""
|
||||
return _as_router(obj).registry()._router # pyright: ignore[reportPrivateUsage] # the shim is part of the router
|
||||
|
||||
|
||||
def _public_attribute_error(obj: object, core_name: str, public_name: str, error: AttributeError) -> AttributeError:
|
||||
"""
|
||||
``error`` renamed to the public attribute when it is the router's own ``core_name`` that is
|
||||
|
|
@ -507,8 +517,8 @@ class DeprecatedRouterMembers:
|
|||
self, model_name: str, model_alias: str | None = None, team_id: str | None = None
|
||||
) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged
|
||||
_note("get_all_deployments", "registry().deployments(model_name, source='all')")
|
||||
return (
|
||||
_as_router(self).registry().deployments(model_name, source="all", model_alias=model_alias, team_id=team_id)
|
||||
return _registry_source(self)._core_get_all_deployments(
|
||||
model_name=model_name, model_alias=model_alias, team_id=team_id
|
||||
)
|
||||
|
||||
_get_all_deployments = _private_alias("get_all_deployments", get_all_deployments)
|
||||
|
|
@ -519,7 +529,7 @@ class DeprecatedRouterMembers:
|
|||
|
||||
def deployments_named(self, model_name: str) -> "tuple[DeploymentTypedDict, ...] | None":
|
||||
_note("deployments_named", "registry().deployments(model_name, source='named')")
|
||||
return _as_router(self).registry().deployments(model_name, source="named")
|
||||
return _registry_source(self)._core_deployments_named(model_name)
|
||||
|
||||
def get_model_list_from_model_alias(
|
||||
self, model_name: str | None = None
|
||||
|
|
@ -535,11 +545,11 @@ class DeprecatedRouterMembers:
|
|||
self, model_group: str
|
||||
) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_list_of_routed_group", "registry().deployments(model_group, source='routed_group')")
|
||||
return _as_router(self).registry().deployments(model_group, source="routed_group")
|
||||
return _registry_source(self)._core_get_model_list_of_routed_group(model_group)
|
||||
|
||||
def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]:
|
||||
_note("wildcard_route_deployments", "registry().deployments(model, source='wildcard')")
|
||||
return _as_router(self).registry().deployments(model, source="wildcard", team_id=team_id)
|
||||
return _registry_source(self)._core_wildcard_route_deployments(model, team_id)
|
||||
|
||||
def get_routable_deployments(
|
||||
self,
|
||||
|
|
@ -550,21 +560,17 @@ class DeprecatedRouterMembers:
|
|||
_note(
|
||||
"get_routable_deployments", "registry().deployments(model, source='routable', request=..., call_type=...)"
|
||||
)
|
||||
return (
|
||||
_as_router(self)
|
||||
.registry()
|
||||
.deployments(model, source="routable", request=request_kwargs, call_type=call_type)
|
||||
)
|
||||
return _registry_source(self)._core_get_routable_deployments(model, request_kwargs, call_type)
|
||||
|
||||
def get_deployment_by_model_group_name(self, model_group_name: str) -> "Deployment | None":
|
||||
_note("get_deployment_by_model_group_name", "registry().deployments(name, source='first_in_group')")
|
||||
return _as_router(self).registry().deployments(model_group_name, source="first_in_group")
|
||||
return _registry_source(self)._core_get_deployment_by_model_group_name(model_group_name=model_group_name)
|
||||
|
||||
def deployments_for_request(
|
||||
self, model: str, request_kwargs: Mapping[str, object]
|
||||
) -> "Sequence[DeploymentTypedDict]":
|
||||
_note("deployments_for_request", "registry().deployments(model, source='for_request', request=...)")
|
||||
return _as_router(self).registry().deployments(model, source="for_request", request=request_kwargs)
|
||||
return _registry_source(self)._core_deployments_for_request(model, request_kwargs)
|
||||
|
||||
def should_include_deployment(
|
||||
self,
|
||||
|
|
@ -599,34 +605,32 @@ class DeprecatedRouterMembers:
|
|||
self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False
|
||||
) -> tuple[str, ...]:
|
||||
_note("resolved_litellm_models", "registry().names(model_name, kind='litellm_models' or 'litellm_base_models')")
|
||||
return (
|
||||
_as_router(self)
|
||||
.registry()
|
||||
.names(model_name, kind="litellm_base_models" if use_base_model else "litellm_models", team_id=team_id)
|
||||
return _registry_source(self)._core_resolved_litellm_models(
|
||||
model_name, team_id=team_id, use_base_model=use_base_model
|
||||
)
|
||||
|
||||
def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]:
|
||||
_note("wildcard_routed_models", "registry().names(model, kind='wildcard')")
|
||||
return _as_router(self).registry().names(model, kind="wildcard", team_id=team_id)
|
||||
return _registry_source(self)._core_wildcard_routed_models(model, team_id)
|
||||
|
||||
def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]:
|
||||
_note("get_candidate_model_ids_for_route", "registry().names(model, kind='route_candidates')")
|
||||
return _as_router(self).registry().names(model, kind="route_candidates", team_id=team_id)
|
||||
return _registry_source(self)._core_get_candidate_model_ids_for_route(model, team_id=team_id)
|
||||
|
||||
# -- registry: resolving names ---------------------------------------------------------------
|
||||
def routable_model_group(self, model_name: str) -> str:
|
||||
_note("routable_model_group", "registry().resolve(name)")
|
||||
return _as_router(self).registry().resolve(model_name)
|
||||
return _registry_source(self)._core_routable_model_group(model_name)
|
||||
|
||||
def get_model_from_alias(self, model: str) -> str | None:
|
||||
_note("get_model_from_alias", "registry().resolve(name, kind='alias')")
|
||||
return _as_router(self).registry().resolve(model, kind="alias")
|
||||
return _registry_source(self)._core_get_model_from_alias(model=model)
|
||||
|
||||
_get_model_from_alias = _private_alias("get_model_from_alias", get_model_from_alias)
|
||||
|
||||
def resolve_served_model_name(self, model_name: str, served_model: str) -> str:
|
||||
_note("resolve_served_model_name", "registry().resolve(name, kind='served', served_model=...)")
|
||||
return _as_router(self).registry().resolve(model_name, kind="served", served_model=served_model)
|
||||
return _registry_source(self)._core_resolve_served_model_name(model_name, served_model)
|
||||
|
||||
def resolve_model_name_from_model_id(
|
||||
self, model_id: str | None, custom_llm_provider: str | None = None
|
||||
|
|
@ -646,23 +650,25 @@ class DeprecatedRouterMembers:
|
|||
|
||||
def is_recognized_model(self, model: str) -> bool:
|
||||
_note("is_recognized_model", "registry().resolve(name, kind='recognized')")
|
||||
return _as_router(self).registry().resolve(model, kind="recognized")
|
||||
return _registry_source(self)._core_is_recognized_model(model)
|
||||
|
||||
def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None:
|
||||
_note("get_routable_upstream_model", "registry().resolve(name, kind='upstream')")
|
||||
return _as_router(self).registry().resolve(model_name, kind="upstream", team_id=team_id)
|
||||
return _registry_source(self)._core_get_routable_upstream_model(model_name, team_id=team_id)
|
||||
|
||||
def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]:
|
||||
_note("get_search_tool_support", "registry().resolve(name, kind='search_tool_support')")
|
||||
return _as_router(self).registry().resolve(model_name, kind="search_tool_support", team_id=team_id)
|
||||
return _registry_source(self)._core_get_search_tool_support(model_name, team_id=team_id)
|
||||
|
||||
def get_model_group(self, id: str) -> list | None: # mutable-ok: the old return type, unchanged
|
||||
_note("get_model_group", "registry().resolve(id, kind='group_of_id')")
|
||||
return _as_router(self).registry().resolve(id, kind="group_of_id")
|
||||
return _registry_source(self)._core_get_model_group(id=id)
|
||||
|
||||
def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool:
|
||||
_note("is_model_access_group_for_wildcard_route", "registry().resolve(group, kind='wildcard_access_group')")
|
||||
return _as_router(self).registry().resolve(model_access_group, kind="wildcard_access_group")
|
||||
return _registry_source(self)._core_is_model_access_group_for_wildcard_route(
|
||||
model_access_group=model_access_group
|
||||
)
|
||||
|
||||
_is_model_access_group_for_wildcard_route = _private_alias(
|
||||
"is_model_access_group_for_wildcard_route", is_model_access_group_for_wildcard_route
|
||||
|
|
@ -670,7 +676,7 @@ class DeprecatedRouterMembers:
|
|||
|
||||
def team_model_has_alternatives(self, deployment_id: str) -> bool:
|
||||
_note("team_model_has_alternatives", "registry().resolve(id, kind='team_model_alternatives')")
|
||||
return _as_router(self).registry().resolve(deployment_id, kind="team_model_alternatives")
|
||||
return _registry_source(self)._core_team_model_has_alternatives(deployment_id)
|
||||
|
||||
def routing_group_has_alternatives(self, model_group: str | None) -> bool:
|
||||
_note("routing_group_has_alternatives", "registry().resolve(group, kind='routing_group_alternatives')")
|
||||
|
|
|
|||
|
|
@ -180,7 +180,7 @@ from litellm.litellm_core_utils.reasoning_effort_capability import (
|
|||
resolve_supported_reasoning_efforts,
|
||||
)
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import (
|
||||
get_configured_request_timeout,
|
||||
configured_request_timeout,
|
||||
get_service_tier_timeout,
|
||||
)
|
||||
from litellm.litellm_core_utils.routing_common_utils import (
|
||||
|
|
@ -366,6 +366,7 @@ from litellm.router_utils.fallback_event_handlers import (
|
|||
mid_stream_fallback_hop_kwargs,
|
||||
mid_stream_fallback_snapshot_kwargs,
|
||||
mid_stream_retry_kwargs,
|
||||
non_stream_fallback_answer_as_chunk,
|
||||
per_request_fallback_controls,
|
||||
record_disable_fallbacks,
|
||||
record_pre_routing_selection,
|
||||
|
|
@ -699,6 +700,12 @@ _NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
|
|||
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
# The router strategies whose pick runs the pre-routing hook (`_async_get_available_deployment`).
|
||||
_PRE_ROUTING_PICK_STRATEGIES: Final = frozenset(
|
||||
{"usage-based-routing-v2", "simple-shuffle", "cost-based-routing", "latency-based-routing", "least-busy"}
|
||||
)
|
||||
# The generic calls (sync and async) resolve their timeout with the LLM pass-through resolver.
|
||||
_GENERIC_CALL_FUNCTION_NAMES: Final = frozenset({"_ageneric_api_call_with_fallbacks", "generic_api_call"})
|
||||
_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
|
||||
_RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
|
||||
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
|
|
@ -2083,7 +2090,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
# Per-attempt request_timeout, independent of router_settings.timeout.
|
||||
# Only stored when a router timeout is also set, since otherwise
|
||||
# request_timeout already flows through self.timeout above.
|
||||
self.request_timeout = get_configured_request_timeout() if timeout is not None else None
|
||||
self.request_timeout = self._process_request_timeout() if timeout is not None else None
|
||||
self.stream_timeout = stream_timeout
|
||||
|
||||
self.retry_after = retry_after
|
||||
|
|
@ -2972,16 +2979,13 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
if selector is None:
|
||||
return None
|
||||
|
||||
# `cost-based-routing` is intentionally omitted —
|
||||
# `LowestCostLoggingHandler` only implements
|
||||
# `async_get_available_deployments`
|
||||
match strategy:
|
||||
case "least-busy":
|
||||
return selector.get_available_deployments(
|
||||
model_group=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
)
|
||||
case "usage-based-routing" | "usage-based-routing-v2":
|
||||
case "usage-based-routing" | "usage-based-routing-v2" | "cost-based-routing":
|
||||
return selector.get_available_deployments(
|
||||
model_group=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
|
|
@ -3411,6 +3415,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
model_client = None
|
||||
else:
|
||||
model_client = potential_model_client
|
||||
self.total_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
|
||||
### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit)
|
||||
## only run if model group given, not model id
|
||||
|
|
@ -3427,6 +3432,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
}
|
||||
)
|
||||
response: Final = self._host.invoke("completion")(**input_kwargs)
|
||||
self.success_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
|
||||
## CHECK CONTENT FILTER ERROR ##
|
||||
|
|
@ -3459,6 +3465,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
return response
|
||||
except Exception as e:
|
||||
verbose_router_logger.info("litellm.completion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
|
||||
if model_name is not None:
|
||||
self.fail_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
# Set per-deployment num_retries on exception for retry logic
|
||||
if deployment is not None:
|
||||
self._set_deployment_num_retries_on_exception(e, deployment)
|
||||
|
|
@ -3733,6 +3741,17 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
adopting_wrapper.adopt_fallback_response_headers(fallback_response, prepared)
|
||||
return prepared
|
||||
|
||||
@staticmethod
|
||||
def _non_stream_fallback_item(
|
||||
wrapper_ref: "weakref.ref[FallbackAwareStreamWrapper]", fallback_response: object
|
||||
) -> object:
|
||||
"""The one item a stream yields for a fallback that answered without a stream: its
|
||||
response as a stream chunk, carrying the fallback's hidden params and headers."""
|
||||
prepared: Final = PythonCore._adopt_fallback_response_headers(wrapper_ref, fallback_response)
|
||||
item: Final = non_stream_fallback_answer_as_chunk(fallback_response)
|
||||
PythonCore._apply_fallback_hidden_params_to_item(item, prepared)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
def _apply_fallback_hidden_params_to_item(
|
||||
fallback_item: object,
|
||||
|
|
@ -3949,8 +3968,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
yield fallback_item
|
||||
else:
|
||||
# If fallback returns a non-streaming response, yield None
|
||||
yield None
|
||||
yield PythonCore._non_stream_fallback_item(wrapper_ref, fallback_response) # pyright: ignore[reportAny] # the legacy untyped router core
|
||||
|
||||
except Exception as fallback_error:
|
||||
# If fallback also fails, log and re-raise original error
|
||||
|
|
@ -4710,7 +4728,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
initial_kwargs["messages"] = messages
|
||||
router_self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs)
|
||||
fallback_response = run_async_function(
|
||||
router_self.async_function_with_fallbacks_common_utils,
|
||||
router_self.async_function_with_fallbacks_common_utils, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
e=e.original_exception
|
||||
if isinstance(e.original_exception, litellm_exceptions.ContextWindowExceededError)
|
||||
else e,
|
||||
|
|
@ -4724,7 +4742,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
include_fallback_errors=initial_kwargs.get("include_fallback_errors", False) is True,
|
||||
)
|
||||
|
||||
if hasattr(fallback_response, "__iter__"):
|
||||
if hasattr(fallback_response, "__iter__") and not isinstance(fallback_response, ModelResponse): # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
prepared_fallback_hidden_params = PythonCore._adopt_fallback_response_headers(
|
||||
wrapper_ref, fallback_response
|
||||
)
|
||||
|
|
@ -4747,7 +4765,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
router_self._combine_fallback_usage(fallback_item, None)
|
||||
yield fallback_item
|
||||
else:
|
||||
yield None
|
||||
yield PythonCore._non_stream_fallback_item(wrapper_ref, fallback_response) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
|
||||
except Exception as fallback_error:
|
||||
verbose_router_logger.error("Fallback also failed: %s", fallback_error)
|
||||
|
|
@ -4766,9 +4784,9 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
"stream_with_fallbacks: error closing model_response: %s",
|
||||
close_err,
|
||||
)
|
||||
if fallback_response is not None and hasattr(fallback_response, "close"):
|
||||
if fallback_response is not None and hasattr(fallback_response, "close"): # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
try:
|
||||
fallback_response.close()
|
||||
fallback_response.close() # pyright: ignore[reportAttributeAccessIssue] # a stream wrapper, closed when it has close()
|
||||
except BaseException as close_err:
|
||||
verbose_router_logger.debug(
|
||||
"stream_with_fallbacks: error closing fallback_response: %s",
|
||||
|
|
@ -5299,17 +5317,19 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
|
||||
kwargs["model_info"] = model_info
|
||||
|
||||
if function_name == "_ageneric_api_call_with_fallbacks":
|
||||
if function_name in _GENERIC_CALL_FUNCTION_NAMES:
|
||||
from litellm.passthrough.timeout_utils import (
|
||||
resolve_llm_passthrough_timeout,
|
||||
)
|
||||
|
||||
# The process request_timeout is read through the host; it ranks right after the
|
||||
# router timeout, as resolve_llm_passthrough_timeout ranks it.
|
||||
_router_timeout: Final = (
|
||||
self.request_timeout
|
||||
if self.request_timeout is not None
|
||||
else float(self._explicit_timeout)
|
||||
if isinstance(self._explicit_timeout, (int, float))
|
||||
else None
|
||||
else self._process_request_timeout()
|
||||
)
|
||||
_router_stream_timeout: Final = (
|
||||
self.stream_timeout
|
||||
|
|
@ -5381,6 +5401,11 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
hop_kwargs, _FALLBACK_HOP_ADAPTER.validate_python(deployment)
|
||||
)
|
||||
|
||||
def _process_request_timeout(self) -> float | None:
|
||||
"""The host's explicitly configured process ``request_timeout``, else None."""
|
||||
dynamic: Final = self._host.dynamic()
|
||||
return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set)
|
||||
|
||||
def _get_async_openai_model_client(self, deployment: dict, kwargs: dict):
|
||||
"""
|
||||
Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment
|
||||
|
|
@ -5926,7 +5951,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
raise e
|
||||
|
||||
def _image_generation(self, prompt: str, model: str, **kwargs):
|
||||
model_name: Final = ""
|
||||
model_name = model # rebind-ok: the group until a deployment is picked, as in _aimage_generation
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _image_generation()- model: %s", model)
|
||||
deployment: Final = self.get_available_deployment(
|
||||
|
|
@ -5937,6 +5962,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs)
|
||||
data: Final = deployment["litellm_params"].copy()
|
||||
model_name = data["model"] # pyright: ignore[reportUnknownVariableType] # rebind-ok: the picked deployment's model, as in _aimage_generation
|
||||
|
||||
model_client: Final = self._get_async_openai_model_client(
|
||||
deployment=deployment,
|
||||
|
|
@ -5958,10 +5984,10 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
}
|
||||
)
|
||||
self.success_calls[model_name] += 1
|
||||
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
|
||||
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
return response
|
||||
except Exception as e:
|
||||
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
|
||||
verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
if model_name is not None:
|
||||
self.fail_calls[model_name] += 1
|
||||
self._stamp_retry_skip_deployment_id(e, kwargs)
|
||||
|
|
@ -6300,37 +6326,64 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
is_async: bool | None = False,
|
||||
**kwargs,
|
||||
):
|
||||
messages: Final = [{"role": "user", "content": prompt}]
|
||||
try:
|
||||
kwargs["model"] = model
|
||||
kwargs["prompt"] = prompt
|
||||
kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries)
|
||||
kwargs.setdefault("metadata", {}).update({"model_group": model})
|
||||
"""The sync twin of `atext_completion`: picks, shapes, retries and falls back the same way."""
|
||||
kwargs["model"] = model # rebind-ok: the request kwargs, shaped in place as atext_completion shapes them
|
||||
kwargs["prompt"] = prompt # rebind-ok: same
|
||||
kwargs["original_function"] = self._text_completion # rebind-ok: same
|
||||
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
return self.function_with_fallbacks(**kwargs) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # the legacy untyped router core
|
||||
|
||||
# pick the one that is available (lowest TPM/RPM)
|
||||
with endpoint_selection("text_completion", model=model):
|
||||
deployment: Final = self.get_available_deployment(
|
||||
model=model,
|
||||
messages=messages,
|
||||
specific_deployment=kwargs.pop("specific_deployment", None),
|
||||
request_kwargs=kwargs,
|
||||
)
|
||||
def _text_completion(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str,
|
||||
**kwargs: object, # kwargs-ok: the request kwargs, sent on to the deployment
|
||||
) -> object:
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _text_completion()- model: %s", model)
|
||||
deployment: Final = self.get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy untyped router core
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
specific_deployment=kwargs.pop("specific_deployment", None), # pyright: ignore[reportArgumentType] # the legacy untyped router core
|
||||
request_kwargs=kwargs,
|
||||
)
|
||||
self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, splits_stream_timeout=True) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
|
||||
data: Final = deployment["litellm_params"].copy()
|
||||
for k, v in self._core_default_litellm_params.items():
|
||||
if k not in kwargs: # prioritize model-specific params > default router params
|
||||
kwargs[k] = v
|
||||
elif k == "metadata":
|
||||
kwargs[k].update(v)
|
||||
pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data))
|
||||
if pinned_model_id is not None:
|
||||
kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults
|
||||
model_name: Final = data["model"] # pyright: ignore[reportUnknownVariableType] # the legacy untyped router core
|
||||
|
||||
# call via litellm.completion()
|
||||
return self._host.invoke("text_completion")(
|
||||
**{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs}
|
||||
potential_model_client: Final = self._get_client(deployment=deployment, kwargs=kwargs, client_type="sync") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy untyped router core
|
||||
dynamic_api_key: Final = kwargs.get("api_key", None)
|
||||
model_client: Final = ( # pyright: ignore[reportUnknownVariableType] # the legacy untyped router core
|
||||
None
|
||||
if dynamic_api_key is not None
|
||||
and potential_model_client is not None
|
||||
and dynamic_api_key != potential_model_client.api_key # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
else potential_model_client
|
||||
)
|
||||
self.total_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
|
||||
### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit)
|
||||
self.routing_strategy_pre_call_checks(deployment=deployment) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
|
||||
response: Final = self._host.invoke("text_completion")( # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
**{
|
||||
**data,
|
||||
"prompt": prompt,
|
||||
"caching": self._core_cache_responses,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
self.success_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
return response
|
||||
except Exception as e:
|
||||
verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model, e)
|
||||
if model is not None: # pyright: ignore[reportUnnecessaryComparison] # the legacy untyped router core
|
||||
self.fail_calls[model] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
self._stamp_retry_skip_deployment_id(e, kwargs)
|
||||
raise e
|
||||
|
||||
async def atext_completion(
|
||||
|
|
@ -7476,8 +7529,11 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
|
||||
# For passthrough routes, use the actual model from deployment
|
||||
# and swap model name in endpoint if present
|
||||
if "endpoint" in kwargs and kwargs["endpoint"]:
|
||||
kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name)
|
||||
self._add_deployment_model_to_endpoint_for_llm_passthrough_route(
|
||||
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
model=model,
|
||||
model_name=model_name, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
)
|
||||
kwargs["model"] = model_name
|
||||
|
||||
# Perform pre-call checks for routing strategy
|
||||
|
|
@ -9473,7 +9529,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
|
||||
Wrapped to reduce code duplication and prevent bugs.
|
||||
"""
|
||||
return run_async_function(self.async_function_with_fallbacks, *args, **kwargs)
|
||||
return run_async_function(self.async_function_with_fallbacks, *args, **kwargs) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
|
||||
def _get_fallback_model_group_for_lookup_groups(
|
||||
self,
|
||||
|
|
@ -10232,11 +10288,22 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
- None
|
||||
|
||||
Raises:
|
||||
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
|
||||
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits; the deployment is
|
||||
cooled down first, as on the async path
|
||||
"""
|
||||
for _callback in self._routing_callbacks():
|
||||
if isinstance(_callback, CustomLogger):
|
||||
_callback.pre_call_check(deployment)
|
||||
try:
|
||||
_callback.pre_call_check(deployment) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
except litellm_exceptions.RateLimitError as e:
|
||||
set_cooldown_deployments(
|
||||
ctx=self._core_ctx,
|
||||
exception_status=e.status_code,
|
||||
original_exception=e,
|
||||
deployment=deployment["model_info"]["id"], # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core
|
||||
time_to_cooldown=self._core_cooldown_time,
|
||||
)
|
||||
raise
|
||||
|
||||
async def async_routing_strategy_pre_call_checks(
|
||||
self,
|
||||
|
|
@ -16234,6 +16301,18 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
return registered_model_name
|
||||
|
||||
def _core_sync_pick_pre_routes(self) -> bool:
|
||||
"""Whether the sync pick runs the pre-routing hook: a strategy router is registered and the
|
||||
router strategy is one whose async pick runs the hook too (see `_async_get_available_deployment`)."""
|
||||
return self._core_routing_strategy in _PRE_ROUTING_PICK_STRATEGIES and any(
|
||||
(
|
||||
self._core_auto_routers,
|
||||
self._core_complexity_routers,
|
||||
self._core_adaptive_routers,
|
||||
self._core_quality_routers,
|
||||
)
|
||||
)
|
||||
|
||||
async def _core_async_pre_routing_hook(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -16646,6 +16725,33 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
"usage-based-routing-v2, cost-based-routing, latency-based-routing, least-busy), "
|
||||
"or remove `plugins` from the Router config."
|
||||
)
|
||||
if self._core_sync_pick_pre_routes():
|
||||
# The async pick's pre-routing hook (auto/complexity/adaptive/quality routers), run to
|
||||
# completion from the sync pick, so a strategy router serves sync calls too.
|
||||
hook_kwargs: Final[dict[str, object]] = ( # pyright: ignore[reportUnknownVariableType] # mutable-ok: the request kwargs the hook stamps
|
||||
request_kwargs if request_kwargs is not None else {}
|
||||
)
|
||||
responses_call: Final = input is not None and messages is None
|
||||
pre_routed: Final = run_async_function(
|
||||
partial(
|
||||
self._core_async_pre_routing_hook, # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
model=model,
|
||||
request_kwargs=hook_kwargs,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
)
|
||||
)
|
||||
if pre_routed is not None:
|
||||
model = pre_routed.model # rebind-ok: the tier group the strategy picked, as the async pick rebinds it
|
||||
messages = pre_routed.messages # rebind-ok: same
|
||||
record_pre_routing_selection(hook_kwargs, model)
|
||||
self._pin_tier_params_onto_request( # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core
|
||||
model=model,
|
||||
tier_litellm_params=pre_routed.litellm_params if pre_routed is not None else None,
|
||||
request_kwargs=hook_kwargs,
|
||||
responses_call=responses_call,
|
||||
)
|
||||
# users need to explicitly call a specific deployment, by setting `specific_deployment = True` as completion()/embedding() kwarg
|
||||
# When this was no explicit we had several issues with fallbacks timing out
|
||||
|
||||
|
|
@ -16926,7 +17032,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
if len(pass_through_deployments) == 0:
|
||||
# No deployments support pass-through
|
||||
raise litellm_exceptions.BadRequestError(
|
||||
message=f"Model {model} has no deployment configured with use_in_pass_through=True. Please add use_in_pass_through: true in the deployment configuration",
|
||||
message=f"Model {model} has no deployments configured with use_in_pass_through=True. Please add use_in_pass_through: true to the deployment configuration",
|
||||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -424,6 +424,19 @@ def failure_kind(error: BaseException) -> str:
|
|||
return next((name for name in names if name in _FAILURE_KINDS), "APIError")
|
||||
|
||||
|
||||
def _crate_int(value: object) -> int | None:
|
||||
"""A count setting kept as given, as the crate reads it: an int, or a number spelled another way
|
||||
(``1.5``, ``"3"``, ``"1e-05"``) truncated to one; None (the crate's default) when it is no number."""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None if value is None else int(value)
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
try:
|
||||
return int(float(str(value)))
|
||||
except (ValueError, OverflowError):
|
||||
return None
|
||||
|
||||
|
||||
def _retry_after_seconds(error: BaseException) -> float | None:
|
||||
response: Final = getattr(error, "response", None)
|
||||
headers: Final[object] = getattr(error, "litellm_response_headers", None) or getattr(response, "headers", None)
|
||||
|
|
@ -490,15 +503,18 @@ class Resolved:
|
|||
|
||||
routing_strategy: str | None
|
||||
routing_strategy_args: Mapping[str, object]
|
||||
num_retries: int
|
||||
max_fallbacks: int
|
||||
allowed_fails: int
|
||||
explicit_allowed_fails: int | None
|
||||
# `num_retries`, `max_fallbacks` and `allowed_fails` as the Python core stores them: as given
|
||||
# (the crate reads them through `_crate_int`).
|
||||
num_retries: object
|
||||
max_fallbacks: object
|
||||
allowed_fails: object
|
||||
explicit_allowed_fails: object
|
||||
cooldown_time: float
|
||||
disable_cooldowns: bool | None
|
||||
allowed_fails_policy: object
|
||||
retry_after: float
|
||||
timeout: float | None
|
||||
# `retry_after` and `timeout` as the Python core stores them: as given (an int stays an int).
|
||||
retry_after: object
|
||||
timeout: object
|
||||
fallbacks: list[object] | None # mutable-ok: the setting as Python spells it
|
||||
context_window_fallbacks: list[object] | None # mutable-ok: same
|
||||
content_policy_fallbacks: list[object] | None # mutable-ok: same
|
||||
|
|
@ -535,15 +551,19 @@ class Resolved:
|
|||
list(_as_list(get("fallbacks"))) or (dynamic.fallbacks if dynamic.fallbacks else None) # pyright: ignore[reportAssignmentType] # the process list as given
|
||||
)
|
||||
refuse_malformed_fallbacks(configured)
|
||||
default_fallbacks: Final = get("default_fallbacks") or dynamic.default_fallbacks
|
||||
if isinstance(default_fallbacks, str):
|
||||
# As the Python core: a given default_fallbacks, even an empty one, adds a `{"*": ...}` entry
|
||||
# (`[]` falls back to the process default_fallbacks, so it may add `{"*": None}`).
|
||||
given_default_fallbacks: Final = get("default_fallbacks")
|
||||
has_default_fallbacks: Final = given_default_fallbacks is not None or dynamic.default_fallbacks is not None
|
||||
default_fallbacks: Final = given_default_fallbacks or dynamic.default_fallbacks
|
||||
if has_default_fallbacks and isinstance(default_fallbacks, str):
|
||||
raise TypeError(
|
||||
f"default_fallbacks must be a list of model names, got '{default_fallbacks}'. Use ['{default_fallbacks}']"
|
||||
)
|
||||
fallbacks: Final[list[object] | None] = ( # mutable-ok: see above
|
||||
configured
|
||||
if default_fallbacks is None
|
||||
else [*(configured if configured is not None else ()), {"*": default_fallbacks}]
|
||||
[*(configured if configured is not None else ()), {"*": default_fallbacks}]
|
||||
if has_default_fallbacks
|
||||
else configured
|
||||
)
|
||||
content_policy: Final = list(_as_list(get("content_policy_fallbacks"))) or (
|
||||
list(dynamic.content_policy_fallbacks) if dynamic.content_policy_fallbacks else None
|
||||
|
|
@ -561,23 +581,23 @@ class Resolved:
|
|||
if strategy is None and "routing_strategy" in config.arguments
|
||||
else normalize_strategy(strategy) or "simple-shuffle",
|
||||
routing_strategy_args=MappingProxyType(dict(as_str_mapping(get("routing_strategy_args")) or {})),
|
||||
num_retries=int(str(num_retries))
|
||||
num_retries=num_retries
|
||||
if num_retries is not None
|
||||
else dynamic.num_retries
|
||||
if dynamic.num_retries is not None
|
||||
else 2,
|
||||
max_fallbacks=int(str(max_fallbacks))
|
||||
max_fallbacks=max_fallbacks
|
||||
if max_fallbacks is not None
|
||||
else dynamic.max_fallbacks
|
||||
if dynamic.max_fallbacks is not None
|
||||
else dynamic.router_max_fallbacks,
|
||||
allowed_fails=int(str(allowed_fails)) if allowed_fails is not None else dynamic.allowed_fails,
|
||||
explicit_allowed_fails=int(str(allowed_fails)) if allowed_fails is not None else None,
|
||||
allowed_fails=allowed_fails if allowed_fails is not None else dynamic.allowed_fails,
|
||||
explicit_allowed_fails=allowed_fails,
|
||||
cooldown_time=float(str(get("cooldown_time") or _DEFAULT_COOLDOWN_SECONDS)),
|
||||
disable_cooldowns=_optional_bool(get("disable_cooldowns")),
|
||||
allowed_fails_policy=resolved_allowed_fails_policy(get("allowed_fails_policy")),
|
||||
retry_after=float(str(get("retry_after") or 0)),
|
||||
timeout=float(str(timeout)) if timeout else dynamic.request_timeout,
|
||||
retry_after=get("retry_after") if "retry_after" in config.arguments else 0,
|
||||
timeout=timeout if timeout else dynamic.request_timeout,
|
||||
fallbacks=fallbacks,
|
||||
context_window_fallbacks=list(_as_list(get("context_window_fallbacks")))
|
||||
or (list(dynamic.context_window_fallbacks) if dynamic.context_window_fallbacks else None),
|
||||
|
|
@ -628,7 +648,7 @@ class Resolved:
|
|||
if name in {"model_group_alias", "default_litellm_params"}:
|
||||
updates[name] = MappingProxyType(dict(as_str_mapping(value) or {}))
|
||||
elif name in _NUMBER_SETTINGS:
|
||||
updates[name] = self._number_setting(name, value)
|
||||
updates[name] = self._number_setting(name, value, updated=name in updated)
|
||||
if name == "allowed_fails":
|
||||
updates["explicit_allowed_fails"] = updates[name]
|
||||
elif name in _SELECTION_SETTINGS:
|
||||
|
|
@ -648,16 +668,18 @@ class Resolved:
|
|||
updates["routing_group_members"] = None
|
||||
return dataclasses.replace(self, **updates, assigned=self.assigned | frozenset(values)) # pyright: ignore[reportArgumentType] # each value converted to its field's type above
|
||||
|
||||
def _number_setting(self, name: str, value: object) -> object:
|
||||
"""A numeric setting converted to its field's type."""
|
||||
if name in ("num_retries", "max_fallbacks", "allowed_fails"):
|
||||
return int(str(value))
|
||||
def _number_setting(self, name: str, value: object, *, updated: bool = False) -> object:
|
||||
"""A numeric setting converted to its field's type. ``retry_after`` and ``timeout`` are kept
|
||||
as the Python core keeps them: ``update_settings`` stores ``int(value)``, an attribute write
|
||||
the value as given."""
|
||||
if name in ("retry_after", "timeout", "num_retries", "allowed_fails"):
|
||||
return int(value) if updated else value # pyright: ignore[reportArgumentType] # int() of the value as given, as Python casts it
|
||||
if name == "max_fallbacks":
|
||||
return value
|
||||
if name == "cooldown_time":
|
||||
# Stored as given (`_core_cooldown_time`), so a failure cools down for that long:
|
||||
# 0 for no time at all; None leaves the cooldown cache's own default, set by the caller.
|
||||
return float(str(value)) if value is not None else self.cooldown_time
|
||||
if name == "retry_after":
|
||||
return float(str(value or 0))
|
||||
return float(str(value)) if value is not None else None
|
||||
|
||||
def _selection_setting(self, name: str, value: object, *, validate: bool) -> object:
|
||||
|
|
@ -681,8 +703,8 @@ class Resolved:
|
|||
"cooldown_time": self.cooldown_time,
|
||||
"disable_cooldowns": self.disable_cooldowns,
|
||||
"allowed_fails_policy": _plain(self.allowed_fails_policy),
|
||||
"retry_after": self.retry_after,
|
||||
"timeout": self.timeout,
|
||||
"retry_after": float(str(self.retry_after or 0)),
|
||||
"timeout": float(str(self.timeout)) if self.timeout is not None else None,
|
||||
"routing_groups": group_documents(self.routing_groups) if self.routing_groups else None,
|
||||
"enable_pre_call_checks": True if self.enable_pre_call_checks else None,
|
||||
"routing_plugins": True if self.routing_plugins else None,
|
||||
|
|
@ -706,16 +728,16 @@ class Resolved:
|
|||
"fallbacks": _plain(self.fallbacks),
|
||||
"context_window_fallbacks": _plain(self.context_window_fallbacks),
|
||||
"content_policy_fallbacks": _plain(self.content_policy_fallbacks),
|
||||
"num_retries": self.num_retries,
|
||||
"max_fallbacks": self.max_fallbacks,
|
||||
"allowed_fails": self._router_allowed_fails(),
|
||||
"num_retries": _crate_int(self.num_retries),
|
||||
"max_fallbacks": _crate_int(self.max_fallbacks),
|
||||
"allowed_fails": _crate_int(self._router_allowed_fails()),
|
||||
"fallback_keys": self._fallback_keys(model_names),
|
||||
"default_deployment": _plain(served_default_deployment(self.default_deployment)[0]),
|
||||
"default_deployment_raises": True if served_default_deployment(self.default_deployment)[1] else None,
|
||||
}
|
||||
return {name: value for name, value in values.items() if value is not None}
|
||||
|
||||
def _router_allowed_fails(self) -> int | None:
|
||||
def _router_allowed_fails(self) -> object:
|
||||
"""``allowed_fails`` when it counts as set on the router (``_is_allowed_fails_set_on_router``):
|
||||
passed explicitly, or different from the process default."""
|
||||
import litellm
|
||||
|
|
@ -1033,7 +1055,7 @@ class RustCore(
|
|||
# Settings the Python core keeps as plain attributes, and settings deleted with `del`.
|
||||
self._plain: Final = initial_plain(config)
|
||||
self._deleted: frozenset[str] = frozenset()
|
||||
self._request_timeout: Final = request_timeout(config)
|
||||
self._request_timeout: Final = request_timeout(config, host)
|
||||
legacy: Final[dict[str, object]] = {"deployment_stats": {}} # mutable-ok: the core's own legacy attributes
|
||||
self._legacy: Final = legacy
|
||||
self._health_view: RustHealth | None = None
|
||||
|
|
@ -1390,7 +1412,7 @@ class RustCore(
|
|||
else default_litellm_params(self._config),
|
||||
default_max_parallel_requests=self._endpoints.default_max_parallel_requests, # pyright: ignore[reportArgumentType] # the setting as given
|
||||
cooldown_time=self._cooldown_given, # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it
|
||||
allowed_fails=r.allowed_fails,
|
||||
allowed_fails=r.allowed_fails, # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it
|
||||
allowed_fails_policy=r.allowed_fails_policy, # pyright: ignore[reportArgumentType] # the policy as passed
|
||||
disable_cooldowns=r.disable_cooldowns,
|
||||
enable_pre_call_checks=r.enable_pre_call_checks,
|
||||
|
|
@ -2018,8 +2040,8 @@ class RustCore(
|
|||
**self._routing_fields(prepared),
|
||||
**pinned_video_deployment(plan, request),
|
||||
**unrouted_fields(plan),
|
||||
# The sync path never runs pre-routing; a custom strategy picks on both.
|
||||
**({} if plan.sync and self._custom_strategy is None else {"pre_routes": True}),
|
||||
# A custom strategy picks on both paths; the sync path pre-routes when it can.
|
||||
**({"pre_routes": True} if not plan.sync or self._sync_pick_pre_routes() else {}),
|
||||
**({"mock_failure": mock} if mock is not None else {}),
|
||||
# Every target of the walk is refused.
|
||||
**({"fallbacks": []} if rejected_fallback_target(request) is not None else {}),
|
||||
|
|
@ -2389,6 +2411,14 @@ class RustCore(
|
|||
messages,
|
||||
)
|
||||
|
||||
def _sync_pick_pre_routes(self) -> bool:
|
||||
"""Whether a sync pick asks for the ``pre_route`` step: always for a custom strategy, else
|
||||
when a strategy router is registered. A router with routing plugins never pre-routes a sync
|
||||
pick, so router-core refuses it (the pipeline only runs on the async path)."""
|
||||
if self._custom_strategy is not None:
|
||||
return True
|
||||
return not self._resolved.routing_plugins and self._prerouter.has_strategy_routers()
|
||||
|
||||
def _custom_pick_sync(
|
||||
self,
|
||||
step: Mapping[str, object],
|
||||
|
|
@ -2397,10 +2427,15 @@ class RustCore(
|
|||
input: object = _NO_INPUT,
|
||||
specific_deployment: object = _NO_INPUT,
|
||||
) -> object:
|
||||
"""A custom strategy's sync pick (``get_available_deployment``); without one, no pre-routing."""
|
||||
"""A custom strategy's sync pick (``get_available_deployment``); without one, the async
|
||||
pre-routing hook run to completion (``_pre_route``)."""
|
||||
custom: Final = self._custom_strategy
|
||||
if custom is None:
|
||||
return None
|
||||
from litellm.litellm_core_utils import asyncify
|
||||
|
||||
return asyncify.run_async_function( # pyright: ignore[reportUnknownMemberType] # the untyped sync bridge
|
||||
partial(self._pre_route, step, request, messages, input, specific_deployment)
|
||||
)
|
||||
picked: Final[object] = custom.get_available_deployment( # pyright: ignore[reportUnknownMemberType] # the strategy's untyped pick
|
||||
model=str(step.get("model")),
|
||||
messages=request.get("messages") if messages is _NO_INPUT else messages, # pyright: ignore[reportArgumentType] # see _pre_route
|
||||
|
|
@ -2665,9 +2700,6 @@ class RustCore(
|
|||
message: Final = (
|
||||
f"Model {model} has no deployments configured with use_in_pass_through=True. "
|
||||
"Please add use_in_pass_through: true to the deployment configuration"
|
||||
if pass_through == "async_pass_through"
|
||||
else f"Model {model} has no deployment configured with use_in_pass_through=True. "
|
||||
"Please add use_in_pass_through: true in the deployment configuration"
|
||||
)
|
||||
return litellm_exceptions.BadRequestError(message=message, model=model, llm_provider="")
|
||||
if kind == "context_window_exceeded":
|
||||
|
|
@ -2805,7 +2837,7 @@ class _Selection:
|
|||
fields["specific_deployment"] = True
|
||||
if probe:
|
||||
fields["health_check_probe"] = True
|
||||
if kind == "pick" or (kind == "sync_pick" and core._custom_strategy is not None): # pyright: ignore[reportPrivateUsage] # the selection serves its own core
|
||||
if kind == "pick" or (kind == "sync_pick" and core._sync_pick_pre_routes()): # pyright: ignore[reportPrivateUsage] # the selection serves its own core
|
||||
fields["pre_routes"] = True
|
||||
self._specific_deployment: Final = specific_deployment
|
||||
# The error a callback filter raised, which a failed selection raises.
|
||||
|
|
@ -3262,15 +3294,35 @@ class _Driver:
|
|||
"""A routing pre-call check refused the attempt (``async_routing_strategy_pre_call_checks``):
|
||||
a rate limit cools the deployment down (``set_cooldown_deployments``); the attempt's failure
|
||||
is recorded by nothing else."""
|
||||
report: Final = self._refusal_report(deployment, error)
|
||||
if report is not None:
|
||||
await self._core._drive_cache_steps(report) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
|
||||
return {**self._failed(step, deployment, call, error), "recorded": True}
|
||||
|
||||
def _refused_sync(
|
||||
self,
|
||||
step: Mapping[str, object],
|
||||
deployment: Mapping[str, object],
|
||||
call: "AttemptCall",
|
||||
error: BaseException,
|
||||
) -> Mapping[str, object]:
|
||||
"""``_refused`` on the sync path (``routing_strategy_pre_call_checks``): a rate limit cools
|
||||
the deployment down here too."""
|
||||
report: Final = self._refusal_report(deployment, error)
|
||||
if report is not None:
|
||||
self._core._drive_sync_steps(report) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
|
||||
return {**self._failed(step, deployment, call, error), "recorded": True}
|
||||
|
||||
def _refusal_report(self, deployment: Mapping[str, object], error: BaseException) -> "RouterCall | None":
|
||||
"""The uncounted failure report that cools down a deployment a pre-call check rate-limited."""
|
||||
import litellm.exceptions as litellm_exceptions
|
||||
|
||||
core: Final = self._core
|
||||
if isinstance(error, litellm_exceptions.RateLimitError):
|
||||
deployment_id: Final = (as_str_mapping(deployment.get("model_info")) or {}).get("id")
|
||||
if isinstance(deployment_id, str):
|
||||
report: Final = core._handle.report_failure(deployment_id, dict(attempt_failure(error)), uncounted=True) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
|
||||
await core._drive_cache_steps(report) # pyright: ignore[reportPrivateUsage] # see above
|
||||
return {**self._failed(step, deployment, call, error), "recorded": True}
|
||||
if not isinstance(error, litellm_exceptions.RateLimitError):
|
||||
return None
|
||||
deployment_id: Final = (as_str_mapping(deployment.get("model_info")) or {}).get("id")
|
||||
if not isinstance(deployment_id, str):
|
||||
return None
|
||||
return self._core._handle.report_failure(deployment_id, dict(attempt_failure(error)), uncounted=True) # pyright: ignore[reportPrivateUsage] # the driver serves its own core
|
||||
|
||||
def _pick_kwargs(self, step: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""The request kwargs the pick sees: the request with its level's retry count."""
|
||||
|
|
@ -3365,8 +3417,7 @@ class _Driver:
|
|||
try:
|
||||
self._core._endpoints.sync_pre_call_checks(self._plan, deployment, self._request) # pyright: ignore[reportPrivateUsage] # see above
|
||||
except Exception as refused: # noqa: BLE001 # handed to the Rust call loop like any attempt failure
|
||||
# The sync path records no outcome for a refused pre-call check: no cooldown.
|
||||
return {**self._failed(step, deployment, call, refused), "recorded": True}
|
||||
return self._refused_sync(step, deployment, call, refused)
|
||||
answered: Final = self._core._host.invoke(call.host_op)(**call.kwargs) # pyright: ignore[reportPrivateUsage] # a sync endpoint
|
||||
if inspect.isawaitable(answered):
|
||||
raise _NotSynchronous(f"the host's {call.host_op} endpoint is not synchronous")
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ Kind = Literal[ # rebind-ok: rebound as the Python router rebinds it
|
|||
"search",
|
||||
]
|
||||
Stamp = Literal["deployment", "retry_skip", "none"] # rebind-ok: rebound as the Python router rebinds it
|
||||
CountKey = Literal["deployment", "group", "empty"] # rebind-ok: rebound as the Python router rebinds it
|
||||
CountKey = Literal["deployment", "group"] # rebind-ok: rebound as the Python router rebinds it
|
||||
|
||||
# Request keys the router reads to route the call; the deployment call never receives them.
|
||||
ROUTING_KEYS: Final = frozenset(
|
||||
|
|
@ -174,9 +174,6 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType(
|
|||
sync=True,
|
||||
splits_stream_timeout=True,
|
||||
stamp="deployment",
|
||||
total_key=None,
|
||||
success_key=None,
|
||||
fail_key=None,
|
||||
named=("messages",),
|
||||
),
|
||||
"atext_completion": Plan(
|
||||
|
|
@ -187,6 +184,15 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType(
|
|||
fail_key="group",
|
||||
named=("prompt",),
|
||||
),
|
||||
"text_completion": Plan(
|
||||
"text_completion",
|
||||
"text_completion",
|
||||
"text",
|
||||
sync=True,
|
||||
splits_stream_timeout=True,
|
||||
fail_key="group",
|
||||
named=("prompt",),
|
||||
),
|
||||
# `_asearch_with_fallbacks`: the search tool name routes as the model, with no deployment
|
||||
# picked; its helper picks among the router's search tools.
|
||||
"asearch": Plan(
|
||||
|
|
@ -208,9 +214,6 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType(
|
|||
"image",
|
||||
sync=True,
|
||||
prepare_field=None,
|
||||
total_key="empty",
|
||||
success_key="empty",
|
||||
fail_key="empty",
|
||||
named=("prompt",),
|
||||
),
|
||||
"atranscription": Plan(
|
||||
|
|
@ -379,7 +382,7 @@ class Counters:
|
|||
|
||||
@staticmethod
|
||||
def _key(which: CountKey, model_name: str, group: str) -> str:
|
||||
return model_name if which == "deployment" else group if which == "group" else ""
|
||||
return model_name if which == "deployment" else group
|
||||
|
||||
def started(self, plan: Plan, call: AttemptCall) -> None:
|
||||
if plan.total_key is not None:
|
||||
|
|
@ -492,9 +495,7 @@ def attempt_call(
|
|||
function_name="generic_api_call",
|
||||
splits_stream_timeout=plan.splits_stream_timeout,
|
||||
)
|
||||
endpoint = kwargs.get("endpoint") # rebind-ok: rebound as the Python router rebinds it
|
||||
if endpoint and isinstance(endpoint, str):
|
||||
kwargs["endpoint"] = endpoint.replace(requested, model_name)
|
||||
rewrite_endpoint(core, kwargs, requested, model_name)
|
||||
kwargs["model"] = model_name
|
||||
final = { # rebind-ok: rebound as the Python router rebinds it
|
||||
**data,
|
||||
|
|
@ -516,6 +517,33 @@ def attempt_call(
|
|||
return AttemptCall(plan.host_op, kwargs, kwargs, requested, group)
|
||||
|
||||
|
||||
def rewrite_endpoint(
|
||||
core: "RustCore",
|
||||
kwargs: dict[str, object], # mutable-ok: the attempt's own kwargs, shaped in place
|
||||
requested: str,
|
||||
model_name: str,
|
||||
) -> None:
|
||||
"""`_add_deployment_model_to_endpoint_for_llm_passthrough_route`: the requested name, as a whole
|
||||
path segment of the request's endpoint, becomes the deployment's model without its provider."""
|
||||
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
|
||||
|
||||
endpoint: Final = kwargs.get("endpoint")
|
||||
if not endpoint:
|
||||
return
|
||||
try:
|
||||
stripped, _, _, _ = core._host.invoke("get_llm_provider")( # pyright: ignore[reportPrivateUsage] # the core's host
|
||||
model=model_name,
|
||||
custom_llm_provider=kwargs.get("custom_llm_provider"),
|
||||
api_base=kwargs.get("api_base"),
|
||||
)
|
||||
replacement = str(stripped) # rebind-ok: rebound as the Python router rebinds it
|
||||
except Exception: # noqa: BLE001 # as the Python router: an unresolvable model is used as is
|
||||
replacement = model_name # rebind-ok: rebound as the Python router rebinds it
|
||||
kwargs["endpoint"] = replace_path_segment( # rebind-ok: the attempt's kwargs, shaped in place
|
||||
str(endpoint), requested, replacement
|
||||
)
|
||||
|
||||
|
||||
def _generic_attempt(
|
||||
core: "RustCore",
|
||||
plan: Plan,
|
||||
|
|
@ -526,7 +554,6 @@ def _generic_attempt(
|
|||
group: str,
|
||||
) -> AttemptCall:
|
||||
"""`_ageneric_api_call_with_fallbacks_helper` for one deployment."""
|
||||
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
|
||||
from litellm.router_rust_websocket import turn_admission
|
||||
|
||||
socket: Final = plan.call_type == "_aresponses_websocket"
|
||||
|
|
@ -540,20 +567,7 @@ def _generic_attempt(
|
|||
opens_responses_socket=socket,
|
||||
)
|
||||
model_name: Final = str(data.get("model"))
|
||||
endpoint: Final = kwargs.get("endpoint")
|
||||
if endpoint:
|
||||
try:
|
||||
stripped, _, _, _ = core._host.invoke("get_llm_provider")( # pyright: ignore[reportPrivateUsage] # the core's host
|
||||
model=model_name,
|
||||
custom_llm_provider=kwargs.get("custom_llm_provider"),
|
||||
api_base=kwargs.get("api_base"),
|
||||
)
|
||||
replacement = str(stripped) # rebind-ok: rebound as the Python router rebinds it
|
||||
except Exception: # noqa: BLE001 # as the Python router: an unresolvable model is used as is
|
||||
replacement = model_name # rebind-ok: rebound as the Python router rebinds it
|
||||
kwargs["endpoint"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them
|
||||
replace_path_segment(str(endpoint), requested, replacement)
|
||||
)
|
||||
rewrite_endpoint(core, kwargs, requested, model_name)
|
||||
provider: Final = _generic_provider(data)
|
||||
final: Final = {
|
||||
**without_retrieve_stream_defaults(data, kwargs),
|
||||
|
|
@ -604,14 +618,14 @@ def model_client(
|
|||
core: "RustCore", plan: Plan, deployment: Mapping[str, object], kwargs: Mapping[str, object]
|
||||
) -> object:
|
||||
"""The SDK client cached for the deployment (`_get_async_openai_model_client`; `_get_client`
|
||||
with the sync keys for sync completion and embedding), dropped when the request carries
|
||||
with the sync keys for sync completion, text completion and embedding), dropped when the request carries
|
||||
another api key."""
|
||||
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
|
||||
|
||||
if as_dict(deployment.get("litellm_params")).get("kubernetes_pod_discovery") is True:
|
||||
return None
|
||||
model_id: Final = as_dict(deployment.get("model_info")).get("id")
|
||||
sync_keys: Final = plan.sync and plan.kind in ("chat", "embedding")
|
||||
sync_keys: Final = plan.sync and plan.kind in ("chat", "embedding", "text")
|
||||
stream: Final = "stream_" if kwargs.get("stream") is True else ""
|
||||
key: Final = f"{model_id}_{stream}client" if sync_keys else f"{model_id}_{stream}async_client"
|
||||
span: Final = get_parent_otel_span_from_kwargs(dict(kwargs))
|
||||
|
|
@ -832,14 +846,13 @@ class RustEndpoints:
|
|||
"""The public endpoint methods of one Rust core, beyond the routed call loop the core drives."""
|
||||
|
||||
def __init__(self, core: "RustCore", config: "RouterConfig") -> None:
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
|
||||
from litellm.router_utils.request_priority import InvalidPriority, parse_default_priority
|
||||
|
||||
self._core: Final = core
|
||||
self.counters: Final = Counters()
|
||||
explicit: Final = config.get("timeout")
|
||||
self._explicit_timeout: Final = explicit
|
||||
self._request_timeout: Final = get_configured_request_timeout() if explicit is not None else None
|
||||
self._request_timeout: Final = self.process_request_timeout() if explicit is not None else None
|
||||
self._stream_timeout: Final = config.get("stream_timeout")
|
||||
# apply() flips it after construction, as on the Python core
|
||||
self._cache_responses: bool = bool(config.get("cache_responses"))
|
||||
|
|
@ -927,8 +940,16 @@ class RustEndpoints:
|
|||
request_timeout=self._request_timeout,
|
||||
stream_timeout=self._stream_timeout,
|
||||
cache_responses=self._cache_responses,
|
||||
process_request_timeout=self.process_request_timeout,
|
||||
)
|
||||
|
||||
def process_request_timeout(self) -> float | None:
|
||||
"""The host's explicitly configured process ``request_timeout``, else None."""
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import configured_request_timeout
|
||||
|
||||
dynamic: Final = self._core._host.dynamic() # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core
|
||||
return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set)
|
||||
|
||||
def default_litellm_params(self, resolved: "Resolved") -> Mapping[str, object]:
|
||||
"""The defaults the Python constructor completes before any call: the router ``timeout``
|
||||
argument, no client retries, and the caching groups in the default metadata."""
|
||||
|
|
@ -1495,30 +1516,12 @@ class RustEndpoints:
|
|||
return await self._direct("amoderation", prepared)
|
||||
|
||||
def text_completion(self, request: Mapping[str, object]) -> object:
|
||||
"""The sync `text_completion`: one pick, no retries, the group name as the model."""
|
||||
from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin
|
||||
|
||||
kwargs: Final = {k: v for k, v in request.items() if k not in ("is_retry", "is_fallback", "is_async")}
|
||||
model: Final = kwargs.get("model")
|
||||
prompt: Final = kwargs.get("prompt")
|
||||
"""The sync `text_completion`: routed, retried and fallen back as `atext_completion` is."""
|
||||
model: Final = request.get("model")
|
||||
if not isinstance(model, str):
|
||||
raise TypeError("text_completion() missing 1 required positional argument: 'model'")
|
||||
kwargs["num_retries"] = kwargs.get("num_retries", self._core._resolved.num_retries) # pyright: ignore[reportPrivateUsage] # see above
|
||||
metadata_bucket(kwargs, "metadata").update({"model_group": model})
|
||||
specific: Final = kwargs.pop("specific_deployment", None)
|
||||
deployment: Final = self.select_sync(model, kwargs, specific=bool(specific))
|
||||
data: Final = as_dict(deployment.get("litellm_params"))
|
||||
for key, value in self._core._resolved.default_litellm_params.items(): # pyright: ignore[reportPrivateUsage] # see above
|
||||
if key not in kwargs:
|
||||
kwargs[key] = value
|
||||
elif key == "metadata":
|
||||
bucket = kwargs[key]
|
||||
if isinstance(bucket, dict):
|
||||
bucket.update(as_dict(value)) # pyright: ignore[reportUnknownMemberType] # the request's metadata dict
|
||||
pinned: Final = deployment_model_id_pin(data)
|
||||
if pinned is not None:
|
||||
kwargs["model_id"] = pinned
|
||||
return self._invoke("text_completion")(**{**data, "prompt": prompt, "caching": self._cache_responses, **kwargs})
|
||||
kwargs: Final = {k: v for k, v in request.items() if k not in ("is_retry", "is_fallback", "is_async")}
|
||||
return self._core._call_sync(PLANS["text_completion"], kwargs) # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core
|
||||
|
||||
def select_sync(self, model: str, request: Mapping[str, object], *, specific: bool = False) -> Mapping[str, object]:
|
||||
"""Pick a deployment for ``model`` on the sync path (`get_available_deployment`)."""
|
||||
|
|
|
|||
|
|
@ -705,6 +705,10 @@ class PreRouter:
|
|||
|
||||
# -- the step --------------------------------------------------------------------------------
|
||||
|
||||
def has_strategy_routers(self) -> bool:
|
||||
"""Whether any auto, complexity, adaptive or quality router is registered."""
|
||||
return self._strategies.any_registered()
|
||||
|
||||
async def pre_route(
|
||||
self,
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -669,7 +669,7 @@ class HostRegistry(HostModelInfo):
|
|||
info_dict: UntypedEntry = model_info if isinstance(model_info, dict) else {} # pyright: ignore[reportAssignmentType] # a configured model_info
|
||||
declared_id = None if info_dict.get("id") is None else str(info_dict["id"])
|
||||
if "id" not in info_dict:
|
||||
info_dict["id"] = deployment_id(str(model_name), litellm_params)
|
||||
info_dict["id"] = deployment_id(model_name, litellm_params) # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it
|
||||
organization = litellm_params.get("organization")
|
||||
if isinstance(organization, list):
|
||||
# Each organization's deployment has its own id (``organization_deployment_id``).
|
||||
|
|
@ -677,7 +677,7 @@ class HostRegistry(HostModelInfo):
|
|||
litellm_params["organization"] = org
|
||||
self._create_deployment(
|
||||
model,
|
||||
str(model_name),
|
||||
model_name, # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it
|
||||
litellm_params,
|
||||
{
|
||||
**info_dict,
|
||||
|
|
@ -685,7 +685,7 @@ class HostRegistry(HostModelInfo):
|
|||
declared_id,
|
||||
position,
|
||||
org,
|
||||
lambda: deployment_id(str(model_name), litellm_params),
|
||||
lambda: deployment_id(model_name, litellm_params), # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it
|
||||
),
|
||||
},
|
||||
declared_id=declared_id,
|
||||
|
|
@ -694,12 +694,14 @@ class HostRegistry(HostModelInfo):
|
|||
else:
|
||||
self._create_deployment(
|
||||
model,
|
||||
str(model_name),
|
||||
model_name, # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it
|
||||
litellm_params,
|
||||
info_dict,
|
||||
declared_id=declared_id,
|
||||
duplicate_ids=duplicate_ids,
|
||||
)
|
||||
# Python logs the group names here, which reads every alias entry (a malformed one raises).
|
||||
verbose_router_logger.debug("\nInitialized Model List %s", self._core_get_model_names())
|
||||
self._core_model_names = {str(_LITELLM_PARAMS_ADAPTER.validate_python(m).get("model_name")) for m in model_list}
|
||||
if self.strategies is not None:
|
||||
self.strategies.finalize()
|
||||
|
|
|
|||
|
|
@ -281,13 +281,14 @@ def pick_sync(
|
|||
input: object,
|
||||
request_kwargs: dict[str, object] | None, # mutable-ok: see pick_async
|
||||
) -> object:
|
||||
"""``_select_deployment_sync``: cost-based routing has no sync pick."""
|
||||
"""``_select_deployment_sync``."""
|
||||
match selector:
|
||||
case LeastBusyLoggingHandler() if strategy == "least-busy":
|
||||
return selector.get_available_deployments(model_group=model, healthy_deployments=deployments) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts
|
||||
case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() if strategy in (
|
||||
case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() | LowestCostLoggingHandler() if strategy in (
|
||||
"usage-based-routing",
|
||||
"usage-based-routing-v2",
|
||||
"cost-based-routing",
|
||||
):
|
||||
return selector.get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts
|
||||
model_group=model,
|
||||
|
|
|
|||
|
|
@ -184,11 +184,15 @@ def _as_dict(value: object) -> Mapping[str, object] | None:
|
|||
return value if isinstance(value, dict) else None # pyright: ignore[reportUnknownVariableType] # the argument dict, as passed
|
||||
|
||||
|
||||
def request_timeout(config: "RouterConfig") -> float | None:
|
||||
"""The per-attempt ``request_timeout``: only stored when a router timeout is also set."""
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
|
||||
def request_timeout(config: "RouterConfig", host: "PythonHost") -> float | None:
|
||||
"""The per-attempt ``request_timeout``: the host's configured process ``request_timeout``, only
|
||||
stored when a router timeout is also set."""
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import configured_request_timeout
|
||||
|
||||
return get_configured_request_timeout() if config.get("timeout") is not None else None
|
||||
if config.get("timeout") is None:
|
||||
return None
|
||||
dynamic: Final = host.dynamic()
|
||||
return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set)
|
||||
|
||||
|
||||
def apply_construction_effects(config: "RouterConfig", host: "PythonHost") -> None:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ the same precedence and the same oddities. Nothing here reads a Python router.
|
|||
"""
|
||||
|
||||
import dataclasses
|
||||
from collections.abc import Mapping, MutableMapping, Sequence
|
||||
from collections.abc import Callable, Mapping, MutableMapping, Sequence
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
|
@ -62,6 +62,8 @@ _FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt", "anthro
|
|||
_ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params"
|
||||
# Generic handlers whose stream timeout is split from the request timeout.
|
||||
STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
|
||||
# The generic calls (sync and async) resolve their timeout with the LLM pass-through resolver.
|
||||
GENERIC_CALL_FUNCTION_NAMES: Final = frozenset({"_ageneric_api_call_with_fallbacks", "generic_api_call"})
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
|
|
@ -70,7 +72,8 @@ class Shaping:
|
|||
|
||||
``timeout`` is the router's ``timeout`` (its own setting, else the process ``request_timeout``),
|
||||
``request_timeout`` the explicitly configured process ``request_timeout``, kept only when the
|
||||
router has a timeout of its own, as the Python router keeps them."""
|
||||
router has a timeout of its own, as the Python router keeps them; ``process_request_timeout``
|
||||
reads the host's explicitly configured process ``request_timeout`` at call time."""
|
||||
|
||||
default_litellm_params: Mapping[str, object]
|
||||
timeout: object
|
||||
|
|
@ -78,6 +81,7 @@ class Shaping:
|
|||
request_timeout: float | None
|
||||
stream_timeout: object
|
||||
cache_responses: bool
|
||||
process_request_timeout: Callable[[], float | None] = lambda: None
|
||||
|
||||
|
||||
def as_dict(value: object) -> dict[str, object]: # mutable-ok: a fresh dict of a request mapping
|
||||
|
|
@ -532,15 +536,17 @@ def _set_timeouts(
|
|||
function_name: str | None,
|
||||
splits_stream_timeout: bool,
|
||||
) -> None:
|
||||
if function_name == "_ageneric_api_call_with_fallbacks":
|
||||
if function_name in GENERIC_CALL_FUNCTION_NAMES:
|
||||
from litellm.passthrough.timeout_utils import resolve_llm_passthrough_timeout
|
||||
|
||||
# The process request_timeout, read through the host, ranks right after the router timeout,
|
||||
# as resolve_llm_passthrough_timeout ranks it.
|
||||
router_timeout: Final = (
|
||||
shaping.request_timeout
|
||||
if shaping.request_timeout is not None
|
||||
else float(shaping.explicit_timeout)
|
||||
if isinstance(shaping.explicit_timeout, (int, float))
|
||||
else None
|
||||
else shaping.process_request_timeout()
|
||||
)
|
||||
router_stream_timeout: Final = (
|
||||
shaping.stream_timeout
|
||||
|
|
|
|||
|
|
@ -55,7 +55,8 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
|||
)
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, set_hidden_params
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.types.utils import ModelResponseStream, Usage
|
||||
from litellm.router_utils.fallback_event_handlers import non_stream_fallback_answer_as_chunk
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
|
|
@ -617,6 +618,17 @@ def _adopt_fallback_response_headers(
|
|||
return prepared
|
||||
|
||||
|
||||
def _non_stream_fallback_item(
|
||||
wrapper_ref: "weakref.ref[FallbackAwareStreamWrapper]", fallback_response: object
|
||||
) -> object:
|
||||
"""The one item a stream yields for a fallback that answered without a stream: its response as
|
||||
a stream chunk, carrying the fallback's hidden params and headers."""
|
||||
prepared: Final = _adopt_fallback_response_headers(wrapper_ref, fallback_response)
|
||||
item: Final = non_stream_fallback_answer_as_chunk(fallback_response)
|
||||
_apply_fallback_hidden_params_to_item(item, prepared)
|
||||
return item
|
||||
|
||||
|
||||
def _chat_wrapper_init(
|
||||
wrapper: FallbackAwareStreamWrapper, stream: object, model_response: CustomStreamWrapper
|
||||
) -> None:
|
||||
|
|
@ -748,8 +760,7 @@ def wrap_chat_stream( # noqa: C901 # one function per Python wrapper it ports
|
|||
)
|
||||
yield fallback_item
|
||||
else:
|
||||
# A fallback that answers without a stream yields None (an oddity kept as is).
|
||||
yield None
|
||||
yield _non_stream_fallback_item(wrapper_ref, fallback_response)
|
||||
except Exception as fallback_error:
|
||||
verbose_router_logger.error("Fallback also failed: %s", fallback_error)
|
||||
if isinstance(trigger, MidStreamFallbackError) and trigger.original_exception is not None:
|
||||
|
|
@ -801,7 +812,7 @@ def wrap_chat_stream_sync(
|
|||
trigger: Final = _fallback_trigger(e, (litellm_exceptions.ContextWindowExceededError,))
|
||||
try:
|
||||
fallback_response = attempt.fallback_sync(trigger) # rebind-ok: rebound as the stream advances
|
||||
if hasattr(fallback_response, "__iter__"):
|
||||
if hasattr(fallback_response, "__iter__") and not isinstance(fallback_response, ModelResponse):
|
||||
prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) # rebind-ok: re-adopted
|
||||
settled = False # rebind-ok: rebound as the stream advances
|
||||
for fallback_item in cast("Iterable[object]", fallback_response): # cast-ok: checked
|
||||
|
|
@ -817,7 +828,7 @@ def wrap_chat_stream_sync(
|
|||
_combine_fallback_usage(fallback_item, None)
|
||||
yield fallback_item
|
||||
else:
|
||||
yield None
|
||||
yield _non_stream_fallback_item(wrapper_ref, fallback_response)
|
||||
except Exception as fallback_error:
|
||||
verbose_router_logger.error("Fallback also failed: %s", fallback_error)
|
||||
if isinstance(trigger, MidStreamFallbackError) and trigger.original_exception is not None:
|
||||
|
|
|
|||
|
|
@ -209,15 +209,32 @@ class LowestCostLoggingHandler(CustomLogger):
|
|||
"""
|
||||
Returns a deployment with the lowest cost
|
||||
"""
|
||||
cost_key: Final = f"cost_map:{model_group}"
|
||||
request_count_dict: Final = await self.router_cache.async_get_cache(key=f"cost_map:{model_group}") or {} # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
|
||||
return self._lowest_cost_deployment(request_count_dict, healthy_deployments, messages, input) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
|
||||
|
||||
request_count_dict: Final = await self.router_cache.async_get_cache(key=cost_key) or {}
|
||||
|
||||
# -----------------------
|
||||
# Find lowest used model
|
||||
# ----------------------
|
||||
float("inf")
|
||||
@with_service_target("router_usage")
|
||||
def get_available_deployments( # pyright: ignore[reportUnknownParameterType] # noqa: ANN201 # the selector's untyped deployment dict, as the async twin returns it
|
||||
self,
|
||||
model_group: str,
|
||||
healthy_deployments: list, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the router's list of deployment dicts, as the async twin takes it
|
||||
messages: list[dict[str, str]] | None = None, # mutable-ok: the request's messages, as given
|
||||
input: str | list | None = None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the request's input, as given
|
||||
request_kwargs: dict | None = None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: unused, the same signature as the async twin
|
||||
):
|
||||
"""
|
||||
The sync twin of `async_get_available_deployments`: the deployment with the lowest cost
|
||||
"""
|
||||
request_count_dict: Final = self.router_cache.get_cache(key=f"cost_map:{model_group}") or {} # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
|
||||
return self._lowest_cost_deployment(request_count_dict, healthy_deployments, messages, input) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals
|
||||
|
||||
def _lowest_cost_deployment( # pyright: ignore[reportUnknownParameterType] # noqa: ANN202 # the selector's untyped deployment dict
|
||||
self,
|
||||
request_count_dict: dict | None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the untyped cached usage map, filled in place as before
|
||||
healthy_deployments: list, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the router's list of deployment dicts
|
||||
messages: list[dict[str, str]] | None, # mutable-ok: the request's messages, as given
|
||||
input: str | list | None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the request's input, as given
|
||||
):
|
||||
"""The cheapest healthy deployment still under its tpm/rpm limits this minute, else None."""
|
||||
current_date: Final = datetime.now().strftime("%Y-%m-%d")
|
||||
current_hour: Final = datetime.now().strftime("%H")
|
||||
current_minute: Final = datetime.now().strftime("%M")
|
||||
|
|
@ -296,8 +313,8 @@ class LowestCostLoggingHandler(CustomLogger):
|
|||
verbose_router_logger.debug(
|
||||
"item_cost: %s, item_tpm: %s, item_rpm: %s, model_id: %s",
|
||||
item_cost,
|
||||
item_tpm,
|
||||
item_rpm,
|
||||
item_tpm, # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage map
|
||||
item_rpm, # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage map
|
||||
_deployment.get("model_info", {}).get("id"),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -469,6 +469,68 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
|
|||
[f"{prefix}:rpm:{current_minute}" for prefix in prefixes],
|
||||
)
|
||||
|
||||
def _no_deployments_available_error(
|
||||
self,
|
||||
model_group: str,
|
||||
healthy_deployments: Sequence[object],
|
||||
tpm_values: Sequence[object] | None,
|
||||
rpm_values: Sequence[object] | None,
|
||||
) -> litellm.RateLimitError:
|
||||
"""The error both the sync and the async pick raise when no deployment is under its limits."""
|
||||
### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ###
|
||||
deployment_dict: Final = {}
|
||||
for index, _deployment in enumerate(healthy_deployments):
|
||||
if isinstance(_deployment, dict):
|
||||
id = _deployment.get("model_info", {}).get("id")
|
||||
### GET DEPLOYMENT TPM LIMIT ###
|
||||
_deployment_tpm = None
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("model_info", {}).get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = float("inf")
|
||||
|
||||
### GET CURRENT TPM ###
|
||||
current_tpm = tpm_values[index] if tpm_values else 0
|
||||
|
||||
### GET DEPLOYMENT TPM LIMIT ###
|
||||
_deployment_rpm = None
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("model_info", {}).get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = float("inf")
|
||||
|
||||
### GET CURRENT RPM ###
|
||||
current_rpm = rpm_values[index] if rpm_values else 0
|
||||
|
||||
deployment_dict[id] = {
|
||||
"current_tpm": current_tpm,
|
||||
"tpm_limit": _deployment_tpm,
|
||||
"current_rpm": current_rpm,
|
||||
"rpm_limit": _deployment_rpm,
|
||||
}
|
||||
return litellm.RateLimitError(
|
||||
message=f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}",
|
||||
llm_provider="",
|
||||
model=model_group,
|
||||
response=httpx.Response(
|
||||
status_code=429,
|
||||
content="",
|
||||
headers={"retry-after": str(60)},
|
||||
request=httpx.Request(
|
||||
method="tpm_rpm_limits",
|
||||
url="https://github.com/BerriAI/litellm",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@with_service_target("router_usage")
|
||||
async def async_get_available_deployments(
|
||||
self,
|
||||
|
|
@ -520,63 +582,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
|
|||
input=input,
|
||||
)
|
||||
|
||||
try:
|
||||
assert deployment is not None
|
||||
return deployment
|
||||
except Exception:
|
||||
### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ###
|
||||
deployment_dict: Final = {}
|
||||
for index, _deployment in enumerate(healthy_deployments):
|
||||
if isinstance(_deployment, dict):
|
||||
id = _deployment.get("model_info", {}).get("id")
|
||||
### GET DEPLOYMENT TPM LIMIT ###
|
||||
_deployment_tpm = None
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("model_info", {}).get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = float("inf")
|
||||
|
||||
### GET CURRENT TPM ###
|
||||
current_tpm = tpm_values[index] if tpm_values else 0
|
||||
|
||||
### GET DEPLOYMENT TPM LIMIT ###
|
||||
_deployment_rpm = None
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("model_info", {}).get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = float("inf")
|
||||
|
||||
### GET CURRENT RPM ###
|
||||
current_rpm = rpm_values[index] if rpm_values else 0
|
||||
|
||||
deployment_dict[id] = {
|
||||
"current_tpm": current_tpm,
|
||||
"tpm_limit": _deployment_tpm,
|
||||
"current_rpm": current_rpm,
|
||||
"rpm_limit": _deployment_rpm,
|
||||
}
|
||||
raise litellm.RateLimitError(
|
||||
message=f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}",
|
||||
llm_provider="",
|
||||
model=model_group,
|
||||
response=httpx.Response(
|
||||
status_code=429,
|
||||
content="",
|
||||
headers={"retry-after": str(60)},
|
||||
request=httpx.Request(
|
||||
method="tpm_rpm_limits",
|
||||
url="https://github.com/BerriAI/litellm",
|
||||
),
|
||||
),
|
||||
)
|
||||
if deployment is None:
|
||||
raise self._no_deployments_available_error(model_group, healthy_deployments, tpm_values, rpm_values) # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage lists
|
||||
return deployment
|
||||
|
||||
@with_service_target("router_usage")
|
||||
def get_available_deployments(
|
||||
|
|
@ -631,49 +639,6 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
|
|||
input=input,
|
||||
)
|
||||
|
||||
try:
|
||||
assert deployment is not None
|
||||
return deployment
|
||||
except Exception:
|
||||
### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ###
|
||||
deployment_dict: Final = {}
|
||||
for index, _deployment in enumerate(healthy_deployments):
|
||||
if isinstance(_deployment, dict):
|
||||
id = _deployment.get("model_info", {}).get("id")
|
||||
### GET DEPLOYMENT TPM LIMIT ###
|
||||
_deployment_tpm = None
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("model_info", {}).get("tpm", None)
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = float("inf")
|
||||
|
||||
### GET CURRENT TPM ###
|
||||
current_tpm = tpm_values[index] if tpm_values else 0
|
||||
|
||||
### GET DEPLOYMENT TPM LIMIT ###
|
||||
_deployment_rpm = None
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("model_info", {}).get("rpm", None)
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = float("inf")
|
||||
|
||||
### GET CURRENT RPM ###
|
||||
current_rpm = rpm_values[index] if rpm_values else 0
|
||||
|
||||
deployment_dict[id] = {
|
||||
"current_tpm": current_tpm,
|
||||
"tpm_limit": _deployment_tpm,
|
||||
"current_rpm": current_rpm,
|
||||
"rpm_limit": _deployment_rpm,
|
||||
}
|
||||
raise ValueError(
|
||||
f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}"
|
||||
)
|
||||
if deployment is None:
|
||||
raise self._no_deployments_available_error(model_group, healthy_deployments, tpm_values, rpm_values) # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage lists
|
||||
return deployment
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
|
|||
"_acompletion": "/v1/chat/completions",
|
||||
"text_completion": "/v1/completions",
|
||||
"atext_completion": "/v1/completions",
|
||||
"_text_completion": "/v1/completions",
|
||||
"_atext_completion": "/v1/completions",
|
||||
"anthropic_messages": "/v1/messages",
|
||||
"aresponses": "/v1/responses",
|
||||
|
|
|
|||
|
|
@ -1013,3 +1013,15 @@ _check_non_standard_fallback_format = check_non_standard_fallback_format
|
|||
|
||||
def run_non_standard_fallback_format(fallbacks: Sequence[str] | Sequence[Mapping[str, object]], model_group: str):
|
||||
pass
|
||||
|
||||
|
||||
def non_stream_fallback_answer_as_chunk(fallback_response: object) -> object:
|
||||
"""A mid-stream fallback that answered without a stream: the whole response as the one stream
|
||||
chunk it is, as a mocked stream sends a full response (`convert_model_response_to_streaming`).
|
||||
Anything that is not a chat response is yielded as it is."""
|
||||
from litellm.llms.base_llm.base_model_iterator import convert_model_response_to_streaming
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
if isinstance(fallback_response, ModelResponse):
|
||||
return convert_model_response_to_streaming(fallback_response)
|
||||
return fallback_response
|
||||
|
|
|
|||
|
|
@ -632,11 +632,10 @@ def test_select_deployment_sync_returns_none_without_selector():
|
|||
assert result is None
|
||||
|
||||
|
||||
def test_select_deployment_sync_does_not_dispatch_cost_based_routing():
|
||||
def test_select_deployment_sync_dispatches_cost_based_routing():
|
||||
"""
|
||||
`LowestCostLoggingHandler` only implements `async_get_available_deployments`,
|
||||
so the sync dispatch must fall through to None for `cost-based-routing`
|
||||
instead of raising AttributeError.
|
||||
`LowestCostLoggingHandler` has a sync `get_available_deployments` as well, so the sync dispatch
|
||||
picks with it for `cost-based-routing` as the async dispatch does.
|
||||
"""
|
||||
router = _build_router(
|
||||
routing_strategy="simple-shuffle",
|
||||
|
|
@ -649,21 +648,19 @@ def test_select_deployment_sync_does_not_dispatch_cost_based_routing():
|
|||
],
|
||||
)
|
||||
cost_selector = router._group_selectors["cheap"]["cost-based-routing"]
|
||||
assert not hasattr(cost_selector, "get_available_deployments"), (
|
||||
"test premise broken: LowestCostLoggingHandler unexpectedly grew a "
|
||||
"sync method — revisit the sync dispatch arm."
|
||||
)
|
||||
healthy = router._core_get_model_list(model_name="filtered-model") # pyright: ignore[reportPrivateUsage] # pytest and the cores' untyped members
|
||||
assert healthy
|
||||
|
||||
result = router._select_deployment_sync(
|
||||
strategy="cost-based-routing",
|
||||
selector=cost_selector,
|
||||
model="filtered-model",
|
||||
healthy_deployments=[],
|
||||
healthy_deployments=healthy,
|
||||
messages=None,
|
||||
input=None,
|
||||
request_kwargs=None,
|
||||
)
|
||||
assert result is None
|
||||
assert result in healthy
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1036,7 +1033,7 @@ def test_sync_usage_based_v2_override_stays_scoped_to_the_request_that_asked_for
|
|||
plain = router.completion(model="other-model", messages=messages, mock_response="ok")
|
||||
assert plain.choices[0].message.content == "ok"
|
||||
|
||||
with pytest.raises(ValueError, match="No deployments available"):
|
||||
with pytest.raises(litellm.RateLimitError, match="No deployments available"): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
router.completion(
|
||||
model="other-model", messages=messages, mock_response="ok", routing_strategy="usage-based-routing-v2"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3448,7 +3448,7 @@ WILDCARD_COOLED: Final = (FIRST_WILDCARD, SECOND_WILDCARD)
|
|||
("simple-shuffle", EVERY_DEPLOYMENT, "async", "refused", None, ALL_COOLED),
|
||||
("cost-based-routing", STRATEGY_COOLED, "sync", "refused", "team-a", None),
|
||||
("cost-based-routing", STRATEGY_COOLED, "async", "refused", "team-a", None),
|
||||
("cost-based-routing", STRATEGY_COOLED, "pass-through", "refused", "team-a", None),
|
||||
("cost-based-routing", STRATEGY_COOLED, "pass-through-chat", "refused", "team-a", None),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "async", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED),
|
||||
("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", (ALIAS_TARGET,)),
|
||||
|
|
|
|||
|
|
@ -263,24 +263,32 @@ def test_request_timeout_is_unset_without_a_router_timeout(
|
|||
assert _one(make_core).request_timeout is None # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
|
||||
def test_request_timeout_reads_litellm_request_timeout_directly_when_a_router_timeout_is_set(
|
||||
def test_request_timeout_reads_the_host_request_timeout_when_a_router_timeout_is_set(
|
||||
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Oddity: this one setting is read from ``litellm.request_timeout`` itself, not from the host's
|
||||
process settings (which say 33 here)."""
|
||||
"""Read from the host's process settings (33 here), not from ``litellm.request_timeout`` itself."""
|
||||
host.set_dynamic(request_timeout=33.0)
|
||||
monkeypatch.setattr(litellm, "request_timeout", 77.0)
|
||||
assert _one(make_core, timeout=10).request_timeout == 77.0 # pyright: ignore[reportAttributeAccessIssue]
|
||||
assert _one(make_core, timeout=10).request_timeout == 33.0 # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
|
||||
|
||||
def test_request_timeout_is_unset_when_litellm_request_timeout_is_its_untouched_default(
|
||||
make_core: MakeCore, monkeypatch: pytest.MonkeyPatch
|
||||
def test_request_timeout_is_unset_when_the_host_request_timeout_is_its_untouched_default(
|
||||
make_core: MakeCore,
|
||||
host: ScriptedHost,
|
||||
monkeypatch: pytest.MonkeyPatch, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "request_timeout", 6000)
|
||||
monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False, raising=False)
|
||||
host.set_dynamic(request_timeout=6000.0, request_timeout_explicitly_set=False)
|
||||
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
assert _one(make_core, timeout=10).request_timeout is None # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
|
||||
def test_request_timeout_keeps_the_default_value_when_the_host_marks_it_explicitly_set(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
host.set_dynamic(request_timeout=6000.0, request_timeout_explicitly_set=True)
|
||||
assert _one(make_core, timeout=10).request_timeout == 6000.0 # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
|
||||
|
||||
def test_stream_timeout_is_kept_as_given(make_core: MakeCore) -> None:
|
||||
assert _one(make_core, stream_timeout=3).stream_timeout == 3 # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
|
|
|
|||
|
|
@ -288,9 +288,9 @@ async def test_generic_call_sends_the_declared_custom_llm_provider(make_core: Ma
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
("endpoint", "expected"),
|
||||
[("aocr", 600.0), ("aresponses", 600.0), ("ocr", 6000.0), ("generate_content", 6000.0)],
|
||||
[("aocr", 600.0), ("aresponses", 600.0), ("ocr", 600.0), ("generate_content", 600.0), ("responses", 600.0)],
|
||||
)
|
||||
async def test_generic_default_timeout_is_the_passthrough_default_for_async_and_the_process_setting_for_sync(
|
||||
async def test_generic_default_timeout_is_the_passthrough_default_for_sync_and_async(
|
||||
make_core: MakeCore, host: EndpointHost, endpoint: str, expected: float
|
||||
):
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")])
|
||||
|
|
@ -306,6 +306,20 @@ async def test_generic_timeout_takes_the_router_timeout(make_core: MakeCore, hos
|
|||
assert host.calls[0].kwargs["timeout"] == 42
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["aocr", "ocr"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
|
||||
async def test_generic_timeout_without_a_router_timeout_takes_the_host_request_timeout(
|
||||
make_core: MakeCore,
|
||||
host: EndpointHost,
|
||||
endpoint: str,
|
||||
monkeypatch: pytest.MonkeyPatch, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members
|
||||
):
|
||||
host.set_dynamic(request_timeout=33.0)
|
||||
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")])
|
||||
await _call(core, endpoint, {"model": "g", "document": {}})
|
||||
assert host.calls[0].kwargs["timeout"] == 33.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("endpoint", "request_body", "expected"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -171,25 +171,17 @@ async def test_cache_responses_turned_on_after_construction_reaches_the_host(
|
|||
# --- sync text completion and image generation ----------------------------------------------
|
||||
|
||||
|
||||
async def test_sync_text_completion_sends_the_router_num_retries_and_no_routing_fields(
|
||||
make_core: MakeCore, host: EndpointHost
|
||||
):
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
|
||||
async def test_sync_text_completion_shapes_the_request_like_atext_completion(make_core: MakeCore, host: EndpointHost):
|
||||
core: Final = make_core(model_list=[deployment("g", "d1", timeout=7)], num_retries=4)
|
||||
core.text_completion(model="g", prompt="p")
|
||||
sent: Final = _sent(host)
|
||||
assert sent["num_retries"] == 4
|
||||
assert "model_info" not in sent
|
||||
assert "litellm_trace_id" not in sent
|
||||
assert sent.get("timeout") is None
|
||||
assert "num_retries" not in sent
|
||||
assert sent["model_info"]["id"] == "d1" # pyright: ignore[reportIndexIssue]
|
||||
trace_id: Final = sent["litellm_trace_id"]
|
||||
assert isinstance(trace_id, str) and len(trace_id) == 36
|
||||
assert sent["timeout"] == 7
|
||||
metadata: Final = _bucket(sent)
|
||||
assert metadata["model_group"] == "g"
|
||||
assert "deployment" not in metadata
|
||||
|
||||
|
||||
async def test_sync_text_completion_request_num_retries_wins(make_core: MakeCore, host: EndpointHost):
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
|
||||
core.text_completion(model="g", prompt="p", num_retries=1)
|
||||
assert _sent(host)["num_retries"] == 1
|
||||
assert (metadata["model_group"], metadata["deployment"]) == ("g", "openai/d1")
|
||||
|
||||
|
||||
async def test_sync_text_completion_merges_default_metadata_into_request_metadata(
|
||||
|
|
@ -204,23 +196,36 @@ async def test_sync_text_completion_merges_default_metadata_into_request_metadat
|
|||
assert (metadata["x"], metadata["y"], metadata["model_group"], sent["user"]) == (1, 2, "g", "u")
|
||||
|
||||
|
||||
async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model(
|
||||
make_core: MakeCore, host: EndpointHost
|
||||
):
|
||||
"""The request's model="g" is spread after the deployment params, so it replaces "openai/d1"."""
|
||||
async def test_sync_text_completion_sends_the_deployment_model(make_core: MakeCore, host: EndpointHost):
|
||||
core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")])
|
||||
core.text_completion(model="g", prompt="p")
|
||||
call: Final = host.calls[0]
|
||||
assert (call.model, call.kwargs["api_base"]) == ("g", "https://d1.example")
|
||||
assert (call.model, call.kwargs["api_base"]) == ("openai/d1", "https://d1.example")
|
||||
|
||||
|
||||
async def test_sync_text_completion_is_not_retried(make_core: MakeCore, host: EndpointHost):
|
||||
host.answer("g", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
|
||||
async def test_sync_text_completion_is_retried_with_the_router_num_retries(make_core: MakeCore, host: EndpointHost):
|
||||
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3)
|
||||
with pytest.raises(litellm.InternalServerError):
|
||||
core.text_completion(model="g", prompt="p")
|
||||
assert len(host.calls) == 1
|
||||
assert host.sleeps == []
|
||||
assert len(host.calls) == 4
|
||||
|
||||
|
||||
async def test_sync_text_completion_request_num_retries_wins(make_core: MakeCore, host: EndpointHost):
|
||||
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4)
|
||||
with pytest.raises(litellm.InternalServerError): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
core.text_completion(model="g", prompt="p", num_retries=1) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
assert len(host.calls) == 2
|
||||
|
||||
|
||||
async def test_sync_text_completion_falls_back_like_atext_completion(make_core: MakeCore, host: EndpointHost):
|
||||
host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1"))
|
||||
core: Final = make_core(
|
||||
model_list=[deployment("g", "d1"), deployment("h", "d2")], num_retries=0, fallbacks=[{"g": ["h"]}]
|
||||
)
|
||||
core.text_completion(model="g", prompt="p") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
assert host.deployments_called() == ["d1", "d2"]
|
||||
|
||||
|
||||
async def test_sync_image_generation_sends_routing_metadata_without_a_trace_id(make_core: MakeCore, host: EndpointHost):
|
||||
|
|
|
|||
|
|
@ -364,11 +364,13 @@ async def test_an_override_matching_a_routing_group_strategy_is_not_bound_to_the
|
|||
|
||||
def test_a_sync_usage_based_v2_override_counts_rpm_before_the_call(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
"""The override selector is on no callback list, so the sync pick runs its pre-call check itself:
|
||||
the first call is counted against ``rpm=1`` and the next pick finds no deployment. Oddity: the sync
|
||||
path raises a bare ``ValueError`` where the async one raises ``RateLimitError``."""
|
||||
the first call is counted against ``rpm=1`` and the next pick finds no deployment, raising the same
|
||||
``RateLimitError`` as the async path."""
|
||||
core: Final = make_core(model_list=[deployment("g", "a", rpm=1)], num_retries=0)
|
||||
core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue]
|
||||
with pytest.raises(ValueError, match=re.escape("No deployments available for selected model. Passed model=g.")):
|
||||
with pytest.raises( # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
litellm.RateLimitError, match=re.escape("No deployments available for selected model. Passed model=g.")
|
||||
):
|
||||
core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue]
|
||||
assert host.deployments_called() == ["a"]
|
||||
|
||||
|
|
@ -424,22 +426,31 @@ def test_sync_completion_under_latency_based_routing_reaches_the_group(make_core
|
|||
assert len(host.calls) == 10
|
||||
|
||||
|
||||
def test_sync_completion_under_cost_based_routing_finds_no_deployment(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
"""Oddity: the cost-based selector has no sync pick, so every sync pick under cost-based routing
|
||||
rate-limits without calling a provider."""
|
||||
def test_sync_completion_under_cost_based_routing_picks_the_cheapest_deployment(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""The cost-based selector picks on the sync path as it does on the async one."""
|
||||
core: Final = make_core(
|
||||
model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="cost-based-routing", num_retries=0
|
||||
model_list=[
|
||||
deployment("g", "a", input_cost_per_token=0.5, output_cost_per_token=0.5),
|
||||
deployment("g", "b", input_cost_per_token=0.1, output_cost_per_token=0.1),
|
||||
],
|
||||
routing_strategy="cost-based-routing",
|
||||
num_retries=0,
|
||||
)
|
||||
with pytest.raises(RouterRateLimitError) as raised:
|
||||
for _ in range(3):
|
||||
core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
|
||||
assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g")
|
||||
assert host.calls == []
|
||||
assert host.deployments_called() == ["b", "b", "b"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pass_through_pick_under_cost_based_routing_finds_no_deployment(make_core: MakeCore) -> None:
|
||||
async def test_pass_through_pick_under_cost_based_routing_picks_the_cheapest_deployment(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(
|
||||
model_list=[deployment("g", "a", use_in_pass_through=True)], routing_strategy="cost-based-routing"
|
||||
model_list=[
|
||||
deployment("g", "a", use_in_pass_through=True, input_cost_per_token=0.5, output_cost_per_token=0.5),
|
||||
deployment("g", "b", use_in_pass_through=True, input_cost_per_token=0.1, output_cost_per_token=0.1),
|
||||
],
|
||||
routing_strategy="cost-based-routing",
|
||||
)
|
||||
with pytest.raises(RouterRateLimitError):
|
||||
await core.select("g", kind="pass_through")
|
||||
picked: Final = await core.select("g", kind="pass_through") # pyright: ignore[reportCallIssue, reportUnknownVariableType] # pytest and the cores' untyped members
|
||||
assert picked["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue, reportCallIssue, reportArgumentType]
|
||||
|
|
|
|||
|
|
@ -328,9 +328,10 @@ async def test_a_per_user_github_copilot_deployment_rejects_another_credential_n
|
|||
async def test_the_process_request_timeout_outranks_the_router_timeout_once_a_router_timeout_is_set(
|
||||
make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Oddity: this per-attempt timeout is read from ``litellm.request_timeout`` itself (not the host's
|
||||
process settings), and only when the router has a ``timeout`` of its own."""
|
||||
monkeypatch.setattr(litellm, "request_timeout", 33.0)
|
||||
"""This per-attempt timeout is read from the host's process settings (not ``litellm.request_timeout``
|
||||
itself), and only when the router has a ``timeout`` of its own."""
|
||||
host.set_dynamic(request_timeout=33.0)
|
||||
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42)
|
||||
await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
|
||||
assert host.calls[0].kwargs["timeout"] == 33.0
|
||||
|
|
@ -341,7 +342,8 @@ async def test_the_process_request_timeout_is_the_stream_timeout_without_a_strea
|
|||
) -> None:
|
||||
from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts
|
||||
|
||||
monkeypatch.setattr(litellm, "request_timeout", 33.0)
|
||||
host.set_dynamic(request_timeout=33.0)
|
||||
monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
host.answer("openai/d1", chat_stream("A"))
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42)
|
||||
await collect_texts(await core.acompletion(model="g", messages=MESSAGES, stream=True)) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
|
@ -424,8 +426,18 @@ async def test_acompletion_counts_total_success_and_failed_calls_per_deployment_
|
|||
assert counts == (2, 1, 1)
|
||||
|
||||
|
||||
def test_sync_completion_does_not_count_calls(make_core: MakeCore, host: ScriptedHost) -> None:
|
||||
"""Oddity: only the async path updates ``total_calls``/``success_calls``/``fail_calls``."""
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")])
|
||||
def test_completion_counts_total_success_and_failed_calls_per_deployment_model(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""The sync path counts calls as ``acompletion`` does."""
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0)
|
||||
core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue]
|
||||
assert core.total_calls["openai/d1"] == 0 # pyright: ignore[reportAttributeAccessIssue]
|
||||
host.answer("openai/d1", rate_limit())
|
||||
with pytest.raises(litellm.RateLimitError): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
counts: Final = ( # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members
|
||||
core.total_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
core.success_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
core.fail_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
)
|
||||
assert counts == (2, 1, 1)
|
||||
|
|
|
|||
|
|
@ -61,33 +61,34 @@ def stream_with_usage(sync: bool = False) -> Callable[..., object]:
|
|||
# --- a fallback that answers without a stream --------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_mid_stream_fallback_that_answers_without_a_stream_yields_one_none_item(
|
||||
async def test_a_mid_stream_fallback_that_answers_without_a_stream_yields_its_response_as_one_chunk(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""Oddity: the fallback's whole response is dropped and the stream yields ``None`` in its place."""
|
||||
host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True))
|
||||
host.answer("openai/b", completion_response("not streamed", model="b"))
|
||||
core: Final = build(make_core)
|
||||
stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue]
|
||||
seen: list[object] = []
|
||||
await drain_into(stream, seen)
|
||||
assert seen[-1] is None
|
||||
last: Final = seen[-1]
|
||||
assert isinstance(last, ModelResponseStream)
|
||||
assert (last.model, last.choices[0].delta.content) == ("b", "not streamed") # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
assert host.deployments_called() == ["a", "b"]
|
||||
|
||||
|
||||
def test_a_sync_mid_stream_fallback_that_answers_without_a_stream_yields_the_response_fields(
|
||||
def test_a_sync_mid_stream_fallback_that_answers_without_a_stream_yields_its_response_as_one_chunk(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""Oddity: a ``ModelResponse`` is iterable (pydantic yields ``(field, value)`` pairs), so the sync
|
||||
wrapper streams the fallback response's fields instead of chunks."""
|
||||
"""The sync wrapper yields the same one chunk as the async one, not the response's pydantic fields."""
|
||||
host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True, sync=True))
|
||||
host.answer("openai/b", completion_response("not streamed", model="b"))
|
||||
core: Final = build(make_core)
|
||||
stream: Final = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue]
|
||||
items: Final = list(stream) # pyright: ignore[reportArgumentType]
|
||||
fields: Final = [item for item in items if isinstance(item, tuple)]
|
||||
assert ("model", "b") in fields
|
||||
assert {name for name, _ in fields} >= {"id", "choices", "usage"}
|
||||
assert not any(isinstance(item, tuple) for item in items) # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members
|
||||
last: Final = items[-1] # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members
|
||||
assert isinstance(last, ModelResponseStream)
|
||||
assert (last.model, last.choices[0].delta.content) == ("b", "not streamed") # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members
|
||||
assert host.deployments_called() == ["a", "b"]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ Python does them today and called out in the test's docstring.
|
|||
"""
|
||||
|
||||
import datetime
|
||||
import inspect
|
||||
import io
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import SimpleNamespace
|
||||
|
|
@ -73,6 +74,10 @@ ROUTED_ENDPOINTS: Final[dict[str, dict[str, object]]] = {
|
|||
"acancel_batch": {"batch_id": "b1"},
|
||||
}
|
||||
|
||||
SYNC_COUNTED: Final[dict[str, dict[str, object]]] = {
|
||||
"completion": {"messages": MESSAGES},
|
||||
"embedding": {"input": ["a"]},
|
||||
}
|
||||
|
||||
# --- batch completions (5565-5790) ----------------------------------------------------------
|
||||
|
||||
|
|
@ -320,17 +325,21 @@ async def test_image_generation_for_an_unknown_group_raises_without_calling_the_
|
|||
("aembedding", "openai/d1", "openai/d1"),
|
||||
("acreate_batch", "openai/d1", "g"),
|
||||
("acancel_batch", "openai/d1", "g"),
|
||||
("image_generation", "", ""),
|
||||
("image_generation", "openai/d1", "openai/d1"),
|
||||
("text_completion", "openai/d1", "g"),
|
||||
("completion", "openai/d1", "openai/d1"),
|
||||
("embedding", "openai/d1", "openai/d1"),
|
||||
],
|
||||
)
|
||||
async def test_call_counters_are_keyed_differently_per_endpoint(
|
||||
async def test_call_counters_are_keyed_per_endpoint_and_the_same_for_sync_and_async(
|
||||
make_core: MakeCore, host: EndpointHost, endpoint: str, total_key: str, fail_key: str
|
||||
):
|
||||
"""total_calls/fail_calls are keyed by the deployment model for some endpoints and by the model group
|
||||
for others (fail_calls even differs from total_calls for some); sync image_generation counts under ""."""
|
||||
for others (fail_calls even differs from total_calls for some); a sync endpoint counts as its async
|
||||
twin does."""
|
||||
host.answer("*", rate_limit())
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=1)
|
||||
request: Final = ROUTED_ENDPOINTS.get(endpoint, {"prompt": "p"})
|
||||
request: Final = SYNC_COUNTED.get(endpoint) or ROUTED_ENDPOINTS.get(endpoint, {"prompt": "p"})
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await core.call(endpoint, {"model": "g", **request})
|
||||
assert dict(core.total_calls) == {total_key: 2} # pyright: ignore[reportAttributeAccessIssue] # a public counter
|
||||
|
|
@ -399,16 +408,22 @@ async def test_async_passthrough_rewrites_only_whole_path_segments_naming_the_gr
|
|||
assert host.calls[0].kwargs["endpoint"] == "/model/us.anthropic.claude/invoke/gg/us.anthropic.claude:stream"
|
||||
|
||||
|
||||
async def test_sync_generic_call_replaces_every_substring_of_the_group_name_in_the_endpoint(
|
||||
make_core: MakeCore, monkeypatch: pytest.MonkeyPatch, host: EndpointHost
|
||||
@pytest.mark.parametrize("endpoint", ["aresponses", "responses"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
|
||||
async def test_generic_call_rewrites_only_whole_path_segments_naming_the_group(
|
||||
make_core: MakeCore,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
host: EndpointHost,
|
||||
endpoint: str, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members
|
||||
):
|
||||
"""The sync path uses str.replace with the full deployment model (provider prefix kept), so a group
|
||||
name inside other path segments is rewritten too, unlike the async path's segment-bounded rewrite."""
|
||||
"""Sync and async both swap the group name for the provider-stripped deployment model one whole path
|
||||
segment at a time, so a segment that merely contains the group name ("gg") is left alone."""
|
||||
patch_direct_endpoints(monkeypatch, host)
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")])
|
||||
core.responses(model="g", input="hi", endpoint="/gg/g")
|
||||
sent: Final = getattr(core, endpoint)(model="g", input="hi", endpoint="/gg/g") # pyright: ignore[reportAny] # pytest and the cores' untyped members
|
||||
if inspect.isawaitable(sent): # pyright: ignore[reportAny] # pytest and the cores' untyped members
|
||||
await sent
|
||||
call: Final = host.calls[0]
|
||||
assert (call.model, call.kwargs["endpoint"]) == ("openai/d1", "/openai/d1openai/d1/openai/d1")
|
||||
assert (call.model, call.kwargs["endpoint"]) == ("openai/d1", "/gg/d1")
|
||||
|
||||
|
||||
# --- batches (8112-8362) --------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -100,10 +100,10 @@ async def test_a_passing_pre_call_check_sees_the_deployment_that_is_then_called(
|
|||
assert gate.seen == [("async", host.deployments_called("acompletion")[0])]
|
||||
|
||||
|
||||
def test_a_rate_limit_from_a_sync_pre_call_check_is_raised_without_a_cooldown(
|
||||
def test_a_rate_limit_from_a_sync_pre_call_check_cools_the_deployment_down_and_is_raised(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
"""Oddity: the sync twin raises the gate's error but, unlike the async path, sets no cooldown."""
|
||||
"""The sync twin cools the refused deployment down as the async path does."""
|
||||
core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33)
|
||||
gate: Final = gated(host, rate_limit(message="gate says no"))
|
||||
|
||||
|
|
@ -111,7 +111,23 @@ def test_a_rate_limit_from_a_sync_pre_call_check_is_raised_without_a_cooldown(
|
|||
|
||||
assert isinstance(error, litellm.RateLimitError)
|
||||
assert host.calls == []
|
||||
assert [kind for kind, _ in gate.seen] == ["sync"]
|
||||
((kind, checked),) = gate.seen
|
||||
assert kind == "sync"
|
||||
cooled: Final = cooled_down(core, ["a", "b"])
|
||||
assert list(cooled) == [checked]
|
||||
assert cooled[checked]["cooldown_time"] == 33
|
||||
|
||||
|
||||
def test_any_other_error_from_a_sync_pre_call_check_is_raised_without_a_cooldown(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33)
|
||||
gated(host, ValueError("gate broke"))
|
||||
|
||||
error: Final = attempt_sync(core)
|
||||
|
||||
assert type(error) is ValueError
|
||||
assert host.calls == []
|
||||
assert cooled_down(core, ["a", "b"]) == {}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,105 @@
|
|||
"""
|
||||
Settings and configs both cores must treat as ``PythonCore`` does (oddities included): count and
|
||||
timeout settings kept as given, ``default_fallbacks=[]``, malformed model lists that fail at
|
||||
construction, and deprecated registry members called with a ``None`` name.
|
||||
"""
|
||||
|
||||
import warnings
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.router import Router
|
||||
from litellm.router_core import RouterCore
|
||||
from tests.unit.test_router.core.retries_health_helpers import MakeCore
|
||||
from tests.unit.test_router.core.scripted_host import deployment
|
||||
|
||||
|
||||
def as_router(core: RouterCore) -> Router:
|
||||
router: Final = Router.__new__(Router)
|
||||
router._attach(core) # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue, reportUnknownMemberType] # the facade's own attach, as from_config calls it
|
||||
return router
|
||||
|
||||
|
||||
def read(router: Router, name: str) -> object:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
return getattr(router, name) # pyright: ignore[reportAny] # a deprecated attribute, as given
|
||||
|
||||
|
||||
# --- settings kept as given -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_retry_after_and_an_int_timeout_stay_ints(make_core: MakeCore) -> None:
|
||||
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], timeout=1))
|
||||
assert (repr(read(router, "retry_after")), repr(read(router, "timeout"))) == ("0", "1")
|
||||
settings: Final = router.get_settings()
|
||||
assert (repr(settings["retry_after"]), repr(settings["timeout"])) == ("0", "1")
|
||||
|
||||
|
||||
def test_update_settings_casts_timeout_and_retry_after_to_int(make_core: MakeCore) -> None:
|
||||
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
|
||||
router.update_settings(timeout=3.7, retry_after=2.2)
|
||||
assert (repr(read(router, "timeout")), repr(read(router, "retry_after"))) == ("3", "2")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["num_retries", "allowed_fails", "max_fallbacks"])
|
||||
def test_a_count_setting_that_is_no_int_is_kept_as_given(make_core: MakeCore, name: str) -> None:
|
||||
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], **{name: "1e-05"}))
|
||||
assert read(router, name) == "1e-05"
|
||||
|
||||
|
||||
# --- default_fallbacks ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_an_empty_default_fallbacks_adds_a_star_entry_with_no_models(make_core: MakeCore) -> None:
|
||||
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], default_fallbacks=[]))
|
||||
assert read(router, "fallbacks") == [{"*": None}]
|
||||
|
||||
|
||||
# --- malformed model lists --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", [None, 5])
|
||||
def test_a_model_name_that_is_no_string_fails_at_construction(make_core: MakeCore, model_name: object) -> None:
|
||||
entry: Final = {"model_name": model_name, "litellm_params": {"model": "openai/gpt-4o", "api_key": "k"}}
|
||||
with pytest.raises(TypeError, match="expected str instance"):
|
||||
make_core(model_list=[entry])
|
||||
with pytest.raises(ValidationError, match="model_name"):
|
||||
make_core(model_list=[{**entry, "model_info": {"id": "x"}}])
|
||||
|
||||
|
||||
def test_an_alias_entry_without_hidden_fails_at_construction(make_core: MakeCore) -> None:
|
||||
with pytest.raises(KeyError, match="hidden"):
|
||||
make_core(model_list=[deployment("gpt", "a")], model_group_alias={"al": {"model": "gpt"}})
|
||||
|
||||
|
||||
# --- deprecated registry members with a None name -----------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("member", "arguments", "expected"),
|
||||
[
|
||||
("get_model_group", (None,), None),
|
||||
("resolve_served_model_name", (None, {}), None),
|
||||
("get_model_from_alias", (None,), None),
|
||||
("get_routable_upstream_model", (None,), None),
|
||||
("get_candidate_model_ids_for_route", (None,), frozenset()),
|
||||
],
|
||||
)
|
||||
def test_a_deprecated_member_called_with_no_name_returns_the_old_empty_result(
|
||||
make_core: MakeCore, member: str, arguments: tuple[object, ...], expected: object
|
||||
) -> None:
|
||||
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
assert getattr(router, member)(*arguments) == expected # pyright: ignore[reportAny] # the deprecated member
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remaining_usage_of_no_group_is_empty(make_core: MakeCore) -> None:
|
||||
router: Final = as_router(make_core(model_list=[deployment("gpt", "a")]))
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
assert await router.get_remaining_model_group_usage(None) == {} # pyright: ignore[reportArgumentType] # a None group, as the fuzzer passes it
|
||||
|
|
@ -535,13 +535,13 @@ async def test_pass_through_selection_without_pass_through_deployments_raises_ba
|
|||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
await core.select("g", kind="pass_through")
|
||||
assert str(raised.value) == (
|
||||
"litellm.BadRequestError: Model g has no deployment configured with use_in_pass_through=True. "
|
||||
"Please add use_in_pass_through: true in the deployment configuration"
|
||||
"litellm.BadRequestError: Model g has no deployments configured with use_in_pass_through=True. "
|
||||
"Please add use_in_pass_through: true to the deployment configuration"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_pass_through_selection_words_the_missing_deployment_error_differently(
|
||||
async def test_async_pass_through_selection_words_the_missing_deployment_error_the_same_way(
|
||||
make_core: MakeCore,
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=[deployment("g", "a")])
|
||||
|
|
|
|||
|
|
@ -195,15 +195,23 @@ async def test_selecting_a_marker_by_its_id_refuses_it_naming_its_litellm_model(
|
|||
assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="auto_router/complexity_router")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry", ["get_available_deployment", "completion"])
|
||||
def test_the_sync_path_never_runs_pre_routing_and_refuses_the_marker(
|
||||
make_core: MakeCore, host: ScriptedHost, entry: str
|
||||
@pytest.mark.parametrize("model", ["smart", "al"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
|
||||
def test_sync_get_available_deployment_runs_pre_routing_like_the_async_pick(make_core: MakeCore, model: str) -> None:
|
||||
core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"})
|
||||
request: Final[dict[str, object]] = {}
|
||||
chosen: Final = core.get_available_deployment(model, messages=MESSAGES, request_kwargs=request) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
|
||||
assert chosen["model_info"]["id"] == "s"
|
||||
assert _metadata(request)["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("model", "messages", "expected"), [("smart", MESSAGES, "s"), ("al", HARD_MESSAGES, "b")]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members
|
||||
def test_sync_completion_routes_through_a_complexity_router_like_acompletion(
|
||||
make_core: MakeCore, host: ScriptedHost, model: str, messages: list[dict[str, str]], expected: str
|
||||
) -> None:
|
||||
core: Final = make_core(model_list=_tiered())
|
||||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
_sync_entry(core, entry, "smart")
|
||||
assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="smart")
|
||||
assert host.calls == []
|
||||
core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"})
|
||||
core.completion(model=model, messages=messages) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
|
||||
assert host.deployments_called() == [expected]
|
||||
assert host.calls[-1].model_group == model
|
||||
|
||||
|
||||
# --- choosing among tag-scoped markers -----------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue