diff --git a/litellm/litellm_core_utils/asyncify.py b/litellm/litellm_core_utils/asyncify.py index 17cfb3ef84f..d336334c262 100644 --- a/litellm/litellm_core_utils/asyncify.py +++ b/litellm/litellm_core_utils/asyncify.py @@ -81,7 +81,11 @@ def can_block_current_thread() -> bool: return threading.current_thread() is threading.main_thread() and not is_event_loop_running() -def run_async_function(async_function, *args, **kwargs): +def run_async_function( + async_function: Callable[..., Awaitable[T_Retval]], + *args: object, + **kwargs: object, # kwargs-ok: forwarded to async_function as given +) -> T_Retval: """ Helper utility to run an async function in a sync context. Handles the case where there is an existing event loop running. diff --git a/litellm/litellm_core_utils/request_timeout_resolver.py b/litellm/litellm_core_utils/request_timeout_resolver.py index 0e011dc06a0..8a446635a11 100644 --- a/litellm/litellm_core_utils/request_timeout_resolver.py +++ b/litellm/litellm_core_utils/request_timeout_resolver.py @@ -26,8 +26,14 @@ def get_configured_request_timeout() -> float | None: """Return the explicitly-configured ``litellm.request_timeout``, else ``None``.""" import litellm - timeout: Final = float(litellm.request_timeout) - if litellm.request_timeout_explicitly_set: + return configured_request_timeout(litellm.request_timeout, litellm.request_timeout_explicitly_set) + + +def configured_request_timeout(request_timeout: float, explicitly_set: bool) -> float | None: + """``get_configured_request_timeout`` for a process ``request_timeout`` read elsewhere (a router + host's process settings): the value when it was configured, else ``None``.""" + timeout: Final = float(request_timeout) + if explicitly_set: return timeout if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS): return timeout diff --git a/litellm/router_config.py b/litellm/router_config.py index 75c79068191..2227ab3bacb 100644 --- a/litellm/router_config.py +++ b/litellm/router_config.py @@ -58,6 +58,7 @@ class RouterDynamicConfig: input_callback: Sequence[object] callback_manager: "LoggingCallbackManager" cache: "Cache | None" + request_timeout_explicitly_set: bool = False _versions: Final = itertools.count(1) @@ -110,6 +111,7 @@ def current_dynamic_config() -> RouterDynamicConfig: input_callback=litellm.input_callback, callback_manager=litellm.logging_callback_manager, cache=litellm.cache, + request_timeout_explicitly_set=litellm.request_timeout_explicitly_set, ) @@ -120,6 +122,7 @@ _LIVE_SOURCES: Final = MappingProxyType( "router_max_fallbacks": "ROUTER_MAX_FALLBACKS", "allowed_fails": "allowed_fails", "request_timeout": "request_timeout", + "request_timeout_explicitly_set": "request_timeout_explicitly_set", "fallbacks": "fallbacks", "default_fallbacks": "default_fallbacks", "context_window_fallbacks": "context_window_fallbacks", diff --git a/litellm/router_deprecated.py b/litellm/router_deprecated.py index 9489fcf4bb6..ccc757e908d 100644 --- a/litellm/router_deprecated.py +++ b/litellm/router_deprecated.py @@ -98,6 +98,16 @@ def _as_router(obj: "DeprecatedRouterMembers") -> "Router": raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only") +def _registry_source(obj: "DeprecatedRouterMembers") -> "Router": + """What ``registry()`` reads: the Router on the Python core, the host registry on the Rust core. + + A deprecated member whose ``registry()`` query checks for a name reads it directly, as the + member did before ``registry()`` existed, so a ``None`` name gets the old result on both cores + instead of ``registry()``'s ``TypeError``. + """ + return _as_router(obj).registry()._router # pyright: ignore[reportPrivateUsage] # the shim is part of the router + + def _public_attribute_error(obj: object, core_name: str, public_name: str, error: AttributeError) -> AttributeError: """ ``error`` renamed to the public attribute when it is the router's own ``core_name`` that is @@ -507,8 +517,8 @@ class DeprecatedRouterMembers: self, model_name: str, model_alias: str | None = None, team_id: str | None = None ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged _note("get_all_deployments", "registry().deployments(model_name, source='all')") - return ( - _as_router(self).registry().deployments(model_name, source="all", model_alias=model_alias, team_id=team_id) + return _registry_source(self)._core_get_all_deployments( + model_name=model_name, model_alias=model_alias, team_id=team_id ) _get_all_deployments = _private_alias("get_all_deployments", get_all_deployments) @@ -519,7 +529,7 @@ class DeprecatedRouterMembers: def deployments_named(self, model_name: str) -> "tuple[DeploymentTypedDict, ...] | None": _note("deployments_named", "registry().deployments(model_name, source='named')") - return _as_router(self).registry().deployments(model_name, source="named") + return _registry_source(self)._core_deployments_named(model_name) def get_model_list_from_model_alias( self, model_name: str | None = None @@ -535,11 +545,11 @@ class DeprecatedRouterMembers: self, model_group: str ) -> "list[DeploymentTypedDict]": # mutable-ok: the old return type, unchanged _note("get_model_list_of_routed_group", "registry().deployments(model_group, source='routed_group')") - return _as_router(self).registry().deployments(model_group, source="routed_group") + return _registry_source(self)._core_get_model_list_of_routed_group(model_group) def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: _note("wildcard_route_deployments", "registry().deployments(model, source='wildcard')") - return _as_router(self).registry().deployments(model, source="wildcard", team_id=team_id) + return _registry_source(self)._core_wildcard_route_deployments(model, team_id) def get_routable_deployments( self, @@ -550,21 +560,17 @@ class DeprecatedRouterMembers: _note( "get_routable_deployments", "registry().deployments(model, source='routable', request=..., call_type=...)" ) - return ( - _as_router(self) - .registry() - .deployments(model, source="routable", request=request_kwargs, call_type=call_type) - ) + return _registry_source(self)._core_get_routable_deployments(model, request_kwargs, call_type) def get_deployment_by_model_group_name(self, model_group_name: str) -> "Deployment | None": _note("get_deployment_by_model_group_name", "registry().deployments(name, source='first_in_group')") - return _as_router(self).registry().deployments(model_group_name, source="first_in_group") + return _registry_source(self)._core_get_deployment_by_model_group_name(model_group_name=model_group_name) def deployments_for_request( self, model: str, request_kwargs: Mapping[str, object] ) -> "Sequence[DeploymentTypedDict]": _note("deployments_for_request", "registry().deployments(model, source='for_request', request=...)") - return _as_router(self).registry().deployments(model, source="for_request", request=request_kwargs) + return _registry_source(self)._core_deployments_for_request(model, request_kwargs) def should_include_deployment( self, @@ -599,34 +605,32 @@ class DeprecatedRouterMembers: self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False ) -> tuple[str, ...]: _note("resolved_litellm_models", "registry().names(model_name, kind='litellm_models' or 'litellm_base_models')") - return ( - _as_router(self) - .registry() - .names(model_name, kind="litellm_base_models" if use_base_model else "litellm_models", team_id=team_id) + return _registry_source(self)._core_resolved_litellm_models( + model_name, team_id=team_id, use_base_model=use_base_model ) def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: _note("wildcard_routed_models", "registry().names(model, kind='wildcard')") - return _as_router(self).registry().names(model, kind="wildcard", team_id=team_id) + return _registry_source(self)._core_wildcard_routed_models(model, team_id) def get_candidate_model_ids_for_route(self, model: str, team_id: str | None = None) -> frozenset[str]: _note("get_candidate_model_ids_for_route", "registry().names(model, kind='route_candidates')") - return _as_router(self).registry().names(model, kind="route_candidates", team_id=team_id) + return _registry_source(self)._core_get_candidate_model_ids_for_route(model, team_id=team_id) # -- registry: resolving names --------------------------------------------------------------- def routable_model_group(self, model_name: str) -> str: _note("routable_model_group", "registry().resolve(name)") - return _as_router(self).registry().resolve(model_name) + return _registry_source(self)._core_routable_model_group(model_name) def get_model_from_alias(self, model: str) -> str | None: _note("get_model_from_alias", "registry().resolve(name, kind='alias')") - return _as_router(self).registry().resolve(model, kind="alias") + return _registry_source(self)._core_get_model_from_alias(model=model) _get_model_from_alias = _private_alias("get_model_from_alias", get_model_from_alias) def resolve_served_model_name(self, model_name: str, served_model: str) -> str: _note("resolve_served_model_name", "registry().resolve(name, kind='served', served_model=...)") - return _as_router(self).registry().resolve(model_name, kind="served", served_model=served_model) + return _registry_source(self)._core_resolve_served_model_name(model_name, served_model) def resolve_model_name_from_model_id( self, model_id: str | None, custom_llm_provider: str | None = None @@ -646,23 +650,25 @@ class DeprecatedRouterMembers: def is_recognized_model(self, model: str) -> bool: _note("is_recognized_model", "registry().resolve(name, kind='recognized')") - return _as_router(self).registry().resolve(model, kind="recognized") + return _registry_source(self)._core_is_recognized_model(model) def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None: _note("get_routable_upstream_model", "registry().resolve(name, kind='upstream')") - return _as_router(self).registry().resolve(model_name, kind="upstream", team_id=team_id) + return _registry_source(self)._core_get_routable_upstream_model(model_name, team_id=team_id) def get_search_tool_support(self, model_name: str, team_id: str | None = None) -> tuple[bool | None, ...]: _note("get_search_tool_support", "registry().resolve(name, kind='search_tool_support')") - return _as_router(self).registry().resolve(model_name, kind="search_tool_support", team_id=team_id) + return _registry_source(self)._core_get_search_tool_support(model_name, team_id=team_id) def get_model_group(self, id: str) -> list | None: # mutable-ok: the old return type, unchanged _note("get_model_group", "registry().resolve(id, kind='group_of_id')") - return _as_router(self).registry().resolve(id, kind="group_of_id") + return _registry_source(self)._core_get_model_group(id=id) def is_model_access_group_for_wildcard_route(self, model_access_group: str) -> bool: _note("is_model_access_group_for_wildcard_route", "registry().resolve(group, kind='wildcard_access_group')") - return _as_router(self).registry().resolve(model_access_group, kind="wildcard_access_group") + return _registry_source(self)._core_is_model_access_group_for_wildcard_route( + model_access_group=model_access_group + ) _is_model_access_group_for_wildcard_route = _private_alias( "is_model_access_group_for_wildcard_route", is_model_access_group_for_wildcard_route @@ -670,7 +676,7 @@ class DeprecatedRouterMembers: def team_model_has_alternatives(self, deployment_id: str) -> bool: _note("team_model_has_alternatives", "registry().resolve(id, kind='team_model_alternatives')") - return _as_router(self).registry().resolve(deployment_id, kind="team_model_alternatives") + return _registry_source(self)._core_team_model_has_alternatives(deployment_id) def routing_group_has_alternatives(self, model_group: str | None) -> bool: _note("routing_group_has_alternatives", "registry().resolve(group, kind='routing_group_alternatives')") diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 8df9e607e96..7b94fa5a244 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -180,7 +180,7 @@ from litellm.litellm_core_utils.reasoning_effort_capability import ( resolve_supported_reasoning_efforts, ) from litellm.litellm_core_utils.request_timeout_resolver import ( - get_configured_request_timeout, + configured_request_timeout, get_service_tier_timeout, ) from litellm.litellm_core_utils.routing_common_utils import ( @@ -366,6 +366,7 @@ from litellm.router_utils.fallback_event_handlers import ( mid_stream_fallback_hop_kwargs, mid_stream_fallback_snapshot_kwargs, mid_stream_retry_kwargs, + non_stream_fallback_answer_as_chunk, per_request_fallback_controls, record_disable_fallbacks, record_pre_routing_selection, @@ -699,6 +700,12 @@ _NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +# The router strategies whose pick runs the pre-routing hook (`_async_get_available_deployment`). +_PRE_ROUTING_PICK_STRATEGIES: Final = frozenset( + {"usage-based-routing-v2", "simple-shuffle", "cost-based-routing", "latency-based-routing", "least-busy"} +) +# The generic calls (sync and async) resolve their timeout with the LLM pass-through resolver. +_GENERIC_CALL_FUNCTION_NAMES: Final = frozenset({"_ageneric_api_call_with_fallbacks", "generic_api_call"}) _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) @@ -2083,7 +2090,7 @@ class PythonCore(DeprecatedRouterMembers): # Per-attempt request_timeout, independent of router_settings.timeout. # Only stored when a router timeout is also set, since otherwise # request_timeout already flows through self.timeout above. - self.request_timeout = get_configured_request_timeout() if timeout is not None else None + self.request_timeout = self._process_request_timeout() if timeout is not None else None self.stream_timeout = stream_timeout self.retry_after = retry_after @@ -2972,16 +2979,13 @@ class PythonCore(DeprecatedRouterMembers): if selector is None: return None - # `cost-based-routing` is intentionally omitted — - # `LowestCostLoggingHandler` only implements - # `async_get_available_deployments` match strategy: case "least-busy": return selector.get_available_deployments( model_group=model, healthy_deployments=healthy_deployments, ) - case "usage-based-routing" | "usage-based-routing-v2": + case "usage-based-routing" | "usage-based-routing-v2" | "cost-based-routing": return selector.get_available_deployments( model_group=model, healthy_deployments=healthy_deployments, @@ -3411,6 +3415,7 @@ class PythonCore(DeprecatedRouterMembers): model_client = None else: model_client = potential_model_client + self.total_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core ### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit) ## only run if model group given, not model id @@ -3427,6 +3432,7 @@ class PythonCore(DeprecatedRouterMembers): } ) response: Final = self._host.invoke("completion")(**input_kwargs) + self.success_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) ## CHECK CONTENT FILTER ERROR ## @@ -3459,6 +3465,8 @@ class PythonCore(DeprecatedRouterMembers): return response except Exception as e: verbose_router_logger.info("litellm.completion(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) + if model_name is not None: + self.fail_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core # Set per-deployment num_retries on exception for retry logic if deployment is not None: self._set_deployment_num_retries_on_exception(e, deployment) @@ -3733,6 +3741,17 @@ class PythonCore(DeprecatedRouterMembers): adopting_wrapper.adopt_fallback_response_headers(fallback_response, prepared) return prepared + @staticmethod + def _non_stream_fallback_item( + wrapper_ref: "weakref.ref[FallbackAwareStreamWrapper]", fallback_response: object + ) -> object: + """The one item a stream yields for a fallback that answered without a stream: its + response as a stream chunk, carrying the fallback's hidden params and headers.""" + prepared: Final = PythonCore._adopt_fallback_response_headers(wrapper_ref, fallback_response) + item: Final = non_stream_fallback_answer_as_chunk(fallback_response) + PythonCore._apply_fallback_hidden_params_to_item(item, prepared) + return item + @staticmethod def _apply_fallback_hidden_params_to_item( fallback_item: object, @@ -3949,8 +3968,7 @@ class PythonCore(DeprecatedRouterMembers): ) yield fallback_item else: - # If fallback returns a non-streaming response, yield None - yield None + yield PythonCore._non_stream_fallback_item(wrapper_ref, fallback_response) # pyright: ignore[reportAny] # the legacy untyped router core except Exception as fallback_error: # If fallback also fails, log and re-raise original error @@ -4710,7 +4728,7 @@ class PythonCore(DeprecatedRouterMembers): initial_kwargs["messages"] = messages router_self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs) fallback_response = run_async_function( - router_self.async_function_with_fallbacks_common_utils, + router_self.async_function_with_fallbacks_common_utils, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core e=e.original_exception if isinstance(e.original_exception, litellm_exceptions.ContextWindowExceededError) else e, @@ -4724,7 +4742,7 @@ class PythonCore(DeprecatedRouterMembers): include_fallback_errors=initial_kwargs.get("include_fallback_errors", False) is True, ) - if hasattr(fallback_response, "__iter__"): + if hasattr(fallback_response, "__iter__") and not isinstance(fallback_response, ModelResponse): # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core prepared_fallback_hidden_params = PythonCore._adopt_fallback_response_headers( wrapper_ref, fallback_response ) @@ -4747,7 +4765,7 @@ class PythonCore(DeprecatedRouterMembers): router_self._combine_fallback_usage(fallback_item, None) yield fallback_item else: - yield None + yield PythonCore._non_stream_fallback_item(wrapper_ref, fallback_response) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core except Exception as fallback_error: verbose_router_logger.error("Fallback also failed: %s", fallback_error) @@ -4766,9 +4784,9 @@ class PythonCore(DeprecatedRouterMembers): "stream_with_fallbacks: error closing model_response: %s", close_err, ) - if fallback_response is not None and hasattr(fallback_response, "close"): + if fallback_response is not None and hasattr(fallback_response, "close"): # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core try: - fallback_response.close() + fallback_response.close() # pyright: ignore[reportAttributeAccessIssue] # a stream wrapper, closed when it has close() except BaseException as close_err: verbose_router_logger.debug( "stream_with_fallbacks: error closing fallback_response: %s", @@ -5299,17 +5317,19 @@ class PythonCore(DeprecatedRouterMembers): kwargs["model_info"] = model_info - if function_name == "_ageneric_api_call_with_fallbacks": + if function_name in _GENERIC_CALL_FUNCTION_NAMES: from litellm.passthrough.timeout_utils import ( resolve_llm_passthrough_timeout, ) + # The process request_timeout is read through the host; it ranks right after the + # router timeout, as resolve_llm_passthrough_timeout ranks it. _router_timeout: Final = ( self.request_timeout if self.request_timeout is not None else float(self._explicit_timeout) if isinstance(self._explicit_timeout, (int, float)) - else None + else self._process_request_timeout() ) _router_stream_timeout: Final = ( self.stream_timeout @@ -5381,6 +5401,11 @@ class PythonCore(DeprecatedRouterMembers): hop_kwargs, _FALLBACK_HOP_ADAPTER.validate_python(deployment) ) + def _process_request_timeout(self) -> float | None: + """The host's explicitly configured process ``request_timeout``, else None.""" + dynamic: Final = self._host.dynamic() + return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set) + def _get_async_openai_model_client(self, deployment: dict, kwargs: dict): """ Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment @@ -5926,7 +5951,7 @@ class PythonCore(DeprecatedRouterMembers): raise e def _image_generation(self, prompt: str, model: str, **kwargs): - model_name: Final = "" + model_name = model # rebind-ok: the group until a deployment is picked, as in _aimage_generation try: verbose_router_logger.debug("Inside _image_generation()- model: %s", model) deployment: Final = self.get_available_deployment( @@ -5937,6 +5962,7 @@ class PythonCore(DeprecatedRouterMembers): ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs) data: Final = deployment["litellm_params"].copy() + model_name = data["model"] # pyright: ignore[reportUnknownVariableType] # rebind-ok: the picked deployment's model, as in _aimage_generation model_client: Final = self._get_async_openai_model_client( deployment=deployment, @@ -5958,10 +5984,10 @@ class PythonCore(DeprecatedRouterMembers): } ) self.success_calls[model_name] += 1 - verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) + verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core return response except Exception as e: - verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) + verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core if model_name is not None: self.fail_calls[model_name] += 1 self._stamp_retry_skip_deployment_id(e, kwargs) @@ -6300,37 +6326,64 @@ class PythonCore(DeprecatedRouterMembers): is_async: bool | None = False, **kwargs, ): - messages: Final = [{"role": "user", "content": prompt}] - try: - kwargs["model"] = model - kwargs["prompt"] = prompt - kwargs["num_retries"] = kwargs.get("num_retries", self.num_retries) - kwargs.setdefault("metadata", {}).update({"model_group": model}) + """The sync twin of `atext_completion`: picks, shapes, retries and falls back the same way.""" + kwargs["model"] = model # rebind-ok: the request kwargs, shaped in place as atext_completion shapes them + kwargs["prompt"] = prompt # rebind-ok: same + kwargs["original_function"] = self._text_completion # rebind-ok: same + self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core + return self.function_with_fallbacks(**kwargs) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] # the legacy untyped router core - # pick the one that is available (lowest TPM/RPM) - with endpoint_selection("text_completion", model=model): - deployment: Final = self.get_available_deployment( - model=model, - messages=messages, - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + def _text_completion( + self, + model: str, + prompt: str, + **kwargs: object, # kwargs-ok: the request kwargs, sent on to the deployment + ) -> object: + try: + verbose_router_logger.debug("Inside _text_completion()- model: %s", model) + deployment: Final = self.get_available_deployment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy untyped router core + model=model, + messages=[{"role": "user", "content": prompt}], + specific_deployment=kwargs.pop("specific_deployment", None), # pyright: ignore[reportArgumentType] # the legacy untyped router core + request_kwargs=kwargs, + ) + self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, splits_stream_timeout=True) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core data: Final = deployment["litellm_params"].copy() - for k, v in self._core_default_litellm_params.items(): - if k not in kwargs: # prioritize model-specific params > default router params - kwargs[k] = v - elif k == "metadata": - kwargs[k].update(v) - pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data)) - if pinned_model_id is not None: - kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults + model_name: Final = data["model"] # pyright: ignore[reportUnknownVariableType] # the legacy untyped router core - # call via litellm.completion() - return self._host.invoke("text_completion")( - **{**data, "prompt": prompt, "caching": self._core_cache_responses, **kwargs} + potential_model_client: Final = self._get_client(deployment=deployment, kwargs=kwargs, client_type="sync") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy untyped router core + dynamic_api_key: Final = kwargs.get("api_key", None) + model_client: Final = ( # pyright: ignore[reportUnknownVariableType] # the legacy untyped router core + None + if dynamic_api_key is not None + and potential_model_client is not None + and dynamic_api_key != potential_model_client.api_key # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core + else potential_model_client ) + self.total_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core + + ### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit) + self.routing_strategy_pre_call_checks(deployment=deployment) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core + + response: Final = self._host.invoke("text_completion")( # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core + **{ + **data, + "prompt": prompt, + "caching": self._core_cache_responses, + "client": model_client, + **kwargs, + } + ) + + self.success_calls[model_name] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core + verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core + return response except Exception as e: + verbose_router_logger.info("litellm.text_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model, e) + if model is not None: # pyright: ignore[reportUnnecessaryComparison] # the legacy untyped router core + self.fail_calls[model] += 1 # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core + self._stamp_retry_skip_deployment_id(e, kwargs) raise e async def atext_completion( @@ -7476,8 +7529,11 @@ class PythonCore(DeprecatedRouterMembers): # For passthrough routes, use the actual model from deployment # and swap model name in endpoint if present - if "endpoint" in kwargs and kwargs["endpoint"]: - kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name) + self._add_deployment_model_to_endpoint_for_llm_passthrough_route( + kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core + model=model, + model_name=model_name, # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core + ) kwargs["model"] = model_name # Perform pre-call checks for routing strategy @@ -9473,7 +9529,7 @@ class PythonCore(DeprecatedRouterMembers): Wrapped to reduce code duplication and prevent bugs. """ - return run_async_function(self.async_function_with_fallbacks, *args, **kwargs) + return run_async_function(self.async_function_with_fallbacks, *args, **kwargs) # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core def _get_fallback_model_group_for_lookup_groups( self, @@ -10232,11 +10288,22 @@ class PythonCore(DeprecatedRouterMembers): - None Raises: - - Rate Limit Exception - If the deployment is over it's tpm/rpm limits + - Rate Limit Exception - If the deployment is over it's tpm/rpm limits; the deployment is + cooled down first, as on the async path """ for _callback in self._routing_callbacks(): if isinstance(_callback, CustomLogger): - _callback.pre_call_check(deployment) + try: + _callback.pre_call_check(deployment) # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core + except litellm_exceptions.RateLimitError as e: + set_cooldown_deployments( + ctx=self._core_ctx, + exception_status=e.status_code, + original_exception=e, + deployment=deployment["model_info"]["id"], # pyright: ignore[reportUnknownArgumentType] # the legacy untyped router core + time_to_cooldown=self._core_cooldown_time, + ) + raise async def async_routing_strategy_pre_call_checks( self, @@ -16234,6 +16301,18 @@ class PythonCore(DeprecatedRouterMembers): ) return registered_model_name + def _core_sync_pick_pre_routes(self) -> bool: + """Whether the sync pick runs the pre-routing hook: a strategy router is registered and the + router strategy is one whose async pick runs the hook too (see `_async_get_available_deployment`).""" + return self._core_routing_strategy in _PRE_ROUTING_PICK_STRATEGIES and any( + ( + self._core_auto_routers, + self._core_complexity_routers, + self._core_adaptive_routers, + self._core_quality_routers, + ) + ) + async def _core_async_pre_routing_hook( self, model: str, @@ -16646,6 +16725,33 @@ class PythonCore(DeprecatedRouterMembers): "usage-based-routing-v2, cost-based-routing, latency-based-routing, least-busy), " "or remove `plugins` from the Router config." ) + if self._core_sync_pick_pre_routes(): + # The async pick's pre-routing hook (auto/complexity/adaptive/quality routers), run to + # completion from the sync pick, so a strategy router serves sync calls too. + hook_kwargs: Final[dict[str, object]] = ( # pyright: ignore[reportUnknownVariableType] # mutable-ok: the request kwargs the hook stamps + request_kwargs if request_kwargs is not None else {} + ) + responses_call: Final = input is not None and messages is None + pre_routed: Final = run_async_function( + partial( + self._core_async_pre_routing_hook, # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core + model=model, + request_kwargs=hook_kwargs, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) + ) + if pre_routed is not None: + model = pre_routed.model # rebind-ok: the tier group the strategy picked, as the async pick rebinds it + messages = pre_routed.messages # rebind-ok: same + record_pre_routing_selection(hook_kwargs, model) + self._pin_tier_params_onto_request( # pyright: ignore[reportUnknownMemberType] # the legacy untyped router core + model=model, + tier_litellm_params=pre_routed.litellm_params if pre_routed is not None else None, + request_kwargs=hook_kwargs, + responses_call=responses_call, + ) # users need to explicitly call a specific deployment, by setting `specific_deployment = True` as completion()/embedding() kwarg # When this was no explicit we had several issues with fallbacks timing out @@ -16926,7 +17032,7 @@ class PythonCore(DeprecatedRouterMembers): if len(pass_through_deployments) == 0: # No deployments support pass-through raise litellm_exceptions.BadRequestError( - message=f"Model {model} has no deployment configured with use_in_pass_through=True. Please add use_in_pass_through: true in the deployment configuration", + message=f"Model {model} has no deployments configured with use_in_pass_through=True. Please add use_in_pass_through: true to the deployment configuration", model=model, llm_provider="", ) diff --git a/litellm/router_rust_core.py b/litellm/router_rust_core.py index 468fc2f69cd..4d252cbbcba 100644 --- a/litellm/router_rust_core.py +++ b/litellm/router_rust_core.py @@ -424,6 +424,19 @@ def failure_kind(error: BaseException) -> str: return next((name for name in names if name in _FAILURE_KINDS), "APIError") +def _crate_int(value: object) -> int | None: + """A count setting kept as given, as the crate reads it: an int, or a number spelled another way + (``1.5``, ``"3"``, ``"1e-05"``) truncated to one; None (the crate's default) when it is no number.""" + if value is None or isinstance(value, bool): + return None if value is None else int(value) + if isinstance(value, int): + return value + try: + return int(float(str(value))) + except (ValueError, OverflowError): + return None + + def _retry_after_seconds(error: BaseException) -> float | None: response: Final = getattr(error, "response", None) headers: Final[object] = getattr(error, "litellm_response_headers", None) or getattr(response, "headers", None) @@ -490,15 +503,18 @@ class Resolved: routing_strategy: str | None routing_strategy_args: Mapping[str, object] - num_retries: int - max_fallbacks: int - allowed_fails: int - explicit_allowed_fails: int | None + # `num_retries`, `max_fallbacks` and `allowed_fails` as the Python core stores them: as given + # (the crate reads them through `_crate_int`). + num_retries: object + max_fallbacks: object + allowed_fails: object + explicit_allowed_fails: object cooldown_time: float disable_cooldowns: bool | None allowed_fails_policy: object - retry_after: float - timeout: float | None + # `retry_after` and `timeout` as the Python core stores them: as given (an int stays an int). + retry_after: object + timeout: object fallbacks: list[object] | None # mutable-ok: the setting as Python spells it context_window_fallbacks: list[object] | None # mutable-ok: same content_policy_fallbacks: list[object] | None # mutable-ok: same @@ -535,15 +551,19 @@ class Resolved: list(_as_list(get("fallbacks"))) or (dynamic.fallbacks if dynamic.fallbacks else None) # pyright: ignore[reportAssignmentType] # the process list as given ) refuse_malformed_fallbacks(configured) - default_fallbacks: Final = get("default_fallbacks") or dynamic.default_fallbacks - if isinstance(default_fallbacks, str): + # As the Python core: a given default_fallbacks, even an empty one, adds a `{"*": ...}` entry + # (`[]` falls back to the process default_fallbacks, so it may add `{"*": None}`). + given_default_fallbacks: Final = get("default_fallbacks") + has_default_fallbacks: Final = given_default_fallbacks is not None or dynamic.default_fallbacks is not None + default_fallbacks: Final = given_default_fallbacks or dynamic.default_fallbacks + if has_default_fallbacks and isinstance(default_fallbacks, str): raise TypeError( f"default_fallbacks must be a list of model names, got '{default_fallbacks}'. Use ['{default_fallbacks}']" ) fallbacks: Final[list[object] | None] = ( # mutable-ok: see above - configured - if default_fallbacks is None - else [*(configured if configured is not None else ()), {"*": default_fallbacks}] + [*(configured if configured is not None else ()), {"*": default_fallbacks}] + if has_default_fallbacks + else configured ) content_policy: Final = list(_as_list(get("content_policy_fallbacks"))) or ( list(dynamic.content_policy_fallbacks) if dynamic.content_policy_fallbacks else None @@ -561,23 +581,23 @@ class Resolved: if strategy is None and "routing_strategy" in config.arguments else normalize_strategy(strategy) or "simple-shuffle", routing_strategy_args=MappingProxyType(dict(as_str_mapping(get("routing_strategy_args")) or {})), - num_retries=int(str(num_retries)) + num_retries=num_retries if num_retries is not None else dynamic.num_retries if dynamic.num_retries is not None else 2, - max_fallbacks=int(str(max_fallbacks)) + max_fallbacks=max_fallbacks if max_fallbacks is not None else dynamic.max_fallbacks if dynamic.max_fallbacks is not None else dynamic.router_max_fallbacks, - allowed_fails=int(str(allowed_fails)) if allowed_fails is not None else dynamic.allowed_fails, - explicit_allowed_fails=int(str(allowed_fails)) if allowed_fails is not None else None, + allowed_fails=allowed_fails if allowed_fails is not None else dynamic.allowed_fails, + explicit_allowed_fails=allowed_fails, cooldown_time=float(str(get("cooldown_time") or _DEFAULT_COOLDOWN_SECONDS)), disable_cooldowns=_optional_bool(get("disable_cooldowns")), allowed_fails_policy=resolved_allowed_fails_policy(get("allowed_fails_policy")), - retry_after=float(str(get("retry_after") or 0)), - timeout=float(str(timeout)) if timeout else dynamic.request_timeout, + retry_after=get("retry_after") if "retry_after" in config.arguments else 0, + timeout=timeout if timeout else dynamic.request_timeout, fallbacks=fallbacks, context_window_fallbacks=list(_as_list(get("context_window_fallbacks"))) or (list(dynamic.context_window_fallbacks) if dynamic.context_window_fallbacks else None), @@ -628,7 +648,7 @@ class Resolved: if name in {"model_group_alias", "default_litellm_params"}: updates[name] = MappingProxyType(dict(as_str_mapping(value) or {})) elif name in _NUMBER_SETTINGS: - updates[name] = self._number_setting(name, value) + updates[name] = self._number_setting(name, value, updated=name in updated) if name == "allowed_fails": updates["explicit_allowed_fails"] = updates[name] elif name in _SELECTION_SETTINGS: @@ -648,16 +668,18 @@ class Resolved: updates["routing_group_members"] = None return dataclasses.replace(self, **updates, assigned=self.assigned | frozenset(values)) # pyright: ignore[reportArgumentType] # each value converted to its field's type above - def _number_setting(self, name: str, value: object) -> object: - """A numeric setting converted to its field's type.""" - if name in ("num_retries", "max_fallbacks", "allowed_fails"): - return int(str(value)) + def _number_setting(self, name: str, value: object, *, updated: bool = False) -> object: + """A numeric setting converted to its field's type. ``retry_after`` and ``timeout`` are kept + as the Python core keeps them: ``update_settings`` stores ``int(value)``, an attribute write + the value as given.""" + if name in ("retry_after", "timeout", "num_retries", "allowed_fails"): + return int(value) if updated else value # pyright: ignore[reportArgumentType] # int() of the value as given, as Python casts it + if name == "max_fallbacks": + return value if name == "cooldown_time": # Stored as given (`_core_cooldown_time`), so a failure cools down for that long: # 0 for no time at all; None leaves the cooldown cache's own default, set by the caller. return float(str(value)) if value is not None else self.cooldown_time - if name == "retry_after": - return float(str(value or 0)) return float(str(value)) if value is not None else None def _selection_setting(self, name: str, value: object, *, validate: bool) -> object: @@ -681,8 +703,8 @@ class Resolved: "cooldown_time": self.cooldown_time, "disable_cooldowns": self.disable_cooldowns, "allowed_fails_policy": _plain(self.allowed_fails_policy), - "retry_after": self.retry_after, - "timeout": self.timeout, + "retry_after": float(str(self.retry_after or 0)), + "timeout": float(str(self.timeout)) if self.timeout is not None else None, "routing_groups": group_documents(self.routing_groups) if self.routing_groups else None, "enable_pre_call_checks": True if self.enable_pre_call_checks else None, "routing_plugins": True if self.routing_plugins else None, @@ -706,16 +728,16 @@ class Resolved: "fallbacks": _plain(self.fallbacks), "context_window_fallbacks": _plain(self.context_window_fallbacks), "content_policy_fallbacks": _plain(self.content_policy_fallbacks), - "num_retries": self.num_retries, - "max_fallbacks": self.max_fallbacks, - "allowed_fails": self._router_allowed_fails(), + "num_retries": _crate_int(self.num_retries), + "max_fallbacks": _crate_int(self.max_fallbacks), + "allowed_fails": _crate_int(self._router_allowed_fails()), "fallback_keys": self._fallback_keys(model_names), "default_deployment": _plain(served_default_deployment(self.default_deployment)[0]), "default_deployment_raises": True if served_default_deployment(self.default_deployment)[1] else None, } return {name: value for name, value in values.items() if value is not None} - def _router_allowed_fails(self) -> int | None: + def _router_allowed_fails(self) -> object: """``allowed_fails`` when it counts as set on the router (``_is_allowed_fails_set_on_router``): passed explicitly, or different from the process default.""" import litellm @@ -1033,7 +1055,7 @@ class RustCore( # Settings the Python core keeps as plain attributes, and settings deleted with `del`. self._plain: Final = initial_plain(config) self._deleted: frozenset[str] = frozenset() - self._request_timeout: Final = request_timeout(config) + self._request_timeout: Final = request_timeout(config, host) legacy: Final[dict[str, object]] = {"deployment_stats": {}} # mutable-ok: the core's own legacy attributes self._legacy: Final = legacy self._health_view: RustHealth | None = None @@ -1390,7 +1412,7 @@ class RustCore( else default_litellm_params(self._config), default_max_parallel_requests=self._endpoints.default_max_parallel_requests, # pyright: ignore[reportArgumentType] # the setting as given cooldown_time=self._cooldown_given, # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it - allowed_fails=r.allowed_fails, + allowed_fails=r.allowed_fails, # pyright: ignore[reportArgumentType] # the value as given, as the Python core stores it allowed_fails_policy=r.allowed_fails_policy, # pyright: ignore[reportArgumentType] # the policy as passed disable_cooldowns=r.disable_cooldowns, enable_pre_call_checks=r.enable_pre_call_checks, @@ -2018,8 +2040,8 @@ class RustCore( **self._routing_fields(prepared), **pinned_video_deployment(plan, request), **unrouted_fields(plan), - # The sync path never runs pre-routing; a custom strategy picks on both. - **({} if plan.sync and self._custom_strategy is None else {"pre_routes": True}), + # A custom strategy picks on both paths; the sync path pre-routes when it can. + **({"pre_routes": True} if not plan.sync or self._sync_pick_pre_routes() else {}), **({"mock_failure": mock} if mock is not None else {}), # Every target of the walk is refused. **({"fallbacks": []} if rejected_fallback_target(request) is not None else {}), @@ -2389,6 +2411,14 @@ class RustCore( messages, ) + def _sync_pick_pre_routes(self) -> bool: + """Whether a sync pick asks for the ``pre_route`` step: always for a custom strategy, else + when a strategy router is registered. A router with routing plugins never pre-routes a sync + pick, so router-core refuses it (the pipeline only runs on the async path).""" + if self._custom_strategy is not None: + return True + return not self._resolved.routing_plugins and self._prerouter.has_strategy_routers() + def _custom_pick_sync( self, step: Mapping[str, object], @@ -2397,10 +2427,15 @@ class RustCore( input: object = _NO_INPUT, specific_deployment: object = _NO_INPUT, ) -> object: - """A custom strategy's sync pick (``get_available_deployment``); without one, no pre-routing.""" + """A custom strategy's sync pick (``get_available_deployment``); without one, the async + pre-routing hook run to completion (``_pre_route``).""" custom: Final = self._custom_strategy if custom is None: - return None + from litellm.litellm_core_utils import asyncify + + return asyncify.run_async_function( # pyright: ignore[reportUnknownMemberType] # the untyped sync bridge + partial(self._pre_route, step, request, messages, input, specific_deployment) + ) picked: Final[object] = custom.get_available_deployment( # pyright: ignore[reportUnknownMemberType] # the strategy's untyped pick model=str(step.get("model")), messages=request.get("messages") if messages is _NO_INPUT else messages, # pyright: ignore[reportArgumentType] # see _pre_route @@ -2665,9 +2700,6 @@ class RustCore( message: Final = ( f"Model {model} has no deployments configured with use_in_pass_through=True. " "Please add use_in_pass_through: true to the deployment configuration" - if pass_through == "async_pass_through" - else f"Model {model} has no deployment configured with use_in_pass_through=True. " - "Please add use_in_pass_through: true in the deployment configuration" ) return litellm_exceptions.BadRequestError(message=message, model=model, llm_provider="") if kind == "context_window_exceeded": @@ -2805,7 +2837,7 @@ class _Selection: fields["specific_deployment"] = True if probe: fields["health_check_probe"] = True - if kind == "pick" or (kind == "sync_pick" and core._custom_strategy is not None): # pyright: ignore[reportPrivateUsage] # the selection serves its own core + if kind == "pick" or (kind == "sync_pick" and core._sync_pick_pre_routes()): # pyright: ignore[reportPrivateUsage] # the selection serves its own core fields["pre_routes"] = True self._specific_deployment: Final = specific_deployment # The error a callback filter raised, which a failed selection raises. @@ -3262,15 +3294,35 @@ class _Driver: """A routing pre-call check refused the attempt (``async_routing_strategy_pre_call_checks``): a rate limit cools the deployment down (``set_cooldown_deployments``); the attempt's failure is recorded by nothing else.""" + report: Final = self._refusal_report(deployment, error) + if report is not None: + await self._core._drive_cache_steps(report) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + return {**self._failed(step, deployment, call, error), "recorded": True} + + def _refused_sync( + self, + step: Mapping[str, object], + deployment: Mapping[str, object], + call: "AttemptCall", + error: BaseException, + ) -> Mapping[str, object]: + """``_refused`` on the sync path (``routing_strategy_pre_call_checks``): a rate limit cools + the deployment down here too.""" + report: Final = self._refusal_report(deployment, error) + if report is not None: + self._core._drive_sync_steps(report) # pyright: ignore[reportPrivateUsage] # the driver serves its own core + return {**self._failed(step, deployment, call, error), "recorded": True} + + def _refusal_report(self, deployment: Mapping[str, object], error: BaseException) -> "RouterCall | None": + """The uncounted failure report that cools down a deployment a pre-call check rate-limited.""" import litellm.exceptions as litellm_exceptions - core: Final = self._core - if isinstance(error, litellm_exceptions.RateLimitError): - deployment_id: Final = (as_str_mapping(deployment.get("model_info")) or {}).get("id") - if isinstance(deployment_id, str): - report: Final = core._handle.report_failure(deployment_id, dict(attempt_failure(error)), uncounted=True) # pyright: ignore[reportPrivateUsage] # the driver serves its own core - await core._drive_cache_steps(report) # pyright: ignore[reportPrivateUsage] # see above - return {**self._failed(step, deployment, call, error), "recorded": True} + if not isinstance(error, litellm_exceptions.RateLimitError): + return None + deployment_id: Final = (as_str_mapping(deployment.get("model_info")) or {}).get("id") + if not isinstance(deployment_id, str): + return None + return self._core._handle.report_failure(deployment_id, dict(attempt_failure(error)), uncounted=True) # pyright: ignore[reportPrivateUsage] # the driver serves its own core def _pick_kwargs(self, step: Mapping[str, object]) -> Mapping[str, object]: """The request kwargs the pick sees: the request with its level's retry count.""" @@ -3365,8 +3417,7 @@ class _Driver: try: self._core._endpoints.sync_pre_call_checks(self._plan, deployment, self._request) # pyright: ignore[reportPrivateUsage] # see above except Exception as refused: # noqa: BLE001 # handed to the Rust call loop like any attempt failure - # The sync path records no outcome for a refused pre-call check: no cooldown. - return {**self._failed(step, deployment, call, refused), "recorded": True} + return self._refused_sync(step, deployment, call, refused) answered: Final = self._core._host.invoke(call.host_op)(**call.kwargs) # pyright: ignore[reportPrivateUsage] # a sync endpoint if inspect.isawaitable(answered): raise _NotSynchronous(f"the host's {call.host_op} endpoint is not synchronous") diff --git a/litellm/router_rust_endpoints.py b/litellm/router_rust_endpoints.py index d63a9d93436..07f091d58e1 100644 --- a/litellm/router_rust_endpoints.py +++ b/litellm/router_rust_endpoints.py @@ -70,7 +70,7 @@ Kind = Literal[ # rebind-ok: rebound as the Python router rebinds it "search", ] Stamp = Literal["deployment", "retry_skip", "none"] # rebind-ok: rebound as the Python router rebinds it -CountKey = Literal["deployment", "group", "empty"] # rebind-ok: rebound as the Python router rebinds it +CountKey = Literal["deployment", "group"] # rebind-ok: rebound as the Python router rebinds it # Request keys the router reads to route the call; the deployment call never receives them. ROUTING_KEYS: Final = frozenset( @@ -174,9 +174,6 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType( sync=True, splits_stream_timeout=True, stamp="deployment", - total_key=None, - success_key=None, - fail_key=None, named=("messages",), ), "atext_completion": Plan( @@ -187,6 +184,15 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType( fail_key="group", named=("prompt",), ), + "text_completion": Plan( + "text_completion", + "text_completion", + "text", + sync=True, + splits_stream_timeout=True, + fail_key="group", + named=("prompt",), + ), # `_asearch_with_fallbacks`: the search tool name routes as the model, with no deployment # picked; its helper picks among the router's search tools. "asearch": Plan( @@ -208,9 +214,6 @@ PLANS: Final[Mapping[str, Plan]] = MappingProxyType( "image", sync=True, prepare_field=None, - total_key="empty", - success_key="empty", - fail_key="empty", named=("prompt",), ), "atranscription": Plan( @@ -379,7 +382,7 @@ class Counters: @staticmethod def _key(which: CountKey, model_name: str, group: str) -> str: - return model_name if which == "deployment" else group if which == "group" else "" + return model_name if which == "deployment" else group def started(self, plan: Plan, call: AttemptCall) -> None: if plan.total_key is not None: @@ -492,9 +495,7 @@ def attempt_call( function_name="generic_api_call", splits_stream_timeout=plan.splits_stream_timeout, ) - endpoint = kwargs.get("endpoint") # rebind-ok: rebound as the Python router rebinds it - if endpoint and isinstance(endpoint, str): - kwargs["endpoint"] = endpoint.replace(requested, model_name) + rewrite_endpoint(core, kwargs, requested, model_name) kwargs["model"] = model_name final = { # rebind-ok: rebound as the Python router rebinds it **data, @@ -516,6 +517,33 @@ def attempt_call( return AttemptCall(plan.host_op, kwargs, kwargs, requested, group) +def rewrite_endpoint( + core: "RustCore", + kwargs: dict[str, object], # mutable-ok: the attempt's own kwargs, shaped in place + requested: str, + model_name: str, +) -> None: + """`_add_deployment_model_to_endpoint_for_llm_passthrough_route`: the requested name, as a whole + path segment of the request's endpoint, becomes the deployment's model without its provider.""" + from litellm.llms.base_llm.passthrough.transformation import replace_path_segment + + endpoint: Final = kwargs.get("endpoint") + if not endpoint: + return + try: + stripped, _, _, _ = core._host.invoke("get_llm_provider")( # pyright: ignore[reportPrivateUsage] # the core's host + model=model_name, + custom_llm_provider=kwargs.get("custom_llm_provider"), + api_base=kwargs.get("api_base"), + ) + replacement = str(stripped) # rebind-ok: rebound as the Python router rebinds it + except Exception: # noqa: BLE001 # as the Python router: an unresolvable model is used as is + replacement = model_name # rebind-ok: rebound as the Python router rebinds it + kwargs["endpoint"] = replace_path_segment( # rebind-ok: the attempt's kwargs, shaped in place + str(endpoint), requested, replacement + ) + + def _generic_attempt( core: "RustCore", plan: Plan, @@ -526,7 +554,6 @@ def _generic_attempt( group: str, ) -> AttemptCall: """`_ageneric_api_call_with_fallbacks_helper` for one deployment.""" - from litellm.llms.base_llm.passthrough.transformation import replace_path_segment from litellm.router_rust_websocket import turn_admission socket: Final = plan.call_type == "_aresponses_websocket" @@ -540,20 +567,7 @@ def _generic_attempt( opens_responses_socket=socket, ) model_name: Final = str(data.get("model")) - endpoint: Final = kwargs.get("endpoint") - if endpoint: - try: - stripped, _, _, _ = core._host.invoke("get_llm_provider")( # pyright: ignore[reportPrivateUsage] # the core's host - model=model_name, - custom_llm_provider=kwargs.get("custom_llm_provider"), - api_base=kwargs.get("api_base"), - ) - replacement = str(stripped) # rebind-ok: rebound as the Python router rebinds it - except Exception: # noqa: BLE001 # as the Python router: an unresolvable model is used as is - replacement = model_name # rebind-ok: rebound as the Python router rebinds it - kwargs["endpoint"] = ( # rebind-ok: the request kwargs are shaped in place, as the Python router shapes them - replace_path_segment(str(endpoint), requested, replacement) - ) + rewrite_endpoint(core, kwargs, requested, model_name) provider: Final = _generic_provider(data) final: Final = { **without_retrieve_stream_defaults(data, kwargs), @@ -604,14 +618,14 @@ def model_client( core: "RustCore", plan: Plan, deployment: Mapping[str, object], kwargs: Mapping[str, object] ) -> object: """The SDK client cached for the deployment (`_get_async_openai_model_client`; `_get_client` - with the sync keys for sync completion and embedding), dropped when the request carries + with the sync keys for sync completion, text completion and embedding), dropped when the request carries another api key.""" from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs if as_dict(deployment.get("litellm_params")).get("kubernetes_pod_discovery") is True: return None model_id: Final = as_dict(deployment.get("model_info")).get("id") - sync_keys: Final = plan.sync and plan.kind in ("chat", "embedding") + sync_keys: Final = plan.sync and plan.kind in ("chat", "embedding", "text") stream: Final = "stream_" if kwargs.get("stream") is True else "" key: Final = f"{model_id}_{stream}client" if sync_keys else f"{model_id}_{stream}async_client" span: Final = get_parent_otel_span_from_kwargs(dict(kwargs)) @@ -832,14 +846,13 @@ class RustEndpoints: """The public endpoint methods of one Rust core, beyond the routed call loop the core drives.""" def __init__(self, core: "RustCore", config: "RouterConfig") -> None: - from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout from litellm.router_utils.request_priority import InvalidPriority, parse_default_priority self._core: Final = core self.counters: Final = Counters() explicit: Final = config.get("timeout") self._explicit_timeout: Final = explicit - self._request_timeout: Final = get_configured_request_timeout() if explicit is not None else None + self._request_timeout: Final = self.process_request_timeout() if explicit is not None else None self._stream_timeout: Final = config.get("stream_timeout") # apply() flips it after construction, as on the Python core self._cache_responses: bool = bool(config.get("cache_responses")) @@ -927,8 +940,16 @@ class RustEndpoints: request_timeout=self._request_timeout, stream_timeout=self._stream_timeout, cache_responses=self._cache_responses, + process_request_timeout=self.process_request_timeout, ) + def process_request_timeout(self) -> float | None: + """The host's explicitly configured process ``request_timeout``, else None.""" + from litellm.litellm_core_utils.request_timeout_resolver import configured_request_timeout + + dynamic: Final = self._core._host.dynamic() # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core + return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set) + def default_litellm_params(self, resolved: "Resolved") -> Mapping[str, object]: """The defaults the Python constructor completes before any call: the router ``timeout`` argument, no client retries, and the caching groups in the default metadata.""" @@ -1495,30 +1516,12 @@ class RustEndpoints: return await self._direct("amoderation", prepared) def text_completion(self, request: Mapping[str, object]) -> object: - """The sync `text_completion`: one pick, no retries, the group name as the model.""" - from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin - - kwargs: Final = {k: v for k, v in request.items() if k not in ("is_retry", "is_fallback", "is_async")} - model: Final = kwargs.get("model") - prompt: Final = kwargs.get("prompt") + """The sync `text_completion`: routed, retried and fallen back as `atext_completion` is.""" + model: Final = request.get("model") if not isinstance(model, str): raise TypeError("text_completion() missing 1 required positional argument: 'model'") - kwargs["num_retries"] = kwargs.get("num_retries", self._core._resolved.num_retries) # pyright: ignore[reportPrivateUsage] # see above - metadata_bucket(kwargs, "metadata").update({"model_group": model}) - specific: Final = kwargs.pop("specific_deployment", None) - deployment: Final = self.select_sync(model, kwargs, specific=bool(specific)) - data: Final = as_dict(deployment.get("litellm_params")) - for key, value in self._core._resolved.default_litellm_params.items(): # pyright: ignore[reportPrivateUsage] # see above - if key not in kwargs: - kwargs[key] = value - elif key == "metadata": - bucket = kwargs[key] - if isinstance(bucket, dict): - bucket.update(as_dict(value)) # pyright: ignore[reportUnknownMemberType] # the request's metadata dict - pinned: Final = deployment_model_id_pin(data) - if pinned is not None: - kwargs["model_id"] = pinned - return self._invoke("text_completion")(**{**data, "prompt": prompt, "caching": self._cache_responses, **kwargs}) + kwargs: Final = {k: v for k, v in request.items() if k not in ("is_retry", "is_fallback", "is_async")} + return self._core._call_sync(PLANS["text_completion"], kwargs) # pyright: ignore[reportPrivateUsage] # the endpoints serve their own core def select_sync(self, model: str, request: Mapping[str, object], *, specific: bool = False) -> Mapping[str, object]: """Pick a deployment for ``model`` on the sync path (`get_available_deployment`).""" diff --git a/litellm/router_rust_prerouting.py b/litellm/router_rust_prerouting.py index bd86c93456a..e430b61f11a 100644 --- a/litellm/router_rust_prerouting.py +++ b/litellm/router_rust_prerouting.py @@ -705,6 +705,10 @@ class PreRouter: # -- the step -------------------------------------------------------------------------------- + def has_strategy_routers(self) -> bool: + """Whether any auto, complexity, adaptive or quality router is registered.""" + return self._strategies.any_registered() + async def pre_route( self, model: str, diff --git a/litellm/router_rust_registry.py b/litellm/router_rust_registry.py index 6a9b4fac2d7..8216a65e109 100644 --- a/litellm/router_rust_registry.py +++ b/litellm/router_rust_registry.py @@ -669,7 +669,7 @@ class HostRegistry(HostModelInfo): info_dict: UntypedEntry = model_info if isinstance(model_info, dict) else {} # pyright: ignore[reportAssignmentType] # a configured model_info declared_id = None if info_dict.get("id") is None else str(info_dict["id"]) if "id" not in info_dict: - info_dict["id"] = deployment_id(str(model_name), litellm_params) + info_dict["id"] = deployment_id(model_name, litellm_params) # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it organization = litellm_params.get("organization") if isinstance(organization, list): # Each organization's deployment has its own id (``organization_deployment_id``). @@ -677,7 +677,7 @@ class HostRegistry(HostModelInfo): litellm_params["organization"] = org self._create_deployment( model, - str(model_name), + model_name, # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it litellm_params, { **info_dict, @@ -685,7 +685,7 @@ class HostRegistry(HostModelInfo): declared_id, position, org, - lambda: deployment_id(str(model_name), litellm_params), + lambda: deployment_id(model_name, litellm_params), # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it ), }, declared_id=declared_id, @@ -694,12 +694,14 @@ class HostRegistry(HostModelInfo): else: self._create_deployment( model, - str(model_name), + model_name, # pyright: ignore[reportArgumentType] # the configured model_name, as Python passes it litellm_params, info_dict, declared_id=declared_id, duplicate_ids=duplicate_ids, ) + # Python logs the group names here, which reads every alias entry (a malformed one raises). + verbose_router_logger.debug("\nInitialized Model List %s", self._core_get_model_names()) self._core_model_names = {str(_LITELLM_PARAMS_ADAPTER.validate_python(m).get("model_name")) for m in model_list} if self.strategies is not None: self.strategies.finalize() diff --git a/litellm/router_rust_selection.py b/litellm/router_rust_selection.py index 1e2e0386fc8..bb04b48403a 100644 --- a/litellm/router_rust_selection.py +++ b/litellm/router_rust_selection.py @@ -281,13 +281,14 @@ def pick_sync( input: object, request_kwargs: dict[str, object] | None, # mutable-ok: see pick_async ) -> object: - """``_select_deployment_sync``: cost-based routing has no sync pick.""" + """``_select_deployment_sync``.""" match selector: case LeastBusyLoggingHandler() if strategy == "least-busy": return selector.get_available_deployments(model_group=model, healthy_deployments=deployments) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts - case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() if strategy in ( + case LowestTPMLoggingHandler() | LowestTPMLoggingHandler_v2() | LowestCostLoggingHandler() if strategy in ( "usage-based-routing", "usage-based-routing-v2", + "cost-based-routing", ): return selector.get_available_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selectors return untyped deployment dicts model_group=model, diff --git a/litellm/router_rust_settings.py b/litellm/router_rust_settings.py index ac27c7970e2..3d9ab873157 100644 --- a/litellm/router_rust_settings.py +++ b/litellm/router_rust_settings.py @@ -184,11 +184,15 @@ def _as_dict(value: object) -> Mapping[str, object] | None: return value if isinstance(value, dict) else None # pyright: ignore[reportUnknownVariableType] # the argument dict, as passed -def request_timeout(config: "RouterConfig") -> float | None: - """The per-attempt ``request_timeout``: only stored when a router timeout is also set.""" - from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout +def request_timeout(config: "RouterConfig", host: "PythonHost") -> float | None: + """The per-attempt ``request_timeout``: the host's configured process ``request_timeout``, only + stored when a router timeout is also set.""" + from litellm.litellm_core_utils.request_timeout_resolver import configured_request_timeout - return get_configured_request_timeout() if config.get("timeout") is not None else None + if config.get("timeout") is None: + return None + dynamic: Final = host.dynamic() + return configured_request_timeout(dynamic.request_timeout, dynamic.request_timeout_explicitly_set) def apply_construction_effects(config: "RouterConfig", host: "PythonHost") -> None: diff --git a/litellm/router_rust_shaping.py b/litellm/router_rust_shaping.py index 08707dbe9f8..92a8364de24 100644 --- a/litellm/router_rust_shaping.py +++ b/litellm/router_rust_shaping.py @@ -8,7 +8,7 @@ the same precedence and the same oddities. Nothing here reads a Python router. """ import dataclasses -from collections.abc import Mapping, MutableMapping, Sequence +from collections.abc import Callable, Mapping, MutableMapping, Sequence from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Final @@ -62,6 +62,8 @@ _FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt", "anthro _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" # Generic handlers whose stream timeout is split from the request timeout. STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"}) +# The generic calls (sync and async) resolve their timeout with the LLM pass-through resolver. +GENERIC_CALL_FUNCTION_NAMES: Final = frozenset({"_ageneric_api_call_with_fallbacks", "generic_api_call"}) @dataclasses.dataclass(frozen=True, slots=True) @@ -70,7 +72,8 @@ class Shaping: ``timeout`` is the router's ``timeout`` (its own setting, else the process ``request_timeout``), ``request_timeout`` the explicitly configured process ``request_timeout``, kept only when the - router has a timeout of its own, as the Python router keeps them.""" + router has a timeout of its own, as the Python router keeps them; ``process_request_timeout`` + reads the host's explicitly configured process ``request_timeout`` at call time.""" default_litellm_params: Mapping[str, object] timeout: object @@ -78,6 +81,7 @@ class Shaping: request_timeout: float | None stream_timeout: object cache_responses: bool + process_request_timeout: Callable[[], float | None] = lambda: None def as_dict(value: object) -> dict[str, object]: # mutable-ok: a fresh dict of a request mapping @@ -532,15 +536,17 @@ def _set_timeouts( function_name: str | None, splits_stream_timeout: bool, ) -> None: - if function_name == "_ageneric_api_call_with_fallbacks": + if function_name in GENERIC_CALL_FUNCTION_NAMES: from litellm.passthrough.timeout_utils import resolve_llm_passthrough_timeout + # The process request_timeout, read through the host, ranks right after the router timeout, + # as resolve_llm_passthrough_timeout ranks it. router_timeout: Final = ( shaping.request_timeout if shaping.request_timeout is not None else float(shaping.explicit_timeout) if isinstance(shaping.explicit_timeout, (int, float)) - else None + else shaping.process_request_timeout() ) router_stream_timeout: Final = ( shaping.stream_timeout diff --git a/litellm/router_rust_streaming.py b/litellm/router_rust_streaming.py index 8d3fff68ddd..02203cf80d1 100644 --- a/litellm/router_rust_streaming.py +++ b/litellm/router_rust_streaming.py @@ -55,7 +55,8 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( ) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, set_hidden_params from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -from litellm.types.utils import ModelResponseStream, Usage +from litellm.router_utils.fallback_event_handlers import non_stream_fallback_answer_as_chunk +from litellm.types.utils import ModelResponse, ModelResponseStream, Usage if TYPE_CHECKING: from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator @@ -617,6 +618,17 @@ def _adopt_fallback_response_headers( return prepared +def _non_stream_fallback_item( + wrapper_ref: "weakref.ref[FallbackAwareStreamWrapper]", fallback_response: object +) -> object: + """The one item a stream yields for a fallback that answered without a stream: its response as + a stream chunk, carrying the fallback's hidden params and headers.""" + prepared: Final = _adopt_fallback_response_headers(wrapper_ref, fallback_response) + item: Final = non_stream_fallback_answer_as_chunk(fallback_response) + _apply_fallback_hidden_params_to_item(item, prepared) + return item + + def _chat_wrapper_init( wrapper: FallbackAwareStreamWrapper, stream: object, model_response: CustomStreamWrapper ) -> None: @@ -748,8 +760,7 @@ def wrap_chat_stream( # noqa: C901 # one function per Python wrapper it ports ) yield fallback_item else: - # A fallback that answers without a stream yields None (an oddity kept as is). - yield None + yield _non_stream_fallback_item(wrapper_ref, fallback_response) except Exception as fallback_error: verbose_router_logger.error("Fallback also failed: %s", fallback_error) if isinstance(trigger, MidStreamFallbackError) and trigger.original_exception is not None: @@ -801,7 +812,7 @@ def wrap_chat_stream_sync( trigger: Final = _fallback_trigger(e, (litellm_exceptions.ContextWindowExceededError,)) try: fallback_response = attempt.fallback_sync(trigger) # rebind-ok: rebound as the stream advances - if hasattr(fallback_response, "__iter__"): + if hasattr(fallback_response, "__iter__") and not isinstance(fallback_response, ModelResponse): prepared = _adopt_fallback_response_headers(wrapper_ref, fallback_response) # rebind-ok: re-adopted settled = False # rebind-ok: rebound as the stream advances for fallback_item in cast("Iterable[object]", fallback_response): # cast-ok: checked @@ -817,7 +828,7 @@ def wrap_chat_stream_sync( _combine_fallback_usage(fallback_item, None) yield fallback_item else: - yield None + yield _non_stream_fallback_item(wrapper_ref, fallback_response) except Exception as fallback_error: verbose_router_logger.error("Fallback also failed: %s", fallback_error) if isinstance(trigger, MidStreamFallbackError) and trigger.original_exception is not None: diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index 3f0a5412b62..035d18e0317 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -209,15 +209,32 @@ class LowestCostLoggingHandler(CustomLogger): """ Returns a deployment with the lowest cost """ - cost_key: Final = f"cost_map:{model_group}" + request_count_dict: Final = await self.router_cache.async_get_cache(key=f"cost_map:{model_group}") or {} # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals + return self._lowest_cost_deployment(request_count_dict, healthy_deployments, messages, input) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals - request_count_dict: Final = await self.router_cache.async_get_cache(key=cost_key) or {} - - # ----------------------- - # Find lowest used model - # ---------------------- - float("inf") + @with_service_target("router_usage") + def get_available_deployments( # pyright: ignore[reportUnknownParameterType] # noqa: ANN201 # the selector's untyped deployment dict, as the async twin returns it + self, + model_group: str, + healthy_deployments: list, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the router's list of deployment dicts, as the async twin takes it + messages: list[dict[str, str]] | None = None, # mutable-ok: the request's messages, as given + input: str | list | None = None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the request's input, as given + request_kwargs: dict | None = None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: unused, the same signature as the async twin + ): + """ + The sync twin of `async_get_available_deployments`: the deployment with the lowest cost + """ + request_count_dict: Final = self.router_cache.get_cache(key=f"cost_map:{model_group}") or {} # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals + return self._lowest_cost_deployment(request_count_dict, healthy_deployments, messages, input) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy untyped selector internals + def _lowest_cost_deployment( # pyright: ignore[reportUnknownParameterType] # noqa: ANN202 # the selector's untyped deployment dict + self, + request_count_dict: dict | None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the untyped cached usage map, filled in place as before + healthy_deployments: list, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the router's list of deployment dicts + messages: list[dict[str, str]] | None, # mutable-ok: the request's messages, as given + input: str | list | None, # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType] # mutable-ok: the request's input, as given + ): + """The cheapest healthy deployment still under its tpm/rpm limits this minute, else None.""" current_date: Final = datetime.now().strftime("%Y-%m-%d") current_hour: Final = datetime.now().strftime("%H") current_minute: Final = datetime.now().strftime("%M") @@ -296,8 +313,8 @@ class LowestCostLoggingHandler(CustomLogger): verbose_router_logger.debug( "item_cost: %s, item_tpm: %s, item_rpm: %s, model_id: %s", item_cost, - item_tpm, - item_rpm, + item_tpm, # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage map + item_rpm, # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage map _deployment.get("model_info", {}).get("id"), ) diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index 0719b30d36d..833f8382957 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -469,6 +469,68 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): [f"{prefix}:rpm:{current_minute}" for prefix in prefixes], ) + def _no_deployments_available_error( + self, + model_group: str, + healthy_deployments: Sequence[object], + tpm_values: Sequence[object] | None, + rpm_values: Sequence[object] | None, + ) -> litellm.RateLimitError: + """The error both the sync and the async pick raise when no deployment is under its limits.""" + ### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ### + deployment_dict: Final = {} + for index, _deployment in enumerate(healthy_deployments): + if isinstance(_deployment, dict): + id = _deployment.get("model_info", {}).get("id") + ### GET DEPLOYMENT TPM LIMIT ### + _deployment_tpm = None + if _deployment_tpm is None: + _deployment_tpm = _deployment.get("tpm", None) + if _deployment_tpm is None: + _deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None) + if _deployment_tpm is None: + _deployment_tpm = _deployment.get("model_info", {}).get("tpm", None) + if _deployment_tpm is None: + _deployment_tpm = float("inf") + + ### GET CURRENT TPM ### + current_tpm = tpm_values[index] if tpm_values else 0 + + ### GET DEPLOYMENT TPM LIMIT ### + _deployment_rpm = None + if _deployment_rpm is None: + _deployment_rpm = _deployment.get("rpm", None) + if _deployment_rpm is None: + _deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None) + if _deployment_rpm is None: + _deployment_rpm = _deployment.get("model_info", {}).get("rpm", None) + if _deployment_rpm is None: + _deployment_rpm = float("inf") + + ### GET CURRENT RPM ### + current_rpm = rpm_values[index] if rpm_values else 0 + + deployment_dict[id] = { + "current_tpm": current_tpm, + "tpm_limit": _deployment_tpm, + "current_rpm": current_rpm, + "rpm_limit": _deployment_rpm, + } + return litellm.RateLimitError( + message=f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}", + llm_provider="", + model=model_group, + response=httpx.Response( + status_code=429, + content="", + headers={"retry-after": str(60)}, + request=httpx.Request( + method="tpm_rpm_limits", + url="https://github.com/BerriAI/litellm", + ), + ), + ) + @with_service_target("router_usage") async def async_get_available_deployments( self, @@ -520,63 +582,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): input=input, ) - try: - assert deployment is not None - return deployment - except Exception: - ### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ### - deployment_dict: Final = {} - for index, _deployment in enumerate(healthy_deployments): - if isinstance(_deployment, dict): - id = _deployment.get("model_info", {}).get("id") - ### GET DEPLOYMENT TPM LIMIT ### - _deployment_tpm = None - if _deployment_tpm is None: - _deployment_tpm = _deployment.get("tpm", None) - if _deployment_tpm is None: - _deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None) - if _deployment_tpm is None: - _deployment_tpm = _deployment.get("model_info", {}).get("tpm", None) - if _deployment_tpm is None: - _deployment_tpm = float("inf") - - ### GET CURRENT TPM ### - current_tpm = tpm_values[index] if tpm_values else 0 - - ### GET DEPLOYMENT TPM LIMIT ### - _deployment_rpm = None - if _deployment_rpm is None: - _deployment_rpm = _deployment.get("rpm", None) - if _deployment_rpm is None: - _deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None) - if _deployment_rpm is None: - _deployment_rpm = _deployment.get("model_info", {}).get("rpm", None) - if _deployment_rpm is None: - _deployment_rpm = float("inf") - - ### GET CURRENT RPM ### - current_rpm = rpm_values[index] if rpm_values else 0 - - deployment_dict[id] = { - "current_tpm": current_tpm, - "tpm_limit": _deployment_tpm, - "current_rpm": current_rpm, - "rpm_limit": _deployment_rpm, - } - raise litellm.RateLimitError( - message=f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}", - llm_provider="", - model=model_group, - response=httpx.Response( - status_code=429, - content="", - headers={"retry-after": str(60)}, - request=httpx.Request( - method="tpm_rpm_limits", - url="https://github.com/BerriAI/litellm", - ), - ), - ) + if deployment is None: + raise self._no_deployments_available_error(model_group, healthy_deployments, tpm_values, rpm_values) # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage lists + return deployment @with_service_target("router_usage") def get_available_deployments( @@ -631,49 +639,6 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger): input=input, ) - try: - assert deployment is not None - return deployment - except Exception: - ### GET THE DICT OF TPM / RPM + LIMITS PER DEPLOYMENT ### - deployment_dict: Final = {} - for index, _deployment in enumerate(healthy_deployments): - if isinstance(_deployment, dict): - id = _deployment.get("model_info", {}).get("id") - ### GET DEPLOYMENT TPM LIMIT ### - _deployment_tpm = None - if _deployment_tpm is None: - _deployment_tpm = _deployment.get("tpm", None) - if _deployment_tpm is None: - _deployment_tpm = _deployment.get("litellm_params", {}).get("tpm", None) - if _deployment_tpm is None: - _deployment_tpm = _deployment.get("model_info", {}).get("tpm", None) - if _deployment_tpm is None: - _deployment_tpm = float("inf") - - ### GET CURRENT TPM ### - current_tpm = tpm_values[index] if tpm_values else 0 - - ### GET DEPLOYMENT TPM LIMIT ### - _deployment_rpm = None - if _deployment_rpm is None: - _deployment_rpm = _deployment.get("rpm", None) - if _deployment_rpm is None: - _deployment_rpm = _deployment.get("litellm_params", {}).get("rpm", None) - if _deployment_rpm is None: - _deployment_rpm = _deployment.get("model_info", {}).get("rpm", None) - if _deployment_rpm is None: - _deployment_rpm = float("inf") - - ### GET CURRENT RPM ### - current_rpm = rpm_values[index] if rpm_values else 0 - - deployment_dict[id] = { - "current_tpm": current_tpm, - "tpm_limit": _deployment_tpm, - "current_rpm": current_rpm, - "rpm_limit": _deployment_rpm, - } - raise ValueError( - f"{RouterErrors.no_deployments_available.value}. Passed model={model_group}. Deployments={deployment_dict}" - ) + if deployment is None: + raise self._no_deployments_available_error(model_group, healthy_deployments, tpm_values, rpm_values) # pyright: ignore[reportUnknownArgumentType] # the untyped cached usage lists + return deployment diff --git a/litellm/router_utils/endpoint_eligibility.py b/litellm/router_utils/endpoint_eligibility.py index 8e21e10451c..096b55f4d00 100644 --- a/litellm/router_utils/endpoint_eligibility.py +++ b/litellm/router_utils/endpoint_eligibility.py @@ -17,6 +17,7 @@ CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType( "_acompletion": "/v1/chat/completions", "text_completion": "/v1/completions", "atext_completion": "/v1/completions", + "_text_completion": "/v1/completions", "_atext_completion": "/v1/completions", "anthropic_messages": "/v1/messages", "aresponses": "/v1/responses", diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index c929cca44be..86d824a3f78 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -1013,3 +1013,15 @@ _check_non_standard_fallback_format = check_non_standard_fallback_format def run_non_standard_fallback_format(fallbacks: Sequence[str] | Sequence[Mapping[str, object]], model_group: str): pass + + +def non_stream_fallback_answer_as_chunk(fallback_response: object) -> object: + """A mid-stream fallback that answered without a stream: the whole response as the one stream + chunk it is, as a mocked stream sends a full response (`convert_model_response_to_streaming`). + Anything that is not a chat response is yielded as it is.""" + from litellm.llms.base_llm.base_model_iterator import convert_model_response_to_streaming + from litellm.types.utils import ModelResponse + + if isinstance(fallback_response, ModelResponse): + return convert_model_response_to_streaming(fallback_response) + return fallback_response diff --git a/tests/unit/router_strategy/test_router_routing_groups.py b/tests/unit/router_strategy/test_router_routing_groups.py index 5eeb46c3518..e06d8a32993 100644 --- a/tests/unit/router_strategy/test_router_routing_groups.py +++ b/tests/unit/router_strategy/test_router_routing_groups.py @@ -632,11 +632,10 @@ def test_select_deployment_sync_returns_none_without_selector(): assert result is None -def test_select_deployment_sync_does_not_dispatch_cost_based_routing(): +def test_select_deployment_sync_dispatches_cost_based_routing(): """ - `LowestCostLoggingHandler` only implements `async_get_available_deployments`, - so the sync dispatch must fall through to None for `cost-based-routing` - instead of raising AttributeError. + `LowestCostLoggingHandler` has a sync `get_available_deployments` as well, so the sync dispatch + picks with it for `cost-based-routing` as the async dispatch does. """ router = _build_router( routing_strategy="simple-shuffle", @@ -649,21 +648,19 @@ def test_select_deployment_sync_does_not_dispatch_cost_based_routing(): ], ) cost_selector = router._group_selectors["cheap"]["cost-based-routing"] - assert not hasattr(cost_selector, "get_available_deployments"), ( - "test premise broken: LowestCostLoggingHandler unexpectedly grew a " - "sync method — revisit the sync dispatch arm." - ) + healthy = router._core_get_model_list(model_name="filtered-model") # pyright: ignore[reportPrivateUsage] # pytest and the cores' untyped members + assert healthy result = router._select_deployment_sync( strategy="cost-based-routing", selector=cost_selector, model="filtered-model", - healthy_deployments=[], + healthy_deployments=healthy, messages=None, input=None, request_kwargs=None, ) - assert result is None + assert result in healthy @pytest.mark.asyncio @@ -1036,7 +1033,7 @@ def test_sync_usage_based_v2_override_stays_scoped_to_the_request_that_asked_for plain = router.completion(model="other-model", messages=messages, mock_response="ok") assert plain.choices[0].message.content == "ok" - with pytest.raises(ValueError, match="No deployments available"): + with pytest.raises(litellm.RateLimitError, match="No deployments available"): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members router.completion( model="other-model", messages=messages, mock_response="ok", routing_strategy="usage-based-routing-v2" ) diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index a3ff1c30942..618d55ba0c4 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -3448,7 +3448,7 @@ WILDCARD_COOLED: Final = (FIRST_WILDCARD, SECOND_WILDCARD) ("simple-shuffle", EVERY_DEPLOYMENT, "async", "refused", None, ALL_COOLED), ("cost-based-routing", STRATEGY_COOLED, "sync", "refused", "team-a", None), ("cost-based-routing", STRATEGY_COOLED, "async", "refused", "team-a", None), - ("cost-based-routing", STRATEGY_COOLED, "pass-through", "refused", "team-a", None), + ("cost-based-routing", STRATEGY_COOLED, "pass-through-chat", "refused", "team-a", None), ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED), ("simple-shuffle", EVERY_DEPLOYMENT, "async", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED), ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", (ALIAS_TARGET,)), diff --git a/tests/unit/test_router/core/test_construction_precall_characterization.py b/tests/unit/test_router/core/test_construction_precall_characterization.py index a9c07d65552..922aaf76d22 100644 --- a/tests/unit/test_router/core/test_construction_precall_characterization.py +++ b/tests/unit/test_router/core/test_construction_precall_characterization.py @@ -263,24 +263,32 @@ def test_request_timeout_is_unset_without_a_router_timeout( assert _one(make_core).request_timeout is None # pyright: ignore[reportAttributeAccessIssue] -def test_request_timeout_reads_litellm_request_timeout_directly_when_a_router_timeout_is_set( +def test_request_timeout_reads_the_host_request_timeout_when_a_router_timeout_is_set( make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch ) -> None: - """Oddity: this one setting is read from ``litellm.request_timeout`` itself, not from the host's - process settings (which say 33 here).""" + """Read from the host's process settings (33 here), not from ``litellm.request_timeout`` itself.""" host.set_dynamic(request_timeout=33.0) monkeypatch.setattr(litellm, "request_timeout", 77.0) - assert _one(make_core, timeout=10).request_timeout == 77.0 # pyright: ignore[reportAttributeAccessIssue] + assert _one(make_core, timeout=10).request_timeout == 33.0 # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] -def test_request_timeout_is_unset_when_litellm_request_timeout_is_its_untouched_default( - make_core: MakeCore, monkeypatch: pytest.MonkeyPatch +def test_request_timeout_is_unset_when_the_host_request_timeout_is_its_untouched_default( + make_core: MakeCore, + host: ScriptedHost, + monkeypatch: pytest.MonkeyPatch, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members ) -> None: - monkeypatch.setattr(litellm, "request_timeout", 6000) - monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False, raising=False) + host.set_dynamic(request_timeout=6000.0, request_timeout_explicitly_set=False) + monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members assert _one(make_core, timeout=10).request_timeout is None # pyright: ignore[reportAttributeAccessIssue] +def test_request_timeout_keeps_the_default_value_when_the_host_marks_it_explicitly_set( + make_core: MakeCore, host: ScriptedHost +) -> None: + host.set_dynamic(request_timeout=6000.0, request_timeout_explicitly_set=True) + assert _one(make_core, timeout=10).request_timeout == 6000.0 # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + + def test_stream_timeout_is_kept_as_given(make_core: MakeCore) -> None: assert _one(make_core, stream_timeout=3).stream_timeout == 3 # pyright: ignore[reportAttributeAccessIssue] diff --git a/tests/unit/test_router/core/test_endpoints_characterization_generic.py b/tests/unit/test_router/core/test_endpoints_characterization_generic.py index ea4559841d2..382680a24b5 100644 --- a/tests/unit/test_router/core/test_endpoints_characterization_generic.py +++ b/tests/unit/test_router/core/test_endpoints_characterization_generic.py @@ -288,9 +288,9 @@ async def test_generic_call_sends_the_declared_custom_llm_provider(make_core: Ma @pytest.mark.parametrize( ("endpoint", "expected"), - [("aocr", 600.0), ("aresponses", 600.0), ("ocr", 6000.0), ("generate_content", 6000.0)], + [("aocr", 600.0), ("aresponses", 600.0), ("ocr", 600.0), ("generate_content", 600.0), ("responses", 600.0)], ) -async def test_generic_default_timeout_is_the_passthrough_default_for_async_and_the_process_setting_for_sync( +async def test_generic_default_timeout_is_the_passthrough_default_for_sync_and_async( make_core: MakeCore, host: EndpointHost, endpoint: str, expected: float ): core: Final = make_core(model_list=[deployment("g", "d1")]) @@ -306,6 +306,20 @@ async def test_generic_timeout_takes_the_router_timeout(make_core: MakeCore, hos assert host.calls[0].kwargs["timeout"] == 42 +@pytest.mark.parametrize("endpoint", ["aocr", "ocr"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members +async def test_generic_timeout_without_a_router_timeout_takes_the_host_request_timeout( + make_core: MakeCore, + host: EndpointHost, + endpoint: str, + monkeypatch: pytest.MonkeyPatch, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members +): + host.set_dynamic(request_timeout=33.0) + monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members + core: Final = make_core(model_list=[deployment("g", "d1")]) + await _call(core, endpoint, {"model": "g", "document": {}}) + assert host.calls[0].kwargs["timeout"] == 33.0 + + @pytest.mark.parametrize( ("endpoint", "request_body", "expected"), [ diff --git a/tests/unit/test_router/core/test_endpoints_characterization_inference.py b/tests/unit/test_router/core/test_endpoints_characterization_inference.py index 44b90331dc1..6dcbeedf913 100644 --- a/tests/unit/test_router/core/test_endpoints_characterization_inference.py +++ b/tests/unit/test_router/core/test_endpoints_characterization_inference.py @@ -171,25 +171,17 @@ async def test_cache_responses_turned_on_after_construction_reaches_the_host( # --- sync text completion and image generation ---------------------------------------------- -async def test_sync_text_completion_sends_the_router_num_retries_and_no_routing_fields( - make_core: MakeCore, host: EndpointHost -): - core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4) +async def test_sync_text_completion_shapes_the_request_like_atext_completion(make_core: MakeCore, host: EndpointHost): + core: Final = make_core(model_list=[deployment("g", "d1", timeout=7)], num_retries=4) core.text_completion(model="g", prompt="p") sent: Final = _sent(host) - assert sent["num_retries"] == 4 - assert "model_info" not in sent - assert "litellm_trace_id" not in sent - assert sent.get("timeout") is None + assert "num_retries" not in sent + assert sent["model_info"]["id"] == "d1" # pyright: ignore[reportIndexIssue] + trace_id: Final = sent["litellm_trace_id"] + assert isinstance(trace_id, str) and len(trace_id) == 36 + assert sent["timeout"] == 7 metadata: Final = _bucket(sent) - assert metadata["model_group"] == "g" - assert "deployment" not in metadata - - -async def test_sync_text_completion_request_num_retries_wins(make_core: MakeCore, host: EndpointHost): - core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4) - core.text_completion(model="g", prompt="p", num_retries=1) - assert _sent(host)["num_retries"] == 1 + assert (metadata["model_group"], metadata["deployment"]) == ("g", "openai/d1") async def test_sync_text_completion_merges_default_metadata_into_request_metadata( @@ -204,23 +196,36 @@ async def test_sync_text_completion_merges_default_metadata_into_request_metadat assert (metadata["x"], metadata["y"], metadata["model_group"], sent["user"]) == (1, 2, "g", "u") -async def test_sync_text_completion_sends_the_model_group_name_not_the_deployment_model( - make_core: MakeCore, host: EndpointHost -): - """The request's model="g" is spread after the deployment params, so it replaces "openai/d1".""" +async def test_sync_text_completion_sends_the_deployment_model(make_core: MakeCore, host: EndpointHost): core: Final = make_core(model_list=[deployment("g", "d1", api_base="https://d1.example")]) core.text_completion(model="g", prompt="p") call: Final = host.calls[0] - assert (call.model, call.kwargs["api_base"]) == ("g", "https://d1.example") + assert (call.model, call.kwargs["api_base"]) == ("openai/d1", "https://d1.example") -async def test_sync_text_completion_is_not_retried(make_core: MakeCore, host: EndpointHost): - host.answer("g", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) +async def test_sync_text_completion_is_retried_with_the_router_num_retries(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=3) with pytest.raises(litellm.InternalServerError): core.text_completion(model="g", prompt="p") - assert len(host.calls) == 1 - assert host.sleeps == [] + assert len(host.calls) == 4 + + +async def test_sync_text_completion_request_num_retries_wins(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=4) + with pytest.raises(litellm.InternalServerError): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members + core.text_completion(model="g", prompt="p", num_retries=1) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # pytest and the cores' untyped members + assert len(host.calls) == 2 + + +async def test_sync_text_completion_falls_back_like_atext_completion(make_core: MakeCore, host: EndpointHost): + host.answer("openai/d1", litellm.InternalServerError(message="boom", llm_provider="openai", model="d1")) + core: Final = make_core( + model_list=[deployment("g", "d1"), deployment("h", "d2")], num_retries=0, fallbacks=[{"g": ["h"]}] + ) + core.text_completion(model="g", prompt="p") # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # pytest and the cores' untyped members + assert host.deployments_called() == ["d1", "d2"] async def test_sync_image_generation_sends_routing_metadata_without_a_trace_id(make_core: MakeCore, host: EndpointHost): diff --git a/tests/unit/test_router/core/test_gaps_l0001.py b/tests/unit/test_router/core/test_gaps_l0001.py index bb6a4fe6c44..16c2aaf7405 100644 --- a/tests/unit/test_router/core/test_gaps_l0001.py +++ b/tests/unit/test_router/core/test_gaps_l0001.py @@ -364,11 +364,13 @@ async def test_an_override_matching_a_routing_group_strategy_is_not_bound_to_the def test_a_sync_usage_based_v2_override_counts_rpm_before_the_call(make_core: MakeCore, host: ScriptedHost) -> None: """The override selector is on no callback list, so the sync pick runs its pre-call check itself: - the first call is counted against ``rpm=1`` and the next pick finds no deployment. Oddity: the sync - path raises a bare ``ValueError`` where the async one raises ``RateLimitError``.""" + the first call is counted against ``rpm=1`` and the next pick finds no deployment, raising the same + ``RateLimitError`` as the async path.""" core: Final = make_core(model_list=[deployment("g", "a", rpm=1)], num_retries=0) core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue] - with pytest.raises(ValueError, match=re.escape("No deployments available for selected model. Passed model=g.")): + with pytest.raises( # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members + litellm.RateLimitError, match=re.escape("No deployments available for selected model. Passed model=g.") + ): core.completion(model="g", messages=MESSAGES, routing_strategy="usage-based-routing-v2") # pyright: ignore[reportAttributeAccessIssue] assert host.deployments_called() == ["a"] @@ -424,22 +426,31 @@ def test_sync_completion_under_latency_based_routing_reaches_the_group(make_core assert len(host.calls) == 10 -def test_sync_completion_under_cost_based_routing_finds_no_deployment(make_core: MakeCore, host: ScriptedHost) -> None: - """Oddity: the cost-based selector has no sync pick, so every sync pick under cost-based routing - rate-limits without calling a provider.""" +def test_sync_completion_under_cost_based_routing_picks_the_cheapest_deployment( + make_core: MakeCore, host: ScriptedHost +) -> None: + """The cost-based selector picks on the sync path as it does on the async one.""" core: Final = make_core( - model_list=[deployment("g", "a"), deployment("g", "b")], routing_strategy="cost-based-routing", num_retries=0 + model_list=[ + deployment("g", "a", input_cost_per_token=0.5, output_cost_per_token=0.5), + deployment("g", "b", input_cost_per_token=0.1, output_cost_per_token=0.1), + ], + routing_strategy="cost-based-routing", + num_retries=0, ) - with pytest.raises(RouterRateLimitError) as raised: + for _ in range(3): core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] - assert str(raised.value) == NO_DEPLOYMENTS_MESSAGE.format(model="g") - assert host.calls == [] + assert host.deployments_called() == ["b", "b", "b"] @pytest.mark.asyncio -async def test_pass_through_pick_under_cost_based_routing_finds_no_deployment(make_core: MakeCore) -> None: +async def test_pass_through_pick_under_cost_based_routing_picks_the_cheapest_deployment(make_core: MakeCore) -> None: core: Final = make_core( - model_list=[deployment("g", "a", use_in_pass_through=True)], routing_strategy="cost-based-routing" + model_list=[ + deployment("g", "a", use_in_pass_through=True, input_cost_per_token=0.5, output_cost_per_token=0.5), + deployment("g", "b", use_in_pass_through=True, input_cost_per_token=0.1, output_cost_per_token=0.1), + ], + routing_strategy="cost-based-routing", ) - with pytest.raises(RouterRateLimitError): - await core.select("g", kind="pass_through") + picked: Final = await core.select("g", kind="pass_through") # pyright: ignore[reportCallIssue, reportUnknownVariableType] # pytest and the cores' untyped members + assert picked["model_info"]["id"] == "b" # pyright: ignore[reportIndexIssue, reportCallIssue, reportArgumentType] diff --git a/tests/unit/test_router/core/test_gaps_l2800.py b/tests/unit/test_router/core/test_gaps_l2800.py index 78e237f0696..020e482a5ec 100644 --- a/tests/unit/test_router/core/test_gaps_l2800.py +++ b/tests/unit/test_router/core/test_gaps_l2800.py @@ -328,9 +328,10 @@ async def test_a_per_user_github_copilot_deployment_rejects_another_credential_n async def test_the_process_request_timeout_outranks_the_router_timeout_once_a_router_timeout_is_set( make_core: MakeCore, host: ScriptedHost, monkeypatch: pytest.MonkeyPatch ) -> None: - """Oddity: this per-attempt timeout is read from ``litellm.request_timeout`` itself (not the host's - process settings), and only when the router has a ``timeout`` of its own.""" - monkeypatch.setattr(litellm, "request_timeout", 33.0) + """This per-attempt timeout is read from the host's process settings (not ``litellm.request_timeout`` + itself), and only when the router has a ``timeout`` of its own.""" + host.set_dynamic(request_timeout=33.0) + monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42) await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] assert host.calls[0].kwargs["timeout"] == 33.0 @@ -341,7 +342,8 @@ async def test_the_process_request_timeout_is_the_stream_timeout_without_a_strea ) -> None: from tests.unit.test_router.core.streaming_helpers import chat_stream, collect_texts - monkeypatch.setattr(litellm, "request_timeout", 33.0) + host.set_dynamic(request_timeout=33.0) + monkeypatch.setattr(litellm, "request_timeout", 77.0) # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members host.answer("openai/d1", chat_stream("A")) core: Final = make_core(model_list=[deployment("g", "d1")], timeout=42) await collect_texts(await core.acompletion(model="g", messages=MESSAGES, stream=True)) # pyright: ignore[reportAttributeAccessIssue] @@ -424,8 +426,18 @@ async def test_acompletion_counts_total_success_and_failed_calls_per_deployment_ assert counts == (2, 1, 1) -def test_sync_completion_does_not_count_calls(make_core: MakeCore, host: ScriptedHost) -> None: - """Oddity: only the async path updates ``total_calls``/``success_calls``/``fail_calls``.""" - core: Final = make_core(model_list=[deployment("g", "d1")]) +def test_completion_counts_total_success_and_failed_calls_per_deployment_model( + make_core: MakeCore, host: ScriptedHost +) -> None: + """The sync path counts calls as ``acompletion`` does.""" + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=0) core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] - assert core.total_calls["openai/d1"] == 0 # pyright: ignore[reportAttributeAccessIssue] + host.answer("openai/d1", rate_limit()) + with pytest.raises(litellm.RateLimitError): # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members + core.completion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + counts: Final = ( # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members + core.total_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + core.success_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + core.fail_calls["openai/d1"], # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + ) + assert counts == (2, 1, 1) diff --git a/tests/unit/test_router/core/test_gaps_l2800_streaming.py b/tests/unit/test_router/core/test_gaps_l2800_streaming.py index 42c95c6f894..3168c6ef64e 100644 --- a/tests/unit/test_router/core/test_gaps_l2800_streaming.py +++ b/tests/unit/test_router/core/test_gaps_l2800_streaming.py @@ -61,33 +61,34 @@ def stream_with_usage(sync: bool = False) -> Callable[..., object]: # --- a fallback that answers without a stream -------------------------------------------------- -async def test_a_mid_stream_fallback_that_answers_without_a_stream_yields_one_none_item( +async def test_a_mid_stream_fallback_that_answers_without_a_stream_yields_its_response_as_one_chunk( make_core: MakeCore, host: ScriptedHost ) -> None: - """Oddity: the fallback's whole response is dropped and the stream yields ``None`` in its place.""" host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True)) host.answer("openai/b", completion_response("not streamed", model="b")) core: Final = build(make_core) stream: Final = await core.acompletion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] seen: list[object] = [] await drain_into(stream, seen) - assert seen[-1] is None + last: Final = seen[-1] + assert isinstance(last, ModelResponseStream) + assert (last.model, last.choices[0].delta.content) == ("b", "not streamed") # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members assert host.deployments_called() == ["a", "b"] -def test_a_sync_mid_stream_fallback_that_answers_without_a_stream_yields_the_response_fields( +def test_a_sync_mid_stream_fallback_that_answers_without_a_stream_yields_its_response_as_one_chunk( make_core: MakeCore, host: ScriptedHost ) -> None: - """Oddity: a ``ModelResponse`` is iterable (pydantic yields ``(field, value)`` pairs), so the sync - wrapper streams the fallback response's fields instead of chunks.""" + """The sync wrapper yields the same one chunk as the async one, not the response's pydantic fields.""" host.answer("openai/a", chat_stream(error=rate_limit(), leading_role_chunk=True, sync=True)) host.answer("openai/b", completion_response("not streamed", model="b")) core: Final = build(make_core) stream: Final = core.completion(model="a", messages=MESSAGES, stream=True) # pyright: ignore[reportAttributeAccessIssue] items: Final = list(stream) # pyright: ignore[reportArgumentType] - fields: Final = [item for item in items if isinstance(item, tuple)] - assert ("model", "b") in fields - assert {name for name, _ in fields} >= {"id", "choices", "usage"} + assert not any(isinstance(item, tuple) for item in items) # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members + last: Final = items[-1] # pyright: ignore[reportUnknownVariableType] # pytest and the cores' untyped members + assert isinstance(last, ModelResponseStream) + assert (last.model, last.choices[0].delta.content) == ("b", "not streamed") # pyright: ignore[reportUnknownMemberType] # pytest and the cores' untyped members assert host.deployments_called() == ["a", "b"] diff --git a/tests/unit/test_router/core/test_gaps_l5600.py b/tests/unit/test_router/core/test_gaps_l5600.py index 83ab8e943b9..2af11688ad5 100644 --- a/tests/unit/test_router/core/test_gaps_l5600.py +++ b/tests/unit/test_router/core/test_gaps_l5600.py @@ -8,6 +8,7 @@ Python does them today and called out in the test's docstring. """ import datetime +import inspect import io from collections.abc import Callable, Mapping from types import SimpleNamespace @@ -73,6 +74,10 @@ ROUTED_ENDPOINTS: Final[dict[str, dict[str, object]]] = { "acancel_batch": {"batch_id": "b1"}, } +SYNC_COUNTED: Final[dict[str, dict[str, object]]] = { + "completion": {"messages": MESSAGES}, + "embedding": {"input": ["a"]}, +} # --- batch completions (5565-5790) ---------------------------------------------------------- @@ -320,17 +325,21 @@ async def test_image_generation_for_an_unknown_group_raises_without_calling_the_ ("aembedding", "openai/d1", "openai/d1"), ("acreate_batch", "openai/d1", "g"), ("acancel_batch", "openai/d1", "g"), - ("image_generation", "", ""), + ("image_generation", "openai/d1", "openai/d1"), + ("text_completion", "openai/d1", "g"), + ("completion", "openai/d1", "openai/d1"), + ("embedding", "openai/d1", "openai/d1"), ], ) -async def test_call_counters_are_keyed_differently_per_endpoint( +async def test_call_counters_are_keyed_per_endpoint_and_the_same_for_sync_and_async( make_core: MakeCore, host: EndpointHost, endpoint: str, total_key: str, fail_key: str ): """total_calls/fail_calls are keyed by the deployment model for some endpoints and by the model group - for others (fail_calls even differs from total_calls for some); sync image_generation counts under "".""" + for others (fail_calls even differs from total_calls for some); a sync endpoint counts as its async + twin does.""" host.answer("*", rate_limit()) core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=1) - request: Final = ROUTED_ENDPOINTS.get(endpoint, {"prompt": "p"}) + request: Final = SYNC_COUNTED.get(endpoint) or ROUTED_ENDPOINTS.get(endpoint, {"prompt": "p"}) with pytest.raises(litellm.RateLimitError): await core.call(endpoint, {"model": "g", **request}) assert dict(core.total_calls) == {total_key: 2} # pyright: ignore[reportAttributeAccessIssue] # a public counter @@ -399,16 +408,22 @@ async def test_async_passthrough_rewrites_only_whole_path_segments_naming_the_gr assert host.calls[0].kwargs["endpoint"] == "/model/us.anthropic.claude/invoke/gg/us.anthropic.claude:stream" -async def test_sync_generic_call_replaces_every_substring_of_the_group_name_in_the_endpoint( - make_core: MakeCore, monkeypatch: pytest.MonkeyPatch, host: EndpointHost +@pytest.mark.parametrize("endpoint", ["aresponses", "responses"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members +async def test_generic_call_rewrites_only_whole_path_segments_naming_the_group( + make_core: MakeCore, + monkeypatch: pytest.MonkeyPatch, + host: EndpointHost, + endpoint: str, # pyright: ignore[reportUnknownMemberType, reportUnknownParameterType] # pytest and the cores' untyped members ): - """The sync path uses str.replace with the full deployment model (provider prefix kept), so a group - name inside other path segments is rewritten too, unlike the async path's segment-bounded rewrite.""" + """Sync and async both swap the group name for the provider-stripped deployment model one whole path + segment at a time, so a segment that merely contains the group name ("gg") is left alone.""" patch_direct_endpoints(monkeypatch, host) core: Final = make_core(model_list=[deployment("g", "d1")]) - core.responses(model="g", input="hi", endpoint="/gg/g") + sent: Final = getattr(core, endpoint)(model="g", input="hi", endpoint="/gg/g") # pyright: ignore[reportAny] # pytest and the cores' untyped members + if inspect.isawaitable(sent): # pyright: ignore[reportAny] # pytest and the cores' untyped members + await sent call: Final = host.calls[0] - assert (call.model, call.kwargs["endpoint"]) == ("openai/d1", "/openai/d1openai/d1/openai/d1") + assert (call.model, call.kwargs["endpoint"]) == ("openai/d1", "/gg/d1") # --- batches (8112-8362) -------------------------------------------------------------------- diff --git a/tests/unit/test_router/core/test_gaps_l8400_precall.py b/tests/unit/test_router/core/test_gaps_l8400_precall.py index 1e64257bac8..857bbab6603 100644 --- a/tests/unit/test_router/core/test_gaps_l8400_precall.py +++ b/tests/unit/test_router/core/test_gaps_l8400_precall.py @@ -100,10 +100,10 @@ async def test_a_passing_pre_call_check_sees_the_deployment_that_is_then_called( assert gate.seen == [("async", host.deployments_called("acompletion")[0])] -def test_a_rate_limit_from_a_sync_pre_call_check_is_raised_without_a_cooldown( +def test_a_rate_limit_from_a_sync_pre_call_check_cools_the_deployment_down_and_is_raised( make_core: MakeCore, host: ScriptedHost ) -> None: - """Oddity: the sync twin raises the gate's error but, unlike the async path, sets no cooldown.""" + """The sync twin cools the refused deployment down as the async path does.""" core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33) gate: Final = gated(host, rate_limit(message="gate says no")) @@ -111,7 +111,23 @@ def test_a_rate_limit_from_a_sync_pre_call_check_is_raised_without_a_cooldown( assert isinstance(error, litellm.RateLimitError) assert host.calls == [] - assert [kind for kind, _ in gate.seen] == ["sync"] + ((kind, checked),) = gate.seen + assert kind == "sync" + cooled: Final = cooled_down(core, ["a", "b"]) + assert list(cooled) == [checked] + assert cooled[checked]["cooldown_time"] == 33 + + +def test_any_other_error_from_a_sync_pre_call_check_is_raised_without_a_cooldown( + make_core: MakeCore, host: ScriptedHost +) -> None: + core: Final = make_core(model_list=GROUP, num_retries=0, cooldown_time=33) + gated(host, ValueError("gate broke")) + + error: Final = attempt_sync(core) + + assert type(error) is ValueError + assert host.calls == [] assert cooled_down(core, ["a", "b"]) == {} diff --git a/tests/unit/test_router/core/test_gaps_router_core_settings_as_given.py b/tests/unit/test_router/core/test_gaps_router_core_settings_as_given.py new file mode 100644 index 00000000000..f47af7e90eb --- /dev/null +++ b/tests/unit/test_router/core/test_gaps_router_core_settings_as_given.py @@ -0,0 +1,105 @@ +""" +Settings and configs both cores must treat as ``PythonCore`` does (oddities included): count and +timeout settings kept as given, ``default_fallbacks=[]``, malformed model lists that fail at +construction, and deprecated registry members called with a ``None`` name. +""" + +import warnings +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.router import Router +from litellm.router_core import RouterCore +from tests.unit.test_router.core.retries_health_helpers import MakeCore +from tests.unit.test_router.core.scripted_host import deployment + + +def as_router(core: RouterCore) -> Router: + router: Final = Router.__new__(Router) + router._attach(core) # pyright: ignore[reportPrivateUsage, reportAttributeAccessIssue, reportUnknownMemberType] # the facade's own attach, as from_config calls it + return router + + +def read(router: Router, name: str) -> object: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return getattr(router, name) # pyright: ignore[reportAny] # a deprecated attribute, as given + + +# --- settings kept as given ------------------------------------------------------------------------- + + +def test_retry_after_and_an_int_timeout_stay_ints(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], timeout=1)) + assert (repr(read(router, "retry_after")), repr(read(router, "timeout"))) == ("0", "1") + settings: Final = router.get_settings() + assert (repr(settings["retry_after"]), repr(settings["timeout"])) == ("0", "1") + + +def test_update_settings_casts_timeout_and_retry_after_to_int(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + router.update_settings(timeout=3.7, retry_after=2.2) + assert (repr(read(router, "timeout")), repr(read(router, "retry_after"))) == ("3", "2") + + +@pytest.mark.parametrize("name", ["num_retries", "allowed_fails", "max_fallbacks"]) +def test_a_count_setting_that_is_no_int_is_kept_as_given(make_core: MakeCore, name: str) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], **{name: "1e-05"})) + assert read(router, name) == "1e-05" + + +# --- default_fallbacks ------------------------------------------------------------------------------ + + +def test_an_empty_default_fallbacks_adds_a_star_entry_with_no_models(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")], default_fallbacks=[])) + assert read(router, "fallbacks") == [{"*": None}] + + +# --- malformed model lists -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("model_name", [None, 5]) +def test_a_model_name_that_is_no_string_fails_at_construction(make_core: MakeCore, model_name: object) -> None: + entry: Final = {"model_name": model_name, "litellm_params": {"model": "openai/gpt-4o", "api_key": "k"}} + with pytest.raises(TypeError, match="expected str instance"): + make_core(model_list=[entry]) + with pytest.raises(ValidationError, match="model_name"): + make_core(model_list=[{**entry, "model_info": {"id": "x"}}]) + + +def test_an_alias_entry_without_hidden_fails_at_construction(make_core: MakeCore) -> None: + with pytest.raises(KeyError, match="hidden"): + make_core(model_list=[deployment("gpt", "a")], model_group_alias={"al": {"model": "gpt"}}) + + +# --- deprecated registry members with a None name ----------------------------------------------------- + + +@pytest.mark.parametrize( + ("member", "arguments", "expected"), + [ + ("get_model_group", (None,), None), + ("resolve_served_model_name", (None, {}), None), + ("get_model_from_alias", (None,), None), + ("get_routable_upstream_model", (None,), None), + ("get_candidate_model_ids_for_route", (None,), frozenset()), + ], +) +def test_a_deprecated_member_called_with_no_name_returns_the_old_empty_result( + make_core: MakeCore, member: str, arguments: tuple[object, ...], expected: object +) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + assert getattr(router, member)(*arguments) == expected # pyright: ignore[reportAny] # the deprecated member + + +@pytest.mark.asyncio +async def test_remaining_usage_of_no_group_is_empty(make_core: MakeCore) -> None: + router: Final = as_router(make_core(model_list=[deployment("gpt", "a")])) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + assert await router.get_remaining_model_group_usage(None) == {} # pyright: ignore[reportArgumentType] # a None group, as the fuzzer passes it diff --git a/tests/unit/test_router/core/test_selection_characterization_groups.py b/tests/unit/test_router/core/test_selection_characterization_groups.py index 476d57f873f..19865227573 100644 --- a/tests/unit/test_router/core/test_selection_characterization_groups.py +++ b/tests/unit/test_router/core/test_selection_characterization_groups.py @@ -535,13 +535,13 @@ async def test_pass_through_selection_without_pass_through_deployments_raises_ba with pytest.raises(litellm.BadRequestError) as raised: await core.select("g", kind="pass_through") assert str(raised.value) == ( - "litellm.BadRequestError: Model g has no deployment configured with use_in_pass_through=True. " - "Please add use_in_pass_through: true in the deployment configuration" + "litellm.BadRequestError: Model g has no deployments configured with use_in_pass_through=True. " + "Please add use_in_pass_through: true to the deployment configuration" ) @pytest.mark.asyncio -async def test_async_pass_through_selection_words_the_missing_deployment_error_differently( +async def test_async_pass_through_selection_words_the_missing_deployment_error_the_same_way( make_core: MakeCore, ) -> None: core: Final = make_core(model_list=[deployment("g", "a")]) diff --git a/tests/unit/test_router/core/test_selection_characterization_prerouting.py b/tests/unit/test_router/core/test_selection_characterization_prerouting.py index cbe5ebdc2d2..3d3da1a4359 100644 --- a/tests/unit/test_router/core/test_selection_characterization_prerouting.py +++ b/tests/unit/test_router/core/test_selection_characterization_prerouting.py @@ -195,15 +195,23 @@ async def test_selecting_a_marker_by_its_id_refuses_it_naming_its_litellm_model( assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="auto_router/complexity_router") -@pytest.mark.parametrize("entry", ["get_available_deployment", "completion"]) -def test_the_sync_path_never_runs_pre_routing_and_refuses_the_marker( - make_core: MakeCore, host: ScriptedHost, entry: str +@pytest.mark.parametrize("model", ["smart", "al"]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members +def test_sync_get_available_deployment_runs_pre_routing_like_the_async_pick(make_core: MakeCore, model: str) -> None: + core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"}) + request: Final[dict[str, object]] = {} + chosen: Final = core.get_available_deployment(model, messages=MESSAGES, request_kwargs=request) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType] + assert chosen["model_info"]["id"] == "s" + assert _metadata(request)["routing_decision"]["router_model_name"] == "smart" # pyright: ignore[reportIndexIssue] + + +@pytest.mark.parametrize(("model", "messages", "expected"), [("smart", MESSAGES, "s"), ("al", HARD_MESSAGES, "b")]) # pyright: ignore[reportUnknownMemberType, reportUntypedFunctionDecorator] # pytest and the cores' untyped members +def test_sync_completion_routes_through_a_complexity_router_like_acompletion( + make_core: MakeCore, host: ScriptedHost, model: str, messages: list[dict[str, str]], expected: str ) -> None: - core: Final = make_core(model_list=_tiered()) - with pytest.raises(litellm.BadRequestError) as raised: - _sync_entry(core, entry, "smart") - assert str(raised.value) == ONLY_MARKERS_MESSAGE.format(model="smart") - assert host.calls == [] + core: Final = make_core(model_list=_tiered(), model_group_alias={"al": "smart"}) + core.completion(model=model, messages=messages) # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] + assert host.deployments_called() == [expected] + assert host.calls[-1].model_group == model # --- choosing among tag-scoped markers -----------------------------------------------------------------