diff --git a/litellm/router.py b/litellm/router.py index 33f53d9b852..16eadfe159f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -235,6 +235,7 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.router_utils.common_utils import ( MID_STREAM_CONTINUATION_KWARG, + apick_preferring_web_search, as_retry_skipped_deployment_ids, deployments_the_caller_may_use, filter_team_based_models, @@ -245,6 +246,7 @@ from litellm.router_utils.common_utils import ( format_no_fallback_group_message, get_request_team_id, is_proxy_admin_request, + pick_preferring_web_search, prefer_web_search_deployments, provider_for_generic_call, request_user_api_key_auth, @@ -15246,13 +15248,10 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = prefer_web_search_deployments( - self._without_declared_text_only_deployments( - cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts - model, - messages, - cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict - ), + batch_candidates: Final = self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( @@ -15535,26 +15534,31 @@ class Router: return healthy_deployments[0] start_time: Final = time.time() + routing_kwargs: Final = cast(Mapping[str, object], request_kwargs) # cast-ok: str-keyed request kwargs if strategy == "simple-shuffle": shuffled: Final = simple_shuffle( resolve_model_alias=self.get_model_from_alias, - healthy_deployments=healthy_deployments, + healthy_deployments=prefer_web_search_deployments(healthy_deployments, routing_kwargs), model=model, - request_kwargs=request_kwargs, + request_kwargs=routing_kwargs, ) phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return shuffled with PrefetchedUsage.scoped( routing_read_batch.prefetched_usage if routing_read_batch is not None else None ): - deployment: Final = await self._select_deployment_async( - strategy=strategy, - selector=strategy_selector, - model=model, - healthy_deployments=healthy_deployments, - messages=messages, - input=input, - request_kwargs=request_kwargs, + deployment: Final = await apick_preferring_web_search( + lambda candidates: self._select_deployment_async( + strategy=strategy, + selector=strategy_selector, + model=model, + healthy_deployments=candidates, + messages=messages, + input=input, + request_kwargs=request_kwargs, + ), + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + routing_kwargs, ) if deployment is None: exception: Final = await async_raise_no_deployment_exception( @@ -16481,13 +16485,10 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = prefer_web_search_deployments( - self._without_declared_text_only_deployments( - cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts - model, - messages, - cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict - ), + batch_candidates: Final = self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( @@ -16575,20 +16576,24 @@ class Router: ############## Check 'weight' param set for weighted pick ################# shuffled: Final = simple_shuffle( resolve_model_alias=self.get_model_from_alias, - healthy_deployments=healthy_deployments, + healthy_deployments=prefer_web_search_deployments(healthy_deployments, routing_kwargs), model=model, - request_kwargs=request_kwargs, + request_kwargs=routing_kwargs, ) phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return shuffled - deployment: Final = self._select_deployment_sync( - strategy=strategy, - selector=strategy_selector, - model=model, - healthy_deployments=healthy_deployments, - messages=messages, - input=input, - request_kwargs=request_kwargs, + deployment: Final = pick_preferring_web_search( + lambda candidates: self._select_deployment_sync( + strategy=strategy, + selector=strategy_selector, + model=model, + healthy_deployments=candidates, + messages=messages, + input=input, + request_kwargs=request_kwargs, + ), + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + routing_kwargs, ) if deployment is None: diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index dd82f4e7862..5b3f923d936 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -1,6 +1,6 @@ import hashlib import json -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar @@ -33,6 +33,8 @@ from litellm.types.utils import ( ) _V = TypeVar("_V") +_Picked: Final = TypeVar("_Picked") +_DeploymentT: Final = TypeVar("_DeploymentT", bound=dict[str, object]) MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation" ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG}) @@ -53,6 +55,9 @@ _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = tuple[Mapping[str, object], ...] ) _DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...]) +_RETRY_BREADCRUMBS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( + tuple[Mapping[str, object], ...] +) _FALLBACK_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) @@ -493,10 +498,31 @@ def _deployment_id(deployment: Mapping[str, object]) -> object: return None if model_info is None else model_info.get("id") +def _retry_breadcrumbs(metadata: object) -> tuple[Mapping[str, object], ...]: + fields: Final = as_str_mapping(metadata) + breadcrumbs: Final = None if fields is None else fields.get("previous_models") + if breadcrumbs is None: + return () + try: + return _RETRY_BREADCRUMBS_ADAPTER.validate_python(breadcrumbs) + except ValidationError: + return () + + +def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) -> tuple[str, ...]: + breadcrumbs: Final = ( + *_retry_breadcrumbs(request_kwargs.get("metadata")), + *_retry_breadcrumbs(request_kwargs.get("litellm_metadata")), + ) + return tuple( + deployment_id for breadcrumb in breadcrumbs if isinstance(deployment_id := breadcrumb.get("deployment_id"), str) + ) + + def prefer_web_search_deployments( - deployments: list[dict[str, object]], # mutable-ok: router deployment-list contract + deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract request_kwargs: Mapping[str, object] | None, -) -> list[dict[str, object]]: # mutable-ok: returns a mutable deployment list +) -> list[_DeploymentT]: # mutable-ok: returns a mutable deployment list if request_kwargs is None or not _is_web_search_request( request_kwargs.get("tools"), request_kwargs.get("web_search_options") ): @@ -505,10 +531,11 @@ def prefer_web_search_deployments( ( *as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")), *as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")), + *_deployment_ids_retried_in_request(request_kwargs), ) ) untried: Final = (d for d in deployments if _deployment_id(d) not in tried) - orders_with_an_untried_capable_deployment: Final = frozenset( + orders_with_an_untried_capable_deployment: Final = tuple( litellm.utils.get_deployment_order(d) for d in untried if _deployment_supports_web_search(d) ) if not orders_with_an_untried_capable_deployment: @@ -522,6 +549,36 @@ def prefer_web_search_deployments( ] +def pick_preferring_web_search( + pick: Callable[[list[_DeploymentT]], _Picked | None], # mutable-ok: router deployment-list contract + deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract + request_kwargs: Mapping[str, object] | None, +) -> _Picked | None: + preferred: Final = prefer_web_search_deployments(deployments, request_kwargs) + if len(preferred) == len(deployments): + return pick(deployments) + try: + from_preferred: Final = pick(preferred) + except (litellm.RateLimitError, ValueError): + return pick(deployments) + return from_preferred if from_preferred is not None else pick(deployments) + + +async def apick_preferring_web_search( + pick: Callable[[list[_DeploymentT]], Awaitable[_Picked | None]], # mutable-ok: router deployment-list contract + deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract + request_kwargs: Mapping[str, object] | None, +) -> _Picked | None: + preferred: Final = prefer_web_search_deployments(deployments, request_kwargs) + if len(preferred) == len(deployments): + return await pick(deployments) + try: + from_preferred: Final = await pick(preferred) + except (litellm.RateLimitError, ValueError): + return await pick(deployments) + return from_preferred if from_preferred is not None else await pick(deployments) + + # Credential params that only one provider family reads, paired with the providers # that read them. A deployment carrying them while resolving elsewhere is almost # always a missing route prefix: `model: claude-sonnet-5` with `aws_region_name` diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 273973c4847..0049793a354 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -1,3 +1,4 @@ +import asyncio import logging from itertools import chain from typing import Dict, Final, List, Optional, Union @@ -15,6 +16,7 @@ from litellm.router_utils.common_utils import ( add_model_file_id_mappings, filter_team_based_models, filter_web_search_deployments, + pick_preferring_web_search, prefer_web_search_deployments, format_content_policy_no_fallback_note, format_fallback_outcome_message, @@ -523,8 +525,11 @@ def test_sync_pick_keeps_a_group_without_web_search_for_openai_web_search(routin @pytest.mark.asyncio -async def test_legacy_usage_based_async_pick_skips_deployments_without_web_search() -> None: - router: Final = _router_with_a_deployment_that_disables_web_search("usage-based-routing") +@pytest.mark.parametrize( + "routing_strategy", ["usage-based-routing", "usage-based-routing-v2", "least-busy", "latency-based-routing"] +) +async def test_async_strategy_pick_skips_deployments_without_web_search(routing_strategy: str) -> None: + router: Final = _router_with_a_deployment_that_disables_web_search(routing_strategy) picks: Final = { ( await router.async_get_available_deployment( @@ -706,6 +711,151 @@ async def test_web_search_request_prefers_the_capable_deployment_inside_its_orde assert picks == {"search"} +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_web_search_preference_gives_way_once_the_capable_deployment_failed_this_request( + metadata_field: str, +) -> None: + router: Final = _router_with_a_mixed_web_search_group(False) + picks: Final = { + ( + await router.async_get_available_deployment( + model="mixed", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={ + **_ANTHROPIC_WEB_SEARCH_REQUEST, + metadata_field: {"previous_models": ({"deployment_id": "search", "exception_type": "Timeout"},)}, + }, + ) + )["model_info"]["id"] + for _ in range(40) + } + assert picks == {"search", "no-search"} + + +@pytest.mark.asyncio +@_WEB_SEARCH_REQUESTS +async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches_the_other_one( + web_search_request: dict[str, object], +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-x", + "api_key": "fake-key", + "mock_response": Exception("connection error"), + "weight": 1, + }, + "model_info": {"id": "search"}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "weight": 10**6}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + num_retries=1, + disable_cooldowns=True, + ) + response: Final = await router.acompletion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request + ) + assert response._hidden_params["model_id"] == "no-search" + + +def _rate_limited_web_search_group(routing_strategy: str) -> Router: + return Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi", "rpm": 1}, + "model_info": {"id": "search"}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "rpm": 1000}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + routing_strategy=routing_strategy, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("routing_strategy", ["usage-based-routing-v2", "cost-based-routing"]) +async def test_web_search_request_goes_to_the_other_deployment_once_the_capable_one_is_at_its_rpm_limit( + routing_strategy: str, +) -> None: + router: Final = _rate_limited_web_search_group(routing_strategy) + served_by: Final = [] # mutable-ok: collects one id per sequential request + for _ in range(3): + response = await router.acompletion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST + ) + served_by.append(response._hidden_params["model_id"]) + await asyncio.sleep(0.2) + assert served_by == ["search", "no-search", "no-search"] + + +def test_sync_web_search_request_goes_to_the_other_deployment_once_the_capable_one_is_at_its_rpm_limit() -> None: + router: Final = _rate_limited_web_search_group("usage-based-routing-v2") + served_by: Final = tuple( + router.completion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST + )._hidden_params["model_id"] + for _ in range(3) + ) + assert served_by == ("search", "no-search", "no-search") + + +def test_sync_pick_falls_back_to_every_deployment_when_the_strategy_finds_none_that_can_search() -> None: + deployments: Final = [ + {"model_info": {"id": "search"}}, + {"model_info": {"id": "no-search", "supports_web_search": False}}, + ] + picked: Final = pick_preferring_web_search( + lambda candidates: next((d for d in candidates if d["model_info"]["id"] == "no-search"), None), + deployments, + _ANTHROPIC_WEB_SEARCH_REQUEST, + ) + assert picked == deployments[1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("search_order", "no_search_order"), + [([1], [1]), ({"tier": 1}, None)], + ids=["list-order", "dict-order"], +) +async def test_web_search_request_routes_on_a_group_whose_order_is_not_a_number( + search_order: object, no_search_order: object +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key"}, + "model_info": {"id": "search", "order": search_order}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key"}, + "model_info": { + "id": "no-search", + "supports_web_search": False, + **({} if no_search_order is None else {"order": no_search_order}), + }, + }, + ], + ) + deployment: Final = await router.async_get_available_deployment( + model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST) + ) + assert deployment["model_info"]["id"] == "search" + + @pytest.mark.asyncio async def test_tag_routing_to_a_deployment_without_web_search_keeps_it() -> None: deployment: Final = await _router_with_a_mixed_web_search_group(True).async_get_available_deployment(