mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): prefer web search deployments at the strategy pick and fall back when none has capacity
This commit is contained in:
parent
5b2ccc8e58
commit
a179118b3e
3 changed files with 252 additions and 40 deletions
|
|
@ -235,6 +235,7 @@ from litellm.router_utils.clientside_credential_handler import (
|
|||
)
|
||||
from litellm.router_utils.common_utils import (
|
||||
MID_STREAM_CONTINUATION_KWARG,
|
||||
apick_preferring_web_search,
|
||||
as_retry_skipped_deployment_ids,
|
||||
deployments_the_caller_may_use,
|
||||
filter_team_based_models,
|
||||
|
|
@ -245,6 +246,7 @@ from litellm.router_utils.common_utils import (
|
|||
format_no_fallback_group_message,
|
||||
get_request_team_id,
|
||||
is_proxy_admin_request,
|
||||
pick_preferring_web_search,
|
||||
prefer_web_search_deployments,
|
||||
provider_for_generic_call,
|
||||
request_user_api_key_auth,
|
||||
|
|
@ -15246,13 +15248,10 @@ class Router:
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = prefer_web_search_deployments(
|
||||
self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
),
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
|
|
@ -15535,26 +15534,31 @@ class Router:
|
|||
return healthy_deployments[0]
|
||||
|
||||
start_time: Final = time.time()
|
||||
routing_kwargs: Final = cast(Mapping[str, object], request_kwargs) # cast-ok: str-keyed request kwargs
|
||||
if strategy == "simple-shuffle":
|
||||
shuffled: Final = simple_shuffle(
|
||||
resolve_model_alias=self.get_model_from_alias,
|
||||
healthy_deployments=healthy_deployments,
|
||||
healthy_deployments=prefer_web_search_deployments(healthy_deployments, routing_kwargs),
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
request_kwargs=routing_kwargs,
|
||||
)
|
||||
phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes)
|
||||
return shuffled
|
||||
with PrefetchedUsage.scoped(
|
||||
routing_read_batch.prefetched_usage if routing_read_batch is not None else None
|
||||
):
|
||||
deployment: Final = await self._select_deployment_async(
|
||||
strategy=strategy,
|
||||
selector=strategy_selector,
|
||||
model=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
deployment: Final = await apick_preferring_web_search(
|
||||
lambda candidates: self._select_deployment_async(
|
||||
strategy=strategy,
|
||||
selector=strategy_selector,
|
||||
model=model,
|
||||
healthy_deployments=candidates,
|
||||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
),
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
routing_kwargs,
|
||||
)
|
||||
if deployment is None:
|
||||
exception: Final = await async_raise_no_deployment_exception(
|
||||
|
|
@ -16481,13 +16485,10 @@ class Router:
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = prefer_web_search_deployments(
|
||||
self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
),
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
|
|
@ -16575,20 +16576,24 @@ class Router:
|
|||
############## Check 'weight' param set for weighted pick #################
|
||||
shuffled: Final = simple_shuffle(
|
||||
resolve_model_alias=self.get_model_from_alias,
|
||||
healthy_deployments=healthy_deployments,
|
||||
healthy_deployments=prefer_web_search_deployments(healthy_deployments, routing_kwargs),
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
request_kwargs=routing_kwargs,
|
||||
)
|
||||
phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes)
|
||||
return shuffled
|
||||
deployment: Final = self._select_deployment_sync(
|
||||
strategy=strategy,
|
||||
selector=strategy_selector,
|
||||
model=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
deployment: Final = pick_preferring_web_search(
|
||||
lambda candidates: self._select_deployment_sync(
|
||||
strategy=strategy,
|
||||
selector=strategy_selector,
|
||||
model=model,
|
||||
healthy_deployments=candidates,
|
||||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
),
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
routing_kwargs,
|
||||
)
|
||||
|
||||
if deployment is None:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeVar
|
||||
|
||||
|
|
@ -33,6 +33,8 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
_V = TypeVar("_V")
|
||||
_Picked: Final = TypeVar("_Picked")
|
||||
_DeploymentT: Final = TypeVar("_DeploymentT", bound=dict[str, object])
|
||||
|
||||
MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation"
|
||||
ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG})
|
||||
|
|
@ -53,6 +55,9 @@ _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] =
|
|||
tuple[Mapping[str, object], ...]
|
||||
)
|
||||
_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
|
||||
_RETRY_BREADCRUMBS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
|
||||
tuple[Mapping[str, object], ...]
|
||||
)
|
||||
_FALLBACK_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
|
||||
tuple[Mapping[str, object], ...]
|
||||
)
|
||||
|
|
@ -493,10 +498,31 @@ def _deployment_id(deployment: Mapping[str, object]) -> object:
|
|||
return None if model_info is None else model_info.get("id")
|
||||
|
||||
|
||||
def _retry_breadcrumbs(metadata: object) -> tuple[Mapping[str, object], ...]:
|
||||
fields: Final = as_str_mapping(metadata)
|
||||
breadcrumbs: Final = None if fields is None else fields.get("previous_models")
|
||||
if breadcrumbs is None:
|
||||
return ()
|
||||
try:
|
||||
return _RETRY_BREADCRUMBS_ADAPTER.validate_python(breadcrumbs)
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) -> tuple[str, ...]:
|
||||
breadcrumbs: Final = (
|
||||
*_retry_breadcrumbs(request_kwargs.get("metadata")),
|
||||
*_retry_breadcrumbs(request_kwargs.get("litellm_metadata")),
|
||||
)
|
||||
return tuple(
|
||||
deployment_id for breadcrumb in breadcrumbs if isinstance(deployment_id := breadcrumb.get("deployment_id"), str)
|
||||
)
|
||||
|
||||
|
||||
def prefer_web_search_deployments(
|
||||
deployments: list[dict[str, object]], # mutable-ok: router deployment-list contract
|
||||
deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract
|
||||
request_kwargs: Mapping[str, object] | None,
|
||||
) -> list[dict[str, object]]: # mutable-ok: returns a mutable deployment list
|
||||
) -> list[_DeploymentT]: # mutable-ok: returns a mutable deployment list
|
||||
if request_kwargs is None or not _is_web_search_request(
|
||||
request_kwargs.get("tools"), request_kwargs.get("web_search_options")
|
||||
):
|
||||
|
|
@ -505,10 +531,11 @@ def prefer_web_search_deployments(
|
|||
(
|
||||
*as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")),
|
||||
*as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")),
|
||||
*_deployment_ids_retried_in_request(request_kwargs),
|
||||
)
|
||||
)
|
||||
untried: Final = (d for d in deployments if _deployment_id(d) not in tried)
|
||||
orders_with_an_untried_capable_deployment: Final = frozenset(
|
||||
orders_with_an_untried_capable_deployment: Final = tuple(
|
||||
litellm.utils.get_deployment_order(d) for d in untried if _deployment_supports_web_search(d)
|
||||
)
|
||||
if not orders_with_an_untried_capable_deployment:
|
||||
|
|
@ -522,6 +549,36 @@ def prefer_web_search_deployments(
|
|||
]
|
||||
|
||||
|
||||
def pick_preferring_web_search(
|
||||
pick: Callable[[list[_DeploymentT]], _Picked | None], # mutable-ok: router deployment-list contract
|
||||
deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract
|
||||
request_kwargs: Mapping[str, object] | None,
|
||||
) -> _Picked | None:
|
||||
preferred: Final = prefer_web_search_deployments(deployments, request_kwargs)
|
||||
if len(preferred) == len(deployments):
|
||||
return pick(deployments)
|
||||
try:
|
||||
from_preferred: Final = pick(preferred)
|
||||
except (litellm.RateLimitError, ValueError):
|
||||
return pick(deployments)
|
||||
return from_preferred if from_preferred is not None else pick(deployments)
|
||||
|
||||
|
||||
async def apick_preferring_web_search(
|
||||
pick: Callable[[list[_DeploymentT]], Awaitable[_Picked | None]], # mutable-ok: router deployment-list contract
|
||||
deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract
|
||||
request_kwargs: Mapping[str, object] | None,
|
||||
) -> _Picked | None:
|
||||
preferred: Final = prefer_web_search_deployments(deployments, request_kwargs)
|
||||
if len(preferred) == len(deployments):
|
||||
return await pick(deployments)
|
||||
try:
|
||||
from_preferred: Final = await pick(preferred)
|
||||
except (litellm.RateLimitError, ValueError):
|
||||
return await pick(deployments)
|
||||
return from_preferred if from_preferred is not None else await pick(deployments)
|
||||
|
||||
|
||||
# Credential params that only one provider family reads, paired with the providers
|
||||
# that read them. A deployment carrying them while resolving elsewhere is almost
|
||||
# always a missing route prefix: `model: claude-sonnet-5` with `aws_region_name`
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import asyncio
|
||||
import logging
|
||||
from itertools import chain
|
||||
from typing import Dict, Final, List, Optional, Union
|
||||
|
|
@ -15,6 +16,7 @@ from litellm.router_utils.common_utils import (
|
|||
add_model_file_id_mappings,
|
||||
filter_team_based_models,
|
||||
filter_web_search_deployments,
|
||||
pick_preferring_web_search,
|
||||
prefer_web_search_deployments,
|
||||
format_content_policy_no_fallback_note,
|
||||
format_fallback_outcome_message,
|
||||
|
|
@ -523,8 +525,11 @@ def test_sync_pick_keeps_a_group_without_web_search_for_openai_web_search(routin
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_usage_based_async_pick_skips_deployments_without_web_search() -> None:
|
||||
router: Final = _router_with_a_deployment_that_disables_web_search("usage-based-routing")
|
||||
@pytest.mark.parametrize(
|
||||
"routing_strategy", ["usage-based-routing", "usage-based-routing-v2", "least-busy", "latency-based-routing"]
|
||||
)
|
||||
async def test_async_strategy_pick_skips_deployments_without_web_search(routing_strategy: str) -> None:
|
||||
router: Final = _router_with_a_deployment_that_disables_web_search(routing_strategy)
|
||||
picks: Final = {
|
||||
(
|
||||
await router.async_get_available_deployment(
|
||||
|
|
@ -706,6 +711,151 @@ async def test_web_search_request_prefers_the_capable_deployment_inside_its_orde
|
|||
assert picks == {"search"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"])
|
||||
async def test_web_search_preference_gives_way_once_the_capable_deployment_failed_this_request(
|
||||
metadata_field: str,
|
||||
) -> None:
|
||||
router: Final = _router_with_a_mixed_web_search_group(False)
|
||||
picks: Final = {
|
||||
(
|
||||
await router.async_get_available_deployment(
|
||||
model="mixed",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={
|
||||
**_ANTHROPIC_WEB_SEARCH_REQUEST,
|
||||
metadata_field: {"previous_models": ({"deployment_id": "search", "exception_type": "Timeout"},)},
|
||||
},
|
||||
)
|
||||
)["model_info"]["id"]
|
||||
for _ in range(40)
|
||||
}
|
||||
assert picks == {"search", "no-search"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_WEB_SEARCH_REQUESTS
|
||||
async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches_the_other_one(
|
||||
web_search_request: dict[str, object],
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-x",
|
||||
"api_key": "fake-key",
|
||||
"mock_response": Exception("connection error"),
|
||||
"weight": 1,
|
||||
},
|
||||
"model_info": {"id": "search"},
|
||||
},
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "weight": 10**6},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
],
|
||||
num_retries=1,
|
||||
disable_cooldowns=True,
|
||||
)
|
||||
response: Final = await router.acompletion(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request
|
||||
)
|
||||
assert response._hidden_params["model_id"] == "no-search"
|
||||
|
||||
|
||||
def _rate_limited_web_search_group(routing_strategy: str) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi", "rpm": 1},
|
||||
"model_info": {"id": "search"},
|
||||
},
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "rpm": 1000},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
],
|
||||
routing_strategy=routing_strategy,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("routing_strategy", ["usage-based-routing-v2", "cost-based-routing"])
|
||||
async def test_web_search_request_goes_to_the_other_deployment_once_the_capable_one_is_at_its_rpm_limit(
|
||||
routing_strategy: str,
|
||||
) -> None:
|
||||
router: Final = _rate_limited_web_search_group(routing_strategy)
|
||||
served_by: Final = [] # mutable-ok: collects one id per sequential request
|
||||
for _ in range(3):
|
||||
response = await router.acompletion(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST
|
||||
)
|
||||
served_by.append(response._hidden_params["model_id"])
|
||||
await asyncio.sleep(0.2)
|
||||
assert served_by == ["search", "no-search", "no-search"]
|
||||
|
||||
|
||||
def test_sync_web_search_request_goes_to_the_other_deployment_once_the_capable_one_is_at_its_rpm_limit() -> None:
|
||||
router: Final = _rate_limited_web_search_group("usage-based-routing-v2")
|
||||
served_by: Final = tuple(
|
||||
router.completion(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST
|
||||
)._hidden_params["model_id"]
|
||||
for _ in range(3)
|
||||
)
|
||||
assert served_by == ("search", "no-search", "no-search")
|
||||
|
||||
|
||||
def test_sync_pick_falls_back_to_every_deployment_when_the_strategy_finds_none_that_can_search() -> None:
|
||||
deployments: Final = [
|
||||
{"model_info": {"id": "search"}},
|
||||
{"model_info": {"id": "no-search", "supports_web_search": False}},
|
||||
]
|
||||
picked: Final = pick_preferring_web_search(
|
||||
lambda candidates: next((d for d in candidates if d["model_info"]["id"] == "no-search"), None),
|
||||
deployments,
|
||||
_ANTHROPIC_WEB_SEARCH_REQUEST,
|
||||
)
|
||||
assert picked == deployments[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("search_order", "no_search_order"),
|
||||
[([1], [1]), ({"tier": 1}, None)],
|
||||
ids=["list-order", "dict-order"],
|
||||
)
|
||||
async def test_web_search_request_routes_on_a_group_whose_order_is_not_a_number(
|
||||
search_order: object, no_search_order: object
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key"},
|
||||
"model_info": {"id": "search", "order": search_order},
|
||||
},
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key"},
|
||||
"model_info": {
|
||||
"id": "no-search",
|
||||
"supports_web_search": False,
|
||||
**({} if no_search_order is None else {"order": no_search_order}),
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
deployment: Final = await router.async_get_available_deployment(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST)
|
||||
)
|
||||
assert deployment["model_info"]["id"] == "search"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_routing_to_a_deployment_without_web_search_keeps_it() -> None:
|
||||
deployment: Final = await _router_with_a_mixed_web_search_group(True).async_get_available_deployment(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue