diff --git a/litellm/router.py b/litellm/router.py index 3a423b79e49..c1aa18333a9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -118,6 +118,7 @@ from litellm.litellm_core_utils.batch_utils import ( ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, coerce_token_limit, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -254,6 +255,7 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.router_utils.common_utils import ( MID_STREAM_CONTINUATION_KWARG, + apick_preferring_web_search, as_retry_skipped_deployment_ids, deployments_the_caller_may_use, filter_team_based_models, @@ -264,6 +266,8 @@ from litellm.router_utils.common_utils import ( format_no_fallback_group_message, get_request_team_id, is_proxy_admin_request, + pick_preferring_web_search, + prefer_web_search_deployments, provider_for_generic_call, request_user_api_key_auth, resolve_model_group_alias, @@ -10108,14 +10112,15 @@ class Router(DeprecatedRouterMembers): _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs model_group: Final = kwargs.get("model") - model_info: Final = request_metadata.get("model_info") - deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None + model_info: Final = as_str_mapping(request_metadata.get("model_info")) or {} + deployment_id: Final = model_info.get("id") + configured_deployment_id: Final = model_info.get("original_model_id") base_model: Final = get_base_model_from_litellm_call_metadata( metadata=request_metadata ) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None) deployment_model: Final = request_metadata.get("deployment") attempted_retries: Final = request_metadata.get("attempted_retries") - attempt_record: Final[RetryAttemptRecord] = { + attempt_fields: Final[RetryAttemptRecord] = { "model_group": model_group if isinstance(model_group, str) else None, "deployment_id": deployment_id if isinstance(deployment_id, str) else None, "model": deployment_model if isinstance(deployment_model, str) else None, @@ -10124,6 +10129,11 @@ class Router(DeprecatedRouterMembers): "exception_string": str(e), "attempted_retries": attempted_retries if type(attempted_retries) is int else None, } + attempt_record: Final[RetryAttemptRecord] = ( + {**attempt_fields, "configured_deployment_id": configured_deployment_id} + if isinstance(configured_deployment_id, str) + else attempt_fields + ) earlier_breadcrumbs: Final = request_metadata.get("previous_models") kept_breadcrumbs: Final[tuple[object, ...]] = ( tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :] @@ -15792,26 +15802,31 @@ class Router(DeprecatedRouterMembers): return healthy_deployments[0] start_time: Final = time.time() + routing_kwargs: Final = cast(Mapping[str, object], request_kwargs) # cast-ok: str-keyed request kwargs if strategy == "simple-shuffle": shuffled: Final = simple_shuffle( resolve_model_alias=self._core_get_model_from_alias, - healthy_deployments=healthy_deployments, + healthy_deployments=prefer_web_search_deployments(healthy_deployments, routing_kwargs), model=model, - request_kwargs=request_kwargs, + request_kwargs=routing_kwargs, ) phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return shuffled with PrefetchedUsage.scoped( routing_read_batch.prefetched_usage if routing_read_batch is not None else None ): - deployment: Final = await self._select_deployment_async( - strategy=strategy, - selector=strategy_selector, - model=model, - healthy_deployments=healthy_deployments, - messages=messages, - input=input, - request_kwargs=request_kwargs, + deployment: Final = await apick_preferring_web_search( + lambda candidates: self._select_deployment_async( + strategy=strategy, + selector=strategy_selector, + model=model, + healthy_deployments=candidates, + messages=messages, + input=input, + request_kwargs=request_kwargs, + ), + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + routing_kwargs, ) if deployment is None: exception: Final = await async_raise_no_deployment_exception( @@ -16842,20 +16857,24 @@ class Router(DeprecatedRouterMembers): ############## Check 'weight' param set for weighted pick ################# shuffled: Final = simple_shuffle( resolve_model_alias=self._core_get_model_from_alias, - healthy_deployments=healthy_deployments, + healthy_deployments=prefer_web_search_deployments(healthy_deployments, routing_kwargs), model=model, - request_kwargs=request_kwargs, + request_kwargs=routing_kwargs, ) phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return shuffled - deployment: Final = self._select_deployment_sync( - strategy=strategy, - selector=strategy_selector, - model=model, - healthy_deployments=healthy_deployments, - messages=messages, - input=input, - request_kwargs=request_kwargs, + deployment: Final = pick_preferring_web_search( + lambda candidates: self._select_deployment_sync( + strategy=strategy, + selector=strategy_selector, + model=model, + healthy_deployments=candidates, + messages=messages, + input=input, + request_kwargs=request_kwargs, + ), + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + routing_kwargs, ) if deployment is None: diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 2451422b5e6..c2f70503cc2 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -1,6 +1,6 @@ import hashlib import json -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar @@ -13,9 +13,11 @@ import litellm from litellm._logging import verbose_logger, verbose_router_logger from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure +from litellm.rust_bridge.public_call import optional_sequence from litellm.types.router import ( CredentialLiteLLMParams, Deployment, @@ -31,14 +33,31 @@ from litellm.types.utils import ( ) _V = TypeVar("_V") +_Picked: Final = TypeVar("_Picked") +_DeploymentT: Final = TypeVar("_DeploymentT", bound=dict[str, object]) MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation" ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG}) _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) +_GEMINI_SEARCH_TOOL_KEYS: Final = frozenset( + { + "googleSearch", + "google_search", + "googleSearchRetrieval", + "google_search_retrieval", + "enterpriseWebSearch", + "enterprise_web_search", + } +) +_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "google_search"}) +_OPENAI_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"}) _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) _DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...]) +_RETRY_BREADCRUMBS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( + tuple[Mapping[str, object], ...] +) _FALLBACK_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) @@ -407,9 +426,6 @@ def deployments_the_caller_may_use( return () -_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"}) - - def _deployment_supports_web_search(deployment: dict) -> bool: """ Check if a deployment supports web search. @@ -430,6 +446,22 @@ def _deployment_supports_web_search(deployment: dict) -> bool: return True +def _is_web_search_tool(tool: object) -> bool: + tool_fields: Final = as_str_mapping(tool) + if tool_fields is None: + return False + tool_type: Final = tool_fields.get("type") + if isinstance(tool_type, str) and (tool_type in _WEB_SEARCH_TOOL_TYPES or tool_type.startswith("web_search_")): + return True + return not _GEMINI_SEARCH_TOOL_KEYS.isdisjoint(tool_fields) and any( + tool_fields[key] is not None for key in _GEMINI_SEARCH_TOOL_KEYS.intersection(tool_fields) + ) + + +def _is_web_search_request(tools: object, web_search_options: object) -> bool: + return web_search_options is not None or any(_is_web_search_tool(tool) for tool in optional_sequence(tools) or ()) + + def filter_web_search_deployments( healthy_deployments: list[dict] | dict, request_kwargs: dict | None = None, @@ -448,7 +480,7 @@ def filter_web_search_deployments( is_web_search_request: Final = isinstance(tools, (list, tuple)) and any( isinstance(tool, dict) and isinstance(tool_type := tool.get("type"), str) - and tool_type in _WEB_SEARCH_TOOL_TYPES + and tool_type in _OPENAI_WEB_SEARCH_TOOL_TYPES for tool in tools ) if not is_web_search_request: @@ -461,6 +493,105 @@ def filter_web_search_deployments( return final_deployments +def _deployment_id(deployment: Mapping[str, object]) -> object: + model_info: Final = as_str_mapping(deployment.get("model_info")) + return None if model_info is None else model_info.get("id") + + +def _retry_breadcrumbs(metadata: object) -> tuple[Mapping[str, object], ...]: + fields: Final = as_str_mapping(metadata) + breadcrumbs: Final = None if fields is None else fields.get("previous_models") + if breadcrumbs is None: + return () + try: + return _RETRY_BREADCRUMBS_ADAPTER.validate_python(breadcrumbs) + except ValidationError: + return () + + +def _retry_history_was_trimmed(metadata: object) -> bool: + fields: Final = as_str_mapping(metadata) + retry_count: Final = None if fields is None else fields.get("request_retry_count") + return type(retry_count) is int and retry_count > len(_retry_breadcrumbs(metadata)) + + +def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) -> tuple[str, ...]: + breadcrumbs: Final = ( + *_retry_breadcrumbs(request_kwargs.get("metadata")), + *_retry_breadcrumbs(request_kwargs.get("litellm_metadata")), + ) + return tuple( + deployment_id + for breadcrumb in breadcrumbs + if isinstance(deployment_id := breadcrumb.get("configured_deployment_id", breadcrumb.get("deployment_id")), str) + ) + + +def prefer_web_search_deployments( + deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract + request_kwargs: Mapping[str, object] | None, +) -> list[_DeploymentT]: # mutable-ok: returns a mutable deployment list + if request_kwargs is None or not _is_web_search_request( + request_kwargs.get("tools"), request_kwargs.get("web_search_options") + ): + return deployments + if _retry_history_was_trimmed(request_kwargs.get("metadata")) or _retry_history_was_trimmed( + request_kwargs.get("litellm_metadata") + ): + verbose_router_logger.debug("This request's retry history was trimmed, so every deployment stays") + return deployments + tried: Final = frozenset( + ( + *as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")), + *as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")), + *_deployment_ids_retried_in_request(request_kwargs), + ) + ) + untried: Final = (d for d in deployments if _deployment_id(d) not in tried) + orders_with_an_untried_capable_deployment: Final = tuple( + litellm.utils.get_deployment_order(d) for d in untried if _deployment_supports_web_search(d) + ) + if not orders_with_an_untried_capable_deployment: + verbose_router_logger.debug("No deployment left for this web search request supports web search, so all stay") + return deployments + return [ + d + for d in deployments + if _deployment_supports_web_search(d) + or litellm.utils.get_deployment_order(d) not in orders_with_an_untried_capable_deployment + ] + + +def pick_preferring_web_search( + pick: Callable[[list[_DeploymentT]], _Picked | None], # mutable-ok: router deployment-list contract + deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract + request_kwargs: Mapping[str, object] | None, +) -> _Picked | None: + preferred: Final = prefer_web_search_deployments(deployments, request_kwargs) + if len(preferred) == len(deployments): + return pick(deployments) + try: + from_preferred: Final = pick(preferred) + except (litellm.RateLimitError, ValueError): + return pick(deployments) + return from_preferred if from_preferred is not None else pick(deployments) + + +async def apick_preferring_web_search( + pick: Callable[[list[_DeploymentT]], Awaitable[_Picked | None]], # mutable-ok: router deployment-list contract + deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract + request_kwargs: Mapping[str, object] | None, +) -> _Picked | None: + preferred: Final = prefer_web_search_deployments(deployments, request_kwargs) + if len(preferred) == len(deployments): + return await pick(deployments) + try: + from_preferred: Final = await pick(preferred) + except (litellm.RateLimitError, ValueError): + return await pick(deployments) + return from_preferred if from_preferred is not None else await pick(deployments) + + # Credential params that only one provider family reads, paired with the providers # that read them. A deployment carrying them while resolving elsewhere is almost # always a missing route prefix: `model: claude-sonnet-5` with `aws_region_name` diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index daf7d6bb345..62272e751b0 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -31,7 +31,7 @@ from litellm.caching.dual_cache import DualCache from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger, Span from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.router_utils.common_utils import deployment_ids_failed_in_request +from litellm.router_utils.common_utils import deployment_ids_failed_in_request, prefer_web_search_deployments from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes @@ -433,6 +433,10 @@ class DeploymentAffinityCheck(CustomLogger): failed_deployment_ids: Final = deployment_ids_failed_in_request( cast(Mapping[str, object], request_kwargs) # cast-ok: request kwargs keys are param names ) + pin_candidates: Final = prefer_web_search_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + request_kwargs, + ) user_key: Final = ( self.get_user_key_from_request_kwargs(request_kwargs=request_kwargs) if (session_affinity_active or enable_user_key) @@ -464,7 +468,7 @@ class DeploymentAffinityCheck(CustomLogger): if session_model_id and session_model_id not in failed_deployment_ids: session_deployment: Final = self._find_deployment_by_model_id( - healthy_deployments=typed_healthy_deployments, + healthy_deployments=pin_candidates, model_id=session_model_id, ) if session_deployment is not None: @@ -501,7 +505,7 @@ class DeploymentAffinityCheck(CustomLogger): return typed_healthy_deployments deployment = self._find_deployment_by_model_id( - healthy_deployments=typed_healthy_deployments, + healthy_deployments=pin_candidates, model_id=model_id, ) if deployment is None: diff --git a/litellm/types/router.py b/litellm/types/router.py index f927ae0475b..86c9bd011ea 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -26,7 +26,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from openai import Timeout as SDKTimeout from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator -from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable +from typing_extensions import NotRequired, Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -1228,6 +1228,7 @@ class RetryAttemptRecord(TypedDict): exception_type: ReadOnly[str] exception_string: ReadOnly[str] attempted_retries: ReadOnly[int | None] + configured_deployment_id: ReadOnly[NotRequired[str]] VALID_LITELLM_ENVIRONMENTS = [ diff --git a/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py index fa75f980cc0..799acc5d5cc 100644 --- a/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py @@ -1299,3 +1299,101 @@ async def test_previous_response_id_pin_holds_even_after_its_deployment_failed() }, parent_otel_span=None, ) == [healthy_deployments[0]] + + +def _mixed_web_search_router(optional_pre_call_checks: list[str]) -> litellm.Router: # mutable-ok: Router signature + return litellm.Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi"}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi"}, + "model_info": {"id": "search"}, + }, + ], + optional_pre_call_checks=optional_pre_call_checks, + ) + + +_CALLER_METADATA: Final = {"user_api_key_hash": "caller-1", "session_id": "session-1"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "web_search_request", + [{"web_search_options": {}}, {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}], + ids=["web-search-options", "web-search-tool"], +) +@pytest.mark.parametrize( + ("optional_pre_call_checks", "pin_key"), + [ + ( + ["deployment_affinity"], + DeploymentAffinityCheck.get_affinity_cache_key(model_group="mixed", user_key="caller-1"), + ), + ( + ["session_affinity"], + DeploymentAffinityCheck.get_session_affinity_cache_key( + model_group="mixed", session_id="session-1", user_key="caller-1" + ), + ), + ], + ids=["user-key-pin", "session-pin"], +) +async def test_pin_on_a_deployment_without_web_search_yields_to_a_web_search_request( + optional_pre_call_checks: list[str], # mutable-ok: Router signature + pin_key: str, + web_search_request: dict[str, object], # mutable-ok: request kwargs +): + router: Final = _mixed_web_search_router(optional_pre_call_checks) + await router.cache.async_set_cache(pin_key, {"model_id": "no-search"}, ttl=3600) + messages: Final = [{"role": "user", "content": "hi"}] + + plain: Final = await router.acompletion(model="mixed", messages=messages, metadata=dict(_CALLER_METADATA)) + searching: Final = await router.acompletion( + model="mixed", messages=messages, metadata=dict(_CALLER_METADATA), **web_search_request + ) + + assert plain._hidden_params["model_id"] == "no-search" + assert searching._hidden_params["model_id"] == "search" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("search_supported", "tried"), + [((False, False), ()), ((False, True), ("search",))], + ids=["no-deployment-can-search", "capable-deployment-already-tried"], +) +async def test_pin_holds_for_a_web_search_request_when_no_untried_deployment_can_search( + search_supported: tuple[bool, bool], tried: tuple[str, ...] +): + cache: Final = AsyncMock() + cache.async_get_cache = AsyncMock(return_value={"model_id": "no-search"}) + callback: Final = DeploymentAffinityCheck( + cache=cache, ttl_seconds=60, enable_user_key_affinity=True, enable_responses_api_affinity=False + ) + healthy_deployments: Final = [ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x"}, + "model_info": {"id": model_id, "supports_web_search": supported}, + } + for model_id, supported in zip(("no-search", "search"), search_supported) + ] + + filtered: Final = await callback.async_filter_deployments( + model="mixed", + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs={ + "metadata": {"user_api_key_hash": "caller-1"}, + "web_search_options": {}, + "_excluded_deployment_ids": list(tried), + }, + ) + + assert [d["model_info"]["id"] for d in filtered] == ["no-search"] diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 8011ca37cfb..4e76fff5aad 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -1,3 +1,4 @@ +import asyncio import logging from itertools import chain from typing import Dict, Final, List, Optional, Union @@ -15,6 +16,8 @@ from litellm.router_utils.common_utils import ( add_model_file_id_mappings, filter_team_based_models, filter_web_search_deployments, + pick_preferring_web_search, + prefer_web_search_deployments, format_content_policy_no_fallback_note, format_fallback_outcome_message, provider_for_generic_call, @@ -309,6 +312,44 @@ class TestFilterWebSearchDeployments: result_ids = [d["model_info"]["id"] for d in result] assert "deployment-3" not in result_ids + @pytest.mark.parametrize( + "tool", + [ + {"type": "web_search_20250305", "name": "web_search"}, + {"type": "web_search_20260209", "name": "web_search"}, + {"type": "web_search_preview_2025_03_11"}, + {"googleSearch": {}}, + {"google_search": {}}, + {"googleSearchRetrieval": {}}, + {"google_search_retrieval": {}}, + {"enterpriseWebSearch": {}}, + {"enterprise_web_search": {}}, + {"type": "google_search"}, + ], + ) + def test_native_web_search_tools_filter_unsupported( + self, sample_deployments: list[dict[str, object]], tool: dict[str, object] + ) -> None: + result: Final = prefer_web_search_deployments(sample_deployments, {"tools": [tool]}) + assert result == sample_deployments[:2] + + @pytest.mark.parametrize( + "tool", + [ + {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "function", "function": {"name": "web_search"}}, + {"type": "function", "name": "googleSearch"}, + {"urlContext": {}}, + {"googleSearch": None}, + {"google_search": None, "urlContext": {}}, + ], + ) + def test_non_web_search_tools_preserve_deployments( + self, sample_deployments: list[dict[str, object]], tool: dict[str, object] + ) -> None: + result: Final = prefer_web_search_deployments(sample_deployments, {"tools": [tool]}) + assert result == sample_deployments + def test_web_search_with_other_tools(self, sample_deployments): """Web search filtering works when mixed with other tools""" request_kwargs = { @@ -332,15 +373,44 @@ class TestFilterWebSearchDeployments: result = filter_web_search_deployments(deployments, request_kwargs) assert len(result) == 2 - def test_no_deployments_support_web_search(self): - """When no deployments support web search, all are filtered out""" - deployments = [ + @pytest.mark.parametrize("tool_type", ["web_search", "web_search_preview"]) + def test_no_deployment_supporting_openai_web_search_leaves_none(self, tool_type: str) -> None: + deployments: Final = [ {"model_info": {"id": "d1", "supports_web_search": False}}, {"model_info": {"id": "d2", "supports_web_search": False}}, ] - request_kwargs = {"tools": [{"type": "web_search"}]} - result = filter_web_search_deployments(deployments, request_kwargs) - assert len(result) == 0 + assert filter_web_search_deployments(deployments, {"tools": [{"type": tool_type}]}) == [] + + @pytest.mark.parametrize( + "request_kwargs", + [ + {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}, + {"tools": [{"type": "web_search_2025_08_26"}]}, + {"tools": [{"googleSearch": {}}]}, + {"tools": [{"type": "web_search"}]}, + {"web_search_options": {}}, + ], + ) + def test_no_deployment_supporting_web_search_keeps_every_deployment(self, request_kwargs: dict[str, object]) -> None: + deployments: Final[list[dict[str, object]]] = [ + {"model_info": {"id": "d1", "supports_web_search": False}}, + {"model_info": {"id": "d2", "supports_web_search": False}}, + ] + assert prefer_web_search_deployments(deployments, request_kwargs) == deployments + + def test_a_tool_that_is_not_an_object_is_no_web_search_request( + self, sample_deployments: list[dict[str, object]] + ) -> None: + result: Final = prefer_web_search_deployments(sample_deployments, {"tools": ["web_search", None]}) + assert result == sample_deployments + + def test_web_search_options_filter_unsupported(self, sample_deployments: list[dict[str, object]]) -> None: + result: Final = prefer_web_search_deployments(sample_deployments, {"web_search_options": {}}) + assert result == sample_deployments[:2] + + def test_null_web_search_options_preserve_deployments(self, sample_deployments: list[dict[str, object]]) -> None: + result: Final = prefer_web_search_deployments(sample_deployments, {"web_search_options": None}) + assert result == sample_deployments def test_missing_config_defaults_to_supported(self): """Deployments without supports_web_search config default to True""" @@ -375,6 +445,532 @@ class TestFilterWebSearchDeployments: assert result == deployment +def _router_with_a_deployment_that_disables_web_search(routing_strategy: str) -> Router: + return Router( + model_list=[ + { + "model_name": "search-model", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key"}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + { + "model_name": "search-model", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key"}, + "model_info": {"id": "search"}, + }, + ], + routing_strategy=routing_strategy, + ) + + +_ANTHROPIC_WEB_SEARCH_REQUEST: Final = {"tools": [{"type": "web_search_20250305", "name": "web_search"}]} + + +@pytest.mark.parametrize("routing_strategy", ["simple-shuffle", "usage-based-routing"]) +def test_sync_pick_skips_deployments_without_web_search(routing_strategy: str) -> None: + router: Final = _router_with_a_deployment_that_disables_web_search(routing_strategy) + picks: Final = { + router.get_available_deployment(model="search-model", request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST))[ + "model_info" + ]["id"] + for _ in range(20) + } + assert picks == {"search"} + + +def _router_with_a_group_that_disables_web_search(routing_strategy: str = "simple-shuffle") -> Router: + return Router( + model_list=[ + { + "model_name": "flagged-model", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "flagged"}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + { + "model_name": "search-model", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "searched"}, + "model_info": {"id": "search"}, + }, + ], + fallbacks=[{"flagged-model": ["search-model"]}], + routing_strategy=routing_strategy, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tools", "expected_content"), + [ + ([{"type": "web_search"}], "searched"), + ([{"type": "web_search_20250305", "name": "web_search"}], "flagged"), + ([{"googleSearch": dict[str, object]()}], "flagged"), + ], +) +async def test_web_search_to_a_group_without_web_search_routes_like_main( + tools: list[dict[str, object]], expected_content: str +) -> None: + response: Final = await _router_with_a_group_that_disables_web_search().acompletion( + model="flagged-model", messages=[{"role": "user", "content": "hi"}], tools=tools + ) + assert response.choices[0].message.content == expected_content + + +@pytest.mark.parametrize("routing_strategy", ["simple-shuffle", "usage-based-routing"]) +def test_sync_pick_keeps_a_group_without_web_search_for_openai_web_search(routing_strategy: str) -> None: + router: Final = _router_with_a_group_that_disables_web_search(routing_strategy) + deployment: Final = router.get_available_deployment( + model="flagged-model", request_kwargs={"tools": [{"type": "web_search"}]} + ) + assert deployment["model_info"]["id"] == "no-search" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "routing_strategy", ["usage-based-routing", "usage-based-routing-v2", "least-busy", "latency-based-routing"] +) +async def test_async_strategy_pick_skips_deployments_without_web_search(routing_strategy: str) -> None: + router: Final = _router_with_a_deployment_that_disables_web_search(routing_strategy) + picks: Final = { + ( + await router.async_get_available_deployment( + model="search-model", request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST) + ) + )["model_info"]["id"] + for _ in range(20) + } + assert picks == {"search"} + + +def _order_param(order: int | None) -> dict[str, int]: + return {} if order is None else {"order": order} + + +def _router_with_a_mixed_web_search_group( + enable_tag_filtering: bool, orders: tuple[int | None, int | None] = (None, None) +) -> Router: + return Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-x", + "api_key": "fake-key", + "tags": ["cheap"], + **_order_param(orders[0]), + }, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-y", + "api_key": "fake-key", + "tags": ["premium"], + **_order_param(orders[1]), + }, + "model_info": {"id": "search"}, + }, + ], + enable_tag_filtering=enable_tag_filtering, + ) + + +def _cool_down(router: Router, model_ids: tuple[str, ...]) -> None: + for model_id in model_ids: + router.cooldown_cache.add_deployment_to_cooldown( + model_id=model_id, original_exception=Exception("rate limited"), exception_status=429, cooldown_time=300 + ) + + +_LATER_FILTER_CASES: Final = pytest.mark.parametrize( + ("orders", "cooled_down", "request_kwargs", "expected_id"), + [ + ((None, None), (), _ANTHROPIC_WEB_SEARCH_REQUEST, "search"), + ((None, None), (), {"web_search_options": {}}, "search"), + ((1, 2), (), _ANTHROPIC_WEB_SEARCH_REQUEST, "no-search"), + ((1, 2), (), {"web_search_options": {}}, "no-search"), + ((None, None), ("search",), _ANTHROPIC_WEB_SEARCH_REQUEST, "no-search"), + ((None, None), ("search",), {"web_search_options": {}}, "no-search"), + ((None, None), (), {**_ANTHROPIC_WEB_SEARCH_REQUEST, "_excluded_deployment_ids": ["search"]}, "no-search"), + ((None, None), (), {**_ANTHROPIC_WEB_SEARCH_REQUEST, "_retry_skipped_deployment_ids": ("search",)}, "no-search"), + ], + ids=[ + "same-tier", + "same-tier-options", + "lower-order-tier", + "lower-order-tier-options", + "cooldown", + "cooldown-options", + "weighted-failover", + "retry-skip", + ], +) + + +@pytest.mark.asyncio +@_LATER_FILTER_CASES +async def test_async_pick_prefers_web_search_until_later_filters_leave_none_that_can( + orders: tuple[int | None, int | None], + cooled_down: tuple[str, ...], + request_kwargs: dict[str, object], + expected_id: str, +) -> None: + router: Final = _router_with_a_mixed_web_search_group(False, orders) + _cool_down(router, cooled_down) + deployment: Final = await router.async_get_available_deployment( + model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(request_kwargs) + ) + assert deployment["model_info"]["id"] == expected_id + + +@_LATER_FILTER_CASES +def test_sync_pick_prefers_web_search_until_later_filters_leave_none_that_can( + orders: tuple[int | None, int | None], + cooled_down: tuple[str, ...], + request_kwargs: dict[str, object], + expected_id: str, +) -> None: + router: Final = _router_with_a_mixed_web_search_group(False, orders) + _cool_down(router, cooled_down) + deployment: Final = router.get_available_deployment( + model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(request_kwargs) + ) + assert deployment["model_info"]["id"] == expected_id + + +_WEB_SEARCH_REQUESTS: Final = pytest.mark.parametrize( + "web_search_request", + [_ANTHROPIC_WEB_SEARCH_REQUEST, {"web_search_options": {}}], + ids=["web-search-tool", "web-search-options"], +) + + +@pytest.mark.asyncio +@_WEB_SEARCH_REQUESTS +async def test_order_fallback_answers_a_web_search_request_from_the_next_tier_when_the_capable_one_fails( + web_search_request: dict[str, object], +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "ordered", + "litellm_params": { + "model": "openai/gpt-x", + "api_key": "fake-key", + "mock_response": Exception("connection error"), + "order": 1, + }, + "model_info": {"id": "search"}, + }, + { + "model_name": "ordered", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "order": 2}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + num_retries=0, + ) + response: Final = await router.acompletion( + model="ordered", messages=[{"role": "user", "content": "hi"}], **web_search_request + ) + assert response._hidden_params["model_id"] == "no-search" + + +@pytest.mark.asyncio +@_WEB_SEARCH_REQUESTS +async def test_web_search_request_prefers_the_capable_deployment_inside_its_order_tier( + web_search_request: dict[str, object], +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "ordered", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "order": 1}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + { + "model_name": "ordered", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "order": 1}, + "model_info": {"id": "search"}, + }, + { + "model_name": "ordered", + "litellm_params": {"model": "openai/gpt-z", "api_key": "fake-key", "order": 2}, + "model_info": {"id": "later-no-search", "supports_web_search": False}, + }, + ], + ) + picks: Final = { + ( + await router.async_get_available_deployment( + model="ordered", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(web_search_request) + ) + )["model_info"]["id"] + for _ in range(20) + } + assert picks == {"search"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_web_search_preference_gives_way_once_the_capable_deployment_failed_this_request( + metadata_field: str, +) -> None: + router: Final = _router_with_a_mixed_web_search_group(False) + picks: Final = { + ( + await router.async_get_available_deployment( + model="mixed", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={ + **_ANTHROPIC_WEB_SEARCH_REQUEST, + metadata_field: {"previous_models": ({"deployment_id": "search", "exception_type": "Timeout"},)}, + }, + ) + )["model_info"]["id"] + for _ in range(40) + } + assert picks == {"search", "no-search"} + + +@pytest.mark.asyncio +@_WEB_SEARCH_REQUESTS +@pytest.mark.parametrize( + "caller_credentials", [{}, {"api_key": "caller-key"}], ids=["configured-credentials", "caller-credentials"] +) +async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches_the_other_one( + web_search_request: dict[str, object], + caller_credentials: dict[str, str], +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-x", + "api_key": "fake-key", + "mock_response": Exception("connection error"), + "weight": 1, + }, + "model_info": {"id": "search"}, + }, + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-y", + "api_key": "fake-key", + "mock_response": "from no-search", + "weight": 10**6, + }, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + num_retries=1, + disable_cooldowns=True, + ) + response: Final = await router.acompletion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request, **caller_credentials + ) + assert response.choices[0].message.content == "from no-search" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize(("request_retry_count", "flagged_reachable"), [(1, False), (2, True)], ids=["kept", "trimmed"]) +async def test_web_search_preference_steps_aside_only_once_the_retry_history_was_trimmed( + metadata_field: str, request_retry_count: int, flagged_reachable: bool +) -> None: + router: Final = Router( + model_list=[ + *( + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("search-a", "search-b") + ), + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-y", + "api_key": "fake-key", + "mock_response": "hi", + "weight": 10**6, + }, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + ) + picks: Final = { + ( + await router.async_get_available_deployment( + model="mixed", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={ + **_ANTHROPIC_WEB_SEARCH_REQUEST, + metadata_field: { + "previous_models": ({"deployment_id": "search-a", "exception_type": "Timeout"},), + "request_retry_count": request_retry_count, + }, + }, + ) + )["model_info"]["id"] + for _ in range(20) + } + assert ("no-search" in picks) is flagged_reachable + + +@pytest.mark.asyncio +async def test_retries_past_the_kept_retry_history_reach_the_deployment_that_cannot_search() -> None: + capable_deployments: Final = [ + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-x", + "api_key": "fake-key", + "mock_response": Exception("connection error"), + }, + "model_info": {"id": f"search-{index}"}, + } + for index in range(5) + ] + router: Final = Router( + model_list=[ + *capable_deployments, + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-y", + "api_key": "fake-key", + "mock_response": "from no-search", + "weight": 10**6, + }, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + num_retries=5, + disable_cooldowns=True, + ) + response: Final = await router.acompletion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST + ) + assert response.choices[0].message.content == "from no-search" + + +def _rate_limited_web_search_group(routing_strategy: str) -> Router: + return Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi", "rpm": 1}, + "model_info": {"id": "search"}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "rpm": 1000}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + routing_strategy=routing_strategy, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("routing_strategy", ["usage-based-routing-v2", "cost-based-routing"]) +async def test_web_search_request_goes_to_the_other_deployment_once_the_capable_one_is_at_its_rpm_limit( + routing_strategy: str, +) -> None: + router: Final = _rate_limited_web_search_group(routing_strategy) + served_by: Final = [] # mutable-ok: collects one id per sequential request + for _ in range(3): + response = await router.acompletion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST + ) + served_by.append(response._hidden_params["model_id"]) + await asyncio.sleep(0.2) + assert served_by == ["search", "no-search", "no-search"] + + +def test_sync_web_search_request_goes_to_the_other_deployment_once_the_capable_one_is_at_its_rpm_limit() -> None: + router: Final = _rate_limited_web_search_group("usage-based-routing-v2") + served_by: Final = tuple( + router.completion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST + )._hidden_params["model_id"] + for _ in range(3) + ) + assert served_by == ("search", "no-search", "no-search") + + +def test_sync_pick_falls_back_to_every_deployment_when_the_strategy_finds_none_that_can_search() -> None: + deployments: Final = [ + {"model_info": {"id": "search"}}, + {"model_info": {"id": "no-search", "supports_web_search": False}}, + ] + picked: Final = pick_preferring_web_search( + lambda candidates: next((d for d in candidates if d["model_info"]["id"] == "no-search"), None), + deployments, + _ANTHROPIC_WEB_SEARCH_REQUEST, + ) + assert picked == deployments[1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("search_order", "no_search_order"), + [([1], [1]), ({"tier": 1}, None)], + ids=["list-order", "dict-order"], +) +async def test_web_search_request_routes_on_a_group_whose_order_is_not_a_number( + search_order: object, no_search_order: object +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key"}, + "model_info": {"id": "search", "order": search_order}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key"}, + "model_info": { + "id": "no-search", + "supports_web_search": False, + **({} if no_search_order is None else {"order": no_search_order}), + }, + }, + ], + ) + deployment: Final = await router.async_get_available_deployment( + model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST) + ) + assert deployment["model_info"]["id"] == "search" + + +@pytest.mark.asyncio +async def test_tag_routing_to_a_deployment_without_web_search_keeps_it() -> None: + deployment: Final = await _router_with_a_mixed_web_search_group(True).async_get_available_deployment( + model="mixed", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={"web_search_options": {}, "metadata": {"tags": ["cheap"]}}, + ) + assert deployment["model_info"]["id"] == "no-search" + + +@pytest.mark.asyncio +async def test_openai_web_search_still_finds_no_deployment_while_the_only_one_that_can_search_cools_down() -> None: + router: Final = _router_with_a_mixed_web_search_group(False) + _cool_down(router, ("search",)) + with pytest.raises(litellm.RouterRateLimitError): + await router.async_get_available_deployment( + model="mixed", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={"tools": [{"type": "web_search"}]}, + ) + + def test_invalidate_model_group_info_cache(): """Test that _invalidate_model_group_info_cache clears the LRU cache.""" router = Router(