mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 46aed6565f into d302096b5a
This commit is contained in:
commit
27ecff4b82
6 changed files with 887 additions and 38 deletions
|
|
@ -118,6 +118,7 @@ from litellm.litellm_core_utils.batch_utils import (
|
|||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
coerce_token_limit,
|
||||
get_litellm_metadata_from_kwargs,
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
|
|
@ -254,6 +255,7 @@ from litellm.router_utils.clientside_credential_handler import (
|
|||
)
|
||||
from litellm.router_utils.common_utils import (
|
||||
MID_STREAM_CONTINUATION_KWARG,
|
||||
apick_preferring_web_search,
|
||||
as_retry_skipped_deployment_ids,
|
||||
deployments_the_caller_may_use,
|
||||
filter_team_based_models,
|
||||
|
|
@ -264,6 +266,8 @@ from litellm.router_utils.common_utils import (
|
|||
format_no_fallback_group_message,
|
||||
get_request_team_id,
|
||||
is_proxy_admin_request,
|
||||
pick_preferring_web_search,
|
||||
prefer_web_search_deployments,
|
||||
provider_for_generic_call,
|
||||
request_user_api_key_auth,
|
||||
resolve_model_group_alias,
|
||||
|
|
@ -10108,14 +10112,15 @@ class Router(DeprecatedRouterMembers):
|
|||
_metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
|
||||
request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs
|
||||
model_group: Final = kwargs.get("model")
|
||||
model_info: Final = request_metadata.get("model_info")
|
||||
deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None
|
||||
model_info: Final = as_str_mapping(request_metadata.get("model_info")) or {}
|
||||
deployment_id: Final = model_info.get("id")
|
||||
configured_deployment_id: Final = model_info.get("original_model_id")
|
||||
base_model: Final = get_base_model_from_litellm_call_metadata(
|
||||
metadata=request_metadata
|
||||
) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None)
|
||||
deployment_model: Final = request_metadata.get("deployment")
|
||||
attempted_retries: Final = request_metadata.get("attempted_retries")
|
||||
attempt_record: Final[RetryAttemptRecord] = {
|
||||
attempt_fields: Final[RetryAttemptRecord] = {
|
||||
"model_group": model_group if isinstance(model_group, str) else None,
|
||||
"deployment_id": deployment_id if isinstance(deployment_id, str) else None,
|
||||
"model": deployment_model if isinstance(deployment_model, str) else None,
|
||||
|
|
@ -10124,6 +10129,11 @@ class Router(DeprecatedRouterMembers):
|
|||
"exception_string": str(e),
|
||||
"attempted_retries": attempted_retries if type(attempted_retries) is int else None,
|
||||
}
|
||||
attempt_record: Final[RetryAttemptRecord] = (
|
||||
{**attempt_fields, "configured_deployment_id": configured_deployment_id}
|
||||
if isinstance(configured_deployment_id, str)
|
||||
else attempt_fields
|
||||
)
|
||||
earlier_breadcrumbs: Final = request_metadata.get("previous_models")
|
||||
kept_breadcrumbs: Final[tuple[object, ...]] = (
|
||||
tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :]
|
||||
|
|
@ -15792,26 +15802,31 @@ class Router(DeprecatedRouterMembers):
|
|||
return healthy_deployments[0]
|
||||
|
||||
start_time: Final = time.time()
|
||||
routing_kwargs: Final = cast(Mapping[str, object], request_kwargs) # cast-ok: str-keyed request kwargs
|
||||
if strategy == "simple-shuffle":
|
||||
shuffled: Final = simple_shuffle(
|
||||
resolve_model_alias=self._core_get_model_from_alias,
|
||||
healthy_deployments=healthy_deployments,
|
||||
healthy_deployments=prefer_web_search_deployments(healthy_deployments, routing_kwargs),
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
request_kwargs=routing_kwargs,
|
||||
)
|
||||
phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes)
|
||||
return shuffled
|
||||
with PrefetchedUsage.scoped(
|
||||
routing_read_batch.prefetched_usage if routing_read_batch is not None else None
|
||||
):
|
||||
deployment: Final = await self._select_deployment_async(
|
||||
strategy=strategy,
|
||||
selector=strategy_selector,
|
||||
model=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
deployment: Final = await apick_preferring_web_search(
|
||||
lambda candidates: self._select_deployment_async(
|
||||
strategy=strategy,
|
||||
selector=strategy_selector,
|
||||
model=model,
|
||||
healthy_deployments=candidates,
|
||||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
),
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
routing_kwargs,
|
||||
)
|
||||
if deployment is None:
|
||||
exception: Final = await async_raise_no_deployment_exception(
|
||||
|
|
@ -16842,20 +16857,24 @@ class Router(DeprecatedRouterMembers):
|
|||
############## Check 'weight' param set for weighted pick #################
|
||||
shuffled: Final = simple_shuffle(
|
||||
resolve_model_alias=self._core_get_model_from_alias,
|
||||
healthy_deployments=healthy_deployments,
|
||||
healthy_deployments=prefer_web_search_deployments(healthy_deployments, routing_kwargs),
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
request_kwargs=routing_kwargs,
|
||||
)
|
||||
phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes)
|
||||
return shuffled
|
||||
deployment: Final = self._select_deployment_sync(
|
||||
strategy=strategy,
|
||||
selector=strategy_selector,
|
||||
model=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
deployment: Final = pick_preferring_web_search(
|
||||
lambda candidates: self._select_deployment_sync(
|
||||
strategy=strategy,
|
||||
selector=strategy_selector,
|
||||
model=model,
|
||||
healthy_deployments=candidates,
|
||||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
),
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
routing_kwargs,
|
||||
)
|
||||
|
||||
if deployment is None:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeVar
|
||||
|
||||
|
|
@ -13,9 +13,11 @@ import litellm
|
|||
from litellm._logging import verbose_logger, verbose_router_logger
|
||||
from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
|
||||
from litellm.rust_bridge.public_call import optional_sequence
|
||||
from litellm.types.router import (
|
||||
CredentialLiteLLMParams,
|
||||
Deployment,
|
||||
|
|
@ -31,14 +33,31 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
_V = TypeVar("_V")
|
||||
_Picked: Final = TypeVar("_Picked")
|
||||
_DeploymentT: Final = TypeVar("_DeploymentT", bound=dict[str, object])
|
||||
|
||||
MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation"
|
||||
ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG})
|
||||
_OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
|
||||
_GEMINI_SEARCH_TOOL_KEYS: Final = frozenset(
|
||||
{
|
||||
"googleSearch",
|
||||
"google_search",
|
||||
"googleSearchRetrieval",
|
||||
"google_search_retrieval",
|
||||
"enterpriseWebSearch",
|
||||
"enterprise_web_search",
|
||||
}
|
||||
)
|
||||
_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "google_search"})
|
||||
_OPENAI_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"})
|
||||
_DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
|
||||
tuple[Mapping[str, object], ...]
|
||||
)
|
||||
_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
|
||||
_RETRY_BREADCRUMBS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
|
||||
tuple[Mapping[str, object], ...]
|
||||
)
|
||||
_FALLBACK_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
|
||||
tuple[Mapping[str, object], ...]
|
||||
)
|
||||
|
|
@ -407,9 +426,6 @@ def deployments_the_caller_may_use(
|
|||
return ()
|
||||
|
||||
|
||||
_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"})
|
||||
|
||||
|
||||
def _deployment_supports_web_search(deployment: dict) -> bool:
|
||||
"""
|
||||
Check if a deployment supports web search.
|
||||
|
|
@ -430,6 +446,22 @@ def _deployment_supports_web_search(deployment: dict) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
def _is_web_search_tool(tool: object) -> bool:
|
||||
tool_fields: Final = as_str_mapping(tool)
|
||||
if tool_fields is None:
|
||||
return False
|
||||
tool_type: Final = tool_fields.get("type")
|
||||
if isinstance(tool_type, str) and (tool_type in _WEB_SEARCH_TOOL_TYPES or tool_type.startswith("web_search_")):
|
||||
return True
|
||||
return not _GEMINI_SEARCH_TOOL_KEYS.isdisjoint(tool_fields) and any(
|
||||
tool_fields[key] is not None for key in _GEMINI_SEARCH_TOOL_KEYS.intersection(tool_fields)
|
||||
)
|
||||
|
||||
|
||||
def _is_web_search_request(tools: object, web_search_options: object) -> bool:
|
||||
return web_search_options is not None or any(_is_web_search_tool(tool) for tool in optional_sequence(tools) or ())
|
||||
|
||||
|
||||
def filter_web_search_deployments(
|
||||
healthy_deployments: list[dict] | dict,
|
||||
request_kwargs: dict | None = None,
|
||||
|
|
@ -448,7 +480,7 @@ def filter_web_search_deployments(
|
|||
is_web_search_request: Final = isinstance(tools, (list, tuple)) and any(
|
||||
isinstance(tool, dict)
|
||||
and isinstance(tool_type := tool.get("type"), str)
|
||||
and tool_type in _WEB_SEARCH_TOOL_TYPES
|
||||
and tool_type in _OPENAI_WEB_SEARCH_TOOL_TYPES
|
||||
for tool in tools
|
||||
)
|
||||
if not is_web_search_request:
|
||||
|
|
@ -461,6 +493,105 @@ def filter_web_search_deployments(
|
|||
return final_deployments
|
||||
|
||||
|
||||
def _deployment_id(deployment: Mapping[str, object]) -> object:
|
||||
model_info: Final = as_str_mapping(deployment.get("model_info"))
|
||||
return None if model_info is None else model_info.get("id")
|
||||
|
||||
|
||||
def _retry_breadcrumbs(metadata: object) -> tuple[Mapping[str, object], ...]:
|
||||
fields: Final = as_str_mapping(metadata)
|
||||
breadcrumbs: Final = None if fields is None else fields.get("previous_models")
|
||||
if breadcrumbs is None:
|
||||
return ()
|
||||
try:
|
||||
return _RETRY_BREADCRUMBS_ADAPTER.validate_python(breadcrumbs)
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def _retry_history_was_trimmed(metadata: object) -> bool:
|
||||
fields: Final = as_str_mapping(metadata)
|
||||
retry_count: Final = None if fields is None else fields.get("request_retry_count")
|
||||
return type(retry_count) is int and retry_count > len(_retry_breadcrumbs(metadata))
|
||||
|
||||
|
||||
def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) -> tuple[str, ...]:
|
||||
breadcrumbs: Final = (
|
||||
*_retry_breadcrumbs(request_kwargs.get("metadata")),
|
||||
*_retry_breadcrumbs(request_kwargs.get("litellm_metadata")),
|
||||
)
|
||||
return tuple(
|
||||
deployment_id
|
||||
for breadcrumb in breadcrumbs
|
||||
if isinstance(deployment_id := breadcrumb.get("configured_deployment_id", breadcrumb.get("deployment_id")), str)
|
||||
)
|
||||
|
||||
|
||||
def prefer_web_search_deployments(
|
||||
deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract
|
||||
request_kwargs: Mapping[str, object] | None,
|
||||
) -> list[_DeploymentT]: # mutable-ok: returns a mutable deployment list
|
||||
if request_kwargs is None or not _is_web_search_request(
|
||||
request_kwargs.get("tools"), request_kwargs.get("web_search_options")
|
||||
):
|
||||
return deployments
|
||||
if _retry_history_was_trimmed(request_kwargs.get("metadata")) or _retry_history_was_trimmed(
|
||||
request_kwargs.get("litellm_metadata")
|
||||
):
|
||||
verbose_router_logger.debug("This request's retry history was trimmed, so every deployment stays")
|
||||
return deployments
|
||||
tried: Final = frozenset(
|
||||
(
|
||||
*as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")),
|
||||
*as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")),
|
||||
*_deployment_ids_retried_in_request(request_kwargs),
|
||||
)
|
||||
)
|
||||
untried: Final = (d for d in deployments if _deployment_id(d) not in tried)
|
||||
orders_with_an_untried_capable_deployment: Final = tuple(
|
||||
litellm.utils.get_deployment_order(d) for d in untried if _deployment_supports_web_search(d)
|
||||
)
|
||||
if not orders_with_an_untried_capable_deployment:
|
||||
verbose_router_logger.debug("No deployment left for this web search request supports web search, so all stay")
|
||||
return deployments
|
||||
return [
|
||||
d
|
||||
for d in deployments
|
||||
if _deployment_supports_web_search(d)
|
||||
or litellm.utils.get_deployment_order(d) not in orders_with_an_untried_capable_deployment
|
||||
]
|
||||
|
||||
|
||||
def pick_preferring_web_search(
|
||||
pick: Callable[[list[_DeploymentT]], _Picked | None], # mutable-ok: router deployment-list contract
|
||||
deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract
|
||||
request_kwargs: Mapping[str, object] | None,
|
||||
) -> _Picked | None:
|
||||
preferred: Final = prefer_web_search_deployments(deployments, request_kwargs)
|
||||
if len(preferred) == len(deployments):
|
||||
return pick(deployments)
|
||||
try:
|
||||
from_preferred: Final = pick(preferred)
|
||||
except (litellm.RateLimitError, ValueError):
|
||||
return pick(deployments)
|
||||
return from_preferred if from_preferred is not None else pick(deployments)
|
||||
|
||||
|
||||
async def apick_preferring_web_search(
|
||||
pick: Callable[[list[_DeploymentT]], Awaitable[_Picked | None]], # mutable-ok: router deployment-list contract
|
||||
deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract
|
||||
request_kwargs: Mapping[str, object] | None,
|
||||
) -> _Picked | None:
|
||||
preferred: Final = prefer_web_search_deployments(deployments, request_kwargs)
|
||||
if len(preferred) == len(deployments):
|
||||
return await pick(deployments)
|
||||
try:
|
||||
from_preferred: Final = await pick(preferred)
|
||||
except (litellm.RateLimitError, ValueError):
|
||||
return await pick(deployments)
|
||||
return from_preferred if from_preferred is not None else await pick(deployments)
|
||||
|
||||
|
||||
# Credential params that only one provider family reads, paired with the providers
|
||||
# that read them. A deployment carrying them while resolving elsewhere is almost
|
||||
# always a missing route prefix: `model: claude-sonnet-5` with `aws_region_name`
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ from litellm.caching.dual_cache import DualCache
|
|||
from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.common_utils import deployment_ids_failed_in_request
|
||||
from litellm.router_utils.common_utils import deployment_ids_failed_in_request, prefer_web_search_deployments
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
|
|
@ -433,6 +433,10 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
failed_deployment_ids: Final = deployment_ids_failed_in_request(
|
||||
cast(Mapping[str, object], request_kwargs) # cast-ok: request kwargs keys are param names
|
||||
)
|
||||
pin_candidates: Final = prefer_web_search_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
request_kwargs,
|
||||
)
|
||||
user_key: Final = (
|
||||
self.get_user_key_from_request_kwargs(request_kwargs=request_kwargs)
|
||||
if (session_affinity_active or enable_user_key)
|
||||
|
|
@ -464,7 +468,7 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
|
||||
if session_model_id and session_model_id not in failed_deployment_ids:
|
||||
session_deployment: Final = self._find_deployment_by_model_id(
|
||||
healthy_deployments=typed_healthy_deployments,
|
||||
healthy_deployments=pin_candidates,
|
||||
model_id=session_model_id,
|
||||
)
|
||||
if session_deployment is not None:
|
||||
|
|
@ -501,7 +505,7 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
return typed_healthy_deployments
|
||||
|
||||
deployment = self._find_deployment_by_model_id(
|
||||
healthy_deployments=typed_healthy_deployments,
|
||||
healthy_deployments=pin_candidates,
|
||||
model_id=model_id,
|
||||
)
|
||||
if deployment is None:
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|||
import httpx
|
||||
from openai import Timeout as SDKTimeout
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator
|
||||
from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
|
||||
from typing_extensions import NotRequired, Protocol, ReadOnly, Required, TypedDict, runtime_checkable
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -1228,6 +1228,7 @@ class RetryAttemptRecord(TypedDict):
|
|||
exception_type: ReadOnly[str]
|
||||
exception_string: ReadOnly[str]
|
||||
attempted_retries: ReadOnly[int | None]
|
||||
configured_deployment_id: ReadOnly[NotRequired[str]]
|
||||
|
||||
|
||||
VALID_LITELLM_ENVIRONMENTS = [
|
||||
|
|
|
|||
|
|
@ -1299,3 +1299,101 @@ async def test_previous_response_id_pin_holds_even_after_its_deployment_failed()
|
|||
},
|
||||
parent_otel_span=None,
|
||||
) == [healthy_deployments[0]]
|
||||
|
||||
|
||||
def _mixed_web_search_router(optional_pre_call_checks: list[str]) -> litellm.Router: # mutable-ok: Router signature
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi"},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi"},
|
||||
"model_info": {"id": "search"},
|
||||
},
|
||||
],
|
||||
optional_pre_call_checks=optional_pre_call_checks,
|
||||
)
|
||||
|
||||
|
||||
_CALLER_METADATA: Final = {"user_api_key_hash": "caller-1", "session_id": "session-1"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"web_search_request",
|
||||
[{"web_search_options": {}}, {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}],
|
||||
ids=["web-search-options", "web-search-tool"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("optional_pre_call_checks", "pin_key"),
|
||||
[
|
||||
(
|
||||
["deployment_affinity"],
|
||||
DeploymentAffinityCheck.get_affinity_cache_key(model_group="mixed", user_key="caller-1"),
|
||||
),
|
||||
(
|
||||
["session_affinity"],
|
||||
DeploymentAffinityCheck.get_session_affinity_cache_key(
|
||||
model_group="mixed", session_id="session-1", user_key="caller-1"
|
||||
),
|
||||
),
|
||||
],
|
||||
ids=["user-key-pin", "session-pin"],
|
||||
)
|
||||
async def test_pin_on_a_deployment_without_web_search_yields_to_a_web_search_request(
|
||||
optional_pre_call_checks: list[str], # mutable-ok: Router signature
|
||||
pin_key: str,
|
||||
web_search_request: dict[str, object], # mutable-ok: request kwargs
|
||||
):
|
||||
router: Final = _mixed_web_search_router(optional_pre_call_checks)
|
||||
await router.cache.async_set_cache(pin_key, {"model_id": "no-search"}, ttl=3600)
|
||||
messages: Final = [{"role": "user", "content": "hi"}]
|
||||
|
||||
plain: Final = await router.acompletion(model="mixed", messages=messages, metadata=dict(_CALLER_METADATA))
|
||||
searching: Final = await router.acompletion(
|
||||
model="mixed", messages=messages, metadata=dict(_CALLER_METADATA), **web_search_request
|
||||
)
|
||||
|
||||
assert plain._hidden_params["model_id"] == "no-search"
|
||||
assert searching._hidden_params["model_id"] == "search"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("search_supported", "tried"),
|
||||
[((False, False), ()), ((False, True), ("search",))],
|
||||
ids=["no-deployment-can-search", "capable-deployment-already-tried"],
|
||||
)
|
||||
async def test_pin_holds_for_a_web_search_request_when_no_untried_deployment_can_search(
|
||||
search_supported: tuple[bool, bool], tried: tuple[str, ...]
|
||||
):
|
||||
cache: Final = AsyncMock()
|
||||
cache.async_get_cache = AsyncMock(return_value={"model_id": "no-search"})
|
||||
callback: Final = DeploymentAffinityCheck(
|
||||
cache=cache, ttl_seconds=60, enable_user_key_affinity=True, enable_responses_api_affinity=False
|
||||
)
|
||||
healthy_deployments: Final = [
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-x"},
|
||||
"model_info": {"id": model_id, "supports_web_search": supported},
|
||||
}
|
||||
for model_id, supported in zip(("no-search", "search"), search_supported)
|
||||
]
|
||||
|
||||
filtered: Final = await callback.async_filter_deployments(
|
||||
model="mixed",
|
||||
healthy_deployments=healthy_deployments,
|
||||
messages=None,
|
||||
request_kwargs={
|
||||
"metadata": {"user_api_key_hash": "caller-1"},
|
||||
"web_search_options": {},
|
||||
"_excluded_deployment_ids": list(tried),
|
||||
},
|
||||
)
|
||||
|
||||
assert [d["model_info"]["id"] for d in filtered] == ["no-search"]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import asyncio
|
||||
import logging
|
||||
from itertools import chain
|
||||
from typing import Dict, Final, List, Optional, Union
|
||||
|
|
@ -15,6 +16,8 @@ from litellm.router_utils.common_utils import (
|
|||
add_model_file_id_mappings,
|
||||
filter_team_based_models,
|
||||
filter_web_search_deployments,
|
||||
pick_preferring_web_search,
|
||||
prefer_web_search_deployments,
|
||||
format_content_policy_no_fallback_note,
|
||||
format_fallback_outcome_message,
|
||||
provider_for_generic_call,
|
||||
|
|
@ -309,6 +312,44 @@ class TestFilterWebSearchDeployments:
|
|||
result_ids = [d["model_info"]["id"] for d in result]
|
||||
assert "deployment-3" not in result_ids
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
{"type": "web_search_20250305", "name": "web_search"},
|
||||
{"type": "web_search_20260209", "name": "web_search"},
|
||||
{"type": "web_search_preview_2025_03_11"},
|
||||
{"googleSearch": {}},
|
||||
{"google_search": {}},
|
||||
{"googleSearchRetrieval": {}},
|
||||
{"google_search_retrieval": {}},
|
||||
{"enterpriseWebSearch": {}},
|
||||
{"enterprise_web_search": {}},
|
||||
{"type": "google_search"},
|
||||
],
|
||||
)
|
||||
def test_native_web_search_tools_filter_unsupported(
|
||||
self, sample_deployments: list[dict[str, object]], tool: dict[str, object]
|
||||
) -> None:
|
||||
result: Final = prefer_web_search_deployments(sample_deployments, {"tools": [tool]})
|
||||
assert result == sample_deployments[:2]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
{"type": "web_fetch_20250910", "name": "web_fetch"},
|
||||
{"type": "function", "function": {"name": "web_search"}},
|
||||
{"type": "function", "name": "googleSearch"},
|
||||
{"urlContext": {}},
|
||||
{"googleSearch": None},
|
||||
{"google_search": None, "urlContext": {}},
|
||||
],
|
||||
)
|
||||
def test_non_web_search_tools_preserve_deployments(
|
||||
self, sample_deployments: list[dict[str, object]], tool: dict[str, object]
|
||||
) -> None:
|
||||
result: Final = prefer_web_search_deployments(sample_deployments, {"tools": [tool]})
|
||||
assert result == sample_deployments
|
||||
|
||||
def test_web_search_with_other_tools(self, sample_deployments):
|
||||
"""Web search filtering works when mixed with other tools"""
|
||||
request_kwargs = {
|
||||
|
|
@ -332,15 +373,44 @@ class TestFilterWebSearchDeployments:
|
|||
result = filter_web_search_deployments(deployments, request_kwargs)
|
||||
assert len(result) == 2
|
||||
|
||||
def test_no_deployments_support_web_search(self):
|
||||
"""When no deployments support web search, all are filtered out"""
|
||||
deployments = [
|
||||
@pytest.mark.parametrize("tool_type", ["web_search", "web_search_preview"])
|
||||
def test_no_deployment_supporting_openai_web_search_leaves_none(self, tool_type: str) -> None:
|
||||
deployments: Final = [
|
||||
{"model_info": {"id": "d1", "supports_web_search": False}},
|
||||
{"model_info": {"id": "d2", "supports_web_search": False}},
|
||||
]
|
||||
request_kwargs = {"tools": [{"type": "web_search"}]}
|
||||
result = filter_web_search_deployments(deployments, request_kwargs)
|
||||
assert len(result) == 0
|
||||
assert filter_web_search_deployments(deployments, {"tools": [{"type": tool_type}]}) == []
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_kwargs",
|
||||
[
|
||||
{"tools": [{"type": "web_search_20250305", "name": "web_search"}]},
|
||||
{"tools": [{"type": "web_search_2025_08_26"}]},
|
||||
{"tools": [{"googleSearch": {}}]},
|
||||
{"tools": [{"type": "web_search"}]},
|
||||
{"web_search_options": {}},
|
||||
],
|
||||
)
|
||||
def test_no_deployment_supporting_web_search_keeps_every_deployment(self, request_kwargs: dict[str, object]) -> None:
|
||||
deployments: Final[list[dict[str, object]]] = [
|
||||
{"model_info": {"id": "d1", "supports_web_search": False}},
|
||||
{"model_info": {"id": "d2", "supports_web_search": False}},
|
||||
]
|
||||
assert prefer_web_search_deployments(deployments, request_kwargs) == deployments
|
||||
|
||||
def test_a_tool_that_is_not_an_object_is_no_web_search_request(
|
||||
self, sample_deployments: list[dict[str, object]]
|
||||
) -> None:
|
||||
result: Final = prefer_web_search_deployments(sample_deployments, {"tools": ["web_search", None]})
|
||||
assert result == sample_deployments
|
||||
|
||||
def test_web_search_options_filter_unsupported(self, sample_deployments: list[dict[str, object]]) -> None:
|
||||
result: Final = prefer_web_search_deployments(sample_deployments, {"web_search_options": {}})
|
||||
assert result == sample_deployments[:2]
|
||||
|
||||
def test_null_web_search_options_preserve_deployments(self, sample_deployments: list[dict[str, object]]) -> None:
|
||||
result: Final = prefer_web_search_deployments(sample_deployments, {"web_search_options": None})
|
||||
assert result == sample_deployments
|
||||
|
||||
def test_missing_config_defaults_to_supported(self):
|
||||
"""Deployments without supports_web_search config default to True"""
|
||||
|
|
@ -375,6 +445,532 @@ class TestFilterWebSearchDeployments:
|
|||
assert result == deployment
|
||||
|
||||
|
||||
def _router_with_a_deployment_that_disables_web_search(routing_strategy: str) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "search-model",
|
||||
"litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key"},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
{
|
||||
"model_name": "search-model",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key"},
|
||||
"model_info": {"id": "search"},
|
||||
},
|
||||
],
|
||||
routing_strategy=routing_strategy,
|
||||
)
|
||||
|
||||
|
||||
_ANTHROPIC_WEB_SEARCH_REQUEST: Final = {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("routing_strategy", ["simple-shuffle", "usage-based-routing"])
|
||||
def test_sync_pick_skips_deployments_without_web_search(routing_strategy: str) -> None:
|
||||
router: Final = _router_with_a_deployment_that_disables_web_search(routing_strategy)
|
||||
picks: Final = {
|
||||
router.get_available_deployment(model="search-model", request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST))[
|
||||
"model_info"
|
||||
]["id"]
|
||||
for _ in range(20)
|
||||
}
|
||||
assert picks == {"search"}
|
||||
|
||||
|
||||
def _router_with_a_group_that_disables_web_search(routing_strategy: str = "simple-shuffle") -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "flagged-model",
|
||||
"litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "flagged"},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
{
|
||||
"model_name": "search-model",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "searched"},
|
||||
"model_info": {"id": "search"},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"flagged-model": ["search-model"]}],
|
||||
routing_strategy=routing_strategy,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("tools", "expected_content"),
|
||||
[
|
||||
([{"type": "web_search"}], "searched"),
|
||||
([{"type": "web_search_20250305", "name": "web_search"}], "flagged"),
|
||||
([{"googleSearch": dict[str, object]()}], "flagged"),
|
||||
],
|
||||
)
|
||||
async def test_web_search_to_a_group_without_web_search_routes_like_main(
|
||||
tools: list[dict[str, object]], expected_content: str
|
||||
) -> None:
|
||||
response: Final = await _router_with_a_group_that_disables_web_search().acompletion(
|
||||
model="flagged-model", messages=[{"role": "user", "content": "hi"}], tools=tools
|
||||
)
|
||||
assert response.choices[0].message.content == expected_content
|
||||
|
||||
|
||||
@pytest.mark.parametrize("routing_strategy", ["simple-shuffle", "usage-based-routing"])
|
||||
def test_sync_pick_keeps_a_group_without_web_search_for_openai_web_search(routing_strategy: str) -> None:
|
||||
router: Final = _router_with_a_group_that_disables_web_search(routing_strategy)
|
||||
deployment: Final = router.get_available_deployment(
|
||||
model="flagged-model", request_kwargs={"tools": [{"type": "web_search"}]}
|
||||
)
|
||||
assert deployment["model_info"]["id"] == "no-search"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"routing_strategy", ["usage-based-routing", "usage-based-routing-v2", "least-busy", "latency-based-routing"]
|
||||
)
|
||||
async def test_async_strategy_pick_skips_deployments_without_web_search(routing_strategy: str) -> None:
|
||||
router: Final = _router_with_a_deployment_that_disables_web_search(routing_strategy)
|
||||
picks: Final = {
|
||||
(
|
||||
await router.async_get_available_deployment(
|
||||
model="search-model", request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST)
|
||||
)
|
||||
)["model_info"]["id"]
|
||||
for _ in range(20)
|
||||
}
|
||||
assert picks == {"search"}
|
||||
|
||||
|
||||
def _order_param(order: int | None) -> dict[str, int]:
|
||||
return {} if order is None else {"order": order}
|
||||
|
||||
|
||||
def _router_with_a_mixed_web_search_group(
|
||||
enable_tag_filtering: bool, orders: tuple[int | None, int | None] = (None, None)
|
||||
) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-x",
|
||||
"api_key": "fake-key",
|
||||
"tags": ["cheap"],
|
||||
**_order_param(orders[0]),
|
||||
},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-y",
|
||||
"api_key": "fake-key",
|
||||
"tags": ["premium"],
|
||||
**_order_param(orders[1]),
|
||||
},
|
||||
"model_info": {"id": "search"},
|
||||
},
|
||||
],
|
||||
enable_tag_filtering=enable_tag_filtering,
|
||||
)
|
||||
|
||||
|
||||
def _cool_down(router: Router, model_ids: tuple[str, ...]) -> None:
|
||||
for model_id in model_ids:
|
||||
router.cooldown_cache.add_deployment_to_cooldown(
|
||||
model_id=model_id, original_exception=Exception("rate limited"), exception_status=429, cooldown_time=300
|
||||
)
|
||||
|
||||
|
||||
_LATER_FILTER_CASES: Final = pytest.mark.parametrize(
|
||||
("orders", "cooled_down", "request_kwargs", "expected_id"),
|
||||
[
|
||||
((None, None), (), _ANTHROPIC_WEB_SEARCH_REQUEST, "search"),
|
||||
((None, None), (), {"web_search_options": {}}, "search"),
|
||||
((1, 2), (), _ANTHROPIC_WEB_SEARCH_REQUEST, "no-search"),
|
||||
((1, 2), (), {"web_search_options": {}}, "no-search"),
|
||||
((None, None), ("search",), _ANTHROPIC_WEB_SEARCH_REQUEST, "no-search"),
|
||||
((None, None), ("search",), {"web_search_options": {}}, "no-search"),
|
||||
((None, None), (), {**_ANTHROPIC_WEB_SEARCH_REQUEST, "_excluded_deployment_ids": ["search"]}, "no-search"),
|
||||
((None, None), (), {**_ANTHROPIC_WEB_SEARCH_REQUEST, "_retry_skipped_deployment_ids": ("search",)}, "no-search"),
|
||||
],
|
||||
ids=[
|
||||
"same-tier",
|
||||
"same-tier-options",
|
||||
"lower-order-tier",
|
||||
"lower-order-tier-options",
|
||||
"cooldown",
|
||||
"cooldown-options",
|
||||
"weighted-failover",
|
||||
"retry-skip",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_LATER_FILTER_CASES
|
||||
async def test_async_pick_prefers_web_search_until_later_filters_leave_none_that_can(
|
||||
orders: tuple[int | None, int | None],
|
||||
cooled_down: tuple[str, ...],
|
||||
request_kwargs: dict[str, object],
|
||||
expected_id: str,
|
||||
) -> None:
|
||||
router: Final = _router_with_a_mixed_web_search_group(False, orders)
|
||||
_cool_down(router, cooled_down)
|
||||
deployment: Final = await router.async_get_available_deployment(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(request_kwargs)
|
||||
)
|
||||
assert deployment["model_info"]["id"] == expected_id
|
||||
|
||||
|
||||
@_LATER_FILTER_CASES
|
||||
def test_sync_pick_prefers_web_search_until_later_filters_leave_none_that_can(
|
||||
orders: tuple[int | None, int | None],
|
||||
cooled_down: tuple[str, ...],
|
||||
request_kwargs: dict[str, object],
|
||||
expected_id: str,
|
||||
) -> None:
|
||||
router: Final = _router_with_a_mixed_web_search_group(False, orders)
|
||||
_cool_down(router, cooled_down)
|
||||
deployment: Final = router.get_available_deployment(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(request_kwargs)
|
||||
)
|
||||
assert deployment["model_info"]["id"] == expected_id
|
||||
|
||||
|
||||
_WEB_SEARCH_REQUESTS: Final = pytest.mark.parametrize(
|
||||
"web_search_request",
|
||||
[_ANTHROPIC_WEB_SEARCH_REQUEST, {"web_search_options": {}}],
|
||||
ids=["web-search-tool", "web-search-options"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_WEB_SEARCH_REQUESTS
|
||||
async def test_order_fallback_answers_a_web_search_request_from_the_next_tier_when_the_capable_one_fails(
|
||||
web_search_request: dict[str, object],
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "ordered",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-x",
|
||||
"api_key": "fake-key",
|
||||
"mock_response": Exception("connection error"),
|
||||
"order": 1,
|
||||
},
|
||||
"model_info": {"id": "search"},
|
||||
},
|
||||
{
|
||||
"model_name": "ordered",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "order": 2},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
response: Final = await router.acompletion(
|
||||
model="ordered", messages=[{"role": "user", "content": "hi"}], **web_search_request
|
||||
)
|
||||
assert response._hidden_params["model_id"] == "no-search"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_WEB_SEARCH_REQUESTS
|
||||
async def test_web_search_request_prefers_the_capable_deployment_inside_its_order_tier(
|
||||
web_search_request: dict[str, object],
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "ordered",
|
||||
"litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "order": 1},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
{
|
||||
"model_name": "ordered",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "order": 1},
|
||||
"model_info": {"id": "search"},
|
||||
},
|
||||
{
|
||||
"model_name": "ordered",
|
||||
"litellm_params": {"model": "openai/gpt-z", "api_key": "fake-key", "order": 2},
|
||||
"model_info": {"id": "later-no-search", "supports_web_search": False},
|
||||
},
|
||||
],
|
||||
)
|
||||
picks: Final = {
|
||||
(
|
||||
await router.async_get_available_deployment(
|
||||
model="ordered", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(web_search_request)
|
||||
)
|
||||
)["model_info"]["id"]
|
||||
for _ in range(20)
|
||||
}
|
||||
assert picks == {"search"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"])
|
||||
async def test_web_search_preference_gives_way_once_the_capable_deployment_failed_this_request(
|
||||
metadata_field: str,
|
||||
) -> None:
|
||||
router: Final = _router_with_a_mixed_web_search_group(False)
|
||||
picks: Final = {
|
||||
(
|
||||
await router.async_get_available_deployment(
|
||||
model="mixed",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={
|
||||
**_ANTHROPIC_WEB_SEARCH_REQUEST,
|
||||
metadata_field: {"previous_models": ({"deployment_id": "search", "exception_type": "Timeout"},)},
|
||||
},
|
||||
)
|
||||
)["model_info"]["id"]
|
||||
for _ in range(40)
|
||||
}
|
||||
assert picks == {"search", "no-search"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_WEB_SEARCH_REQUESTS
|
||||
@pytest.mark.parametrize(
|
||||
"caller_credentials", [{}, {"api_key": "caller-key"}], ids=["configured-credentials", "caller-credentials"]
|
||||
)
|
||||
async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches_the_other_one(
|
||||
web_search_request: dict[str, object],
|
||||
caller_credentials: dict[str, str],
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-x",
|
||||
"api_key": "fake-key",
|
||||
"mock_response": Exception("connection error"),
|
||||
"weight": 1,
|
||||
},
|
||||
"model_info": {"id": "search"},
|
||||
},
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-y",
|
||||
"api_key": "fake-key",
|
||||
"mock_response": "from no-search",
|
||||
"weight": 10**6,
|
||||
},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
],
|
||||
num_retries=1,
|
||||
disable_cooldowns=True,
|
||||
)
|
||||
response: Final = await router.acompletion(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request, **caller_credentials
|
||||
)
|
||||
assert response.choices[0].message.content == "from no-search"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"])
|
||||
@pytest.mark.parametrize(("request_retry_count", "flagged_reachable"), [(1, False), (2, True)], ids=["kept", "trimmed"])
|
||||
async def test_web_search_preference_steps_aside_only_once_the_retry_history_was_trimmed(
|
||||
metadata_field: str, request_retry_count: int, flagged_reachable: bool
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
*(
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi"},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
for deployment_id in ("search-a", "search-b")
|
||||
),
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-y",
|
||||
"api_key": "fake-key",
|
||||
"mock_response": "hi",
|
||||
"weight": 10**6,
|
||||
},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
],
|
||||
)
|
||||
picks: Final = {
|
||||
(
|
||||
await router.async_get_available_deployment(
|
||||
model="mixed",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={
|
||||
**_ANTHROPIC_WEB_SEARCH_REQUEST,
|
||||
metadata_field: {
|
||||
"previous_models": ({"deployment_id": "search-a", "exception_type": "Timeout"},),
|
||||
"request_retry_count": request_retry_count,
|
||||
},
|
||||
},
|
||||
)
|
||||
)["model_info"]["id"]
|
||||
for _ in range(20)
|
||||
}
|
||||
assert ("no-search" in picks) is flagged_reachable
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retries_past_the_kept_retry_history_reach_the_deployment_that_cannot_search() -> None:
|
||||
capable_deployments: Final = [
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-x",
|
||||
"api_key": "fake-key",
|
||||
"mock_response": Exception("connection error"),
|
||||
},
|
||||
"model_info": {"id": f"search-{index}"},
|
||||
}
|
||||
for index in range(5)
|
||||
]
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
*capable_deployments,
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-y",
|
||||
"api_key": "fake-key",
|
||||
"mock_response": "from no-search",
|
||||
"weight": 10**6,
|
||||
},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
],
|
||||
num_retries=5,
|
||||
disable_cooldowns=True,
|
||||
)
|
||||
response: Final = await router.acompletion(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST
|
||||
)
|
||||
assert response.choices[0].message.content == "from no-search"
|
||||
|
||||
|
||||
def _rate_limited_web_search_group(routing_strategy: str) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi", "rpm": 1},
|
||||
"model_info": {"id": "search"},
|
||||
},
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "rpm": 1000},
|
||||
"model_info": {"id": "no-search", "supports_web_search": False},
|
||||
},
|
||||
],
|
||||
routing_strategy=routing_strategy,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("routing_strategy", ["usage-based-routing-v2", "cost-based-routing"])
|
||||
async def test_web_search_request_goes_to_the_other_deployment_once_the_capable_one_is_at_its_rpm_limit(
|
||||
routing_strategy: str,
|
||||
) -> None:
|
||||
router: Final = _rate_limited_web_search_group(routing_strategy)
|
||||
served_by: Final = [] # mutable-ok: collects one id per sequential request
|
||||
for _ in range(3):
|
||||
response = await router.acompletion(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST
|
||||
)
|
||||
served_by.append(response._hidden_params["model_id"])
|
||||
await asyncio.sleep(0.2)
|
||||
assert served_by == ["search", "no-search", "no-search"]
|
||||
|
||||
|
||||
def test_sync_web_search_request_goes_to_the_other_deployment_once_the_capable_one_is_at_its_rpm_limit() -> None:
|
||||
router: Final = _rate_limited_web_search_group("usage-based-routing-v2")
|
||||
served_by: Final = tuple(
|
||||
router.completion(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST
|
||||
)._hidden_params["model_id"]
|
||||
for _ in range(3)
|
||||
)
|
||||
assert served_by == ("search", "no-search", "no-search")
|
||||
|
||||
|
||||
def test_sync_pick_falls_back_to_every_deployment_when_the_strategy_finds_none_that_can_search() -> None:
|
||||
deployments: Final = [
|
||||
{"model_info": {"id": "search"}},
|
||||
{"model_info": {"id": "no-search", "supports_web_search": False}},
|
||||
]
|
||||
picked: Final = pick_preferring_web_search(
|
||||
lambda candidates: next((d for d in candidates if d["model_info"]["id"] == "no-search"), None),
|
||||
deployments,
|
||||
_ANTHROPIC_WEB_SEARCH_REQUEST,
|
||||
)
|
||||
assert picked == deployments[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("search_order", "no_search_order"),
|
||||
[([1], [1]), ({"tier": 1}, None)],
|
||||
ids=["list-order", "dict-order"],
|
||||
)
|
||||
async def test_web_search_request_routes_on_a_group_whose_order_is_not_a_number(
|
||||
search_order: object, no_search_order: object
|
||||
) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key"},
|
||||
"model_info": {"id": "search", "order": search_order},
|
||||
},
|
||||
{
|
||||
"model_name": "mixed",
|
||||
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key"},
|
||||
"model_info": {
|
||||
"id": "no-search",
|
||||
"supports_web_search": False,
|
||||
**({} if no_search_order is None else {"order": no_search_order}),
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
deployment: Final = await router.async_get_available_deployment(
|
||||
model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST)
|
||||
)
|
||||
assert deployment["model_info"]["id"] == "search"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_routing_to_a_deployment_without_web_search_keeps_it() -> None:
|
||||
deployment: Final = await _router_with_a_mixed_web_search_group(True).async_get_available_deployment(
|
||||
model="mixed",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={"web_search_options": {}, "metadata": {"tags": ["cheap"]}},
|
||||
)
|
||||
assert deployment["model_info"]["id"] == "no-search"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_web_search_still_finds_no_deployment_while_the_only_one_that_can_search_cools_down() -> None:
|
||||
router: Final = _router_with_a_mixed_web_search_group(False)
|
||||
_cool_down(router, ("search",))
|
||||
with pytest.raises(litellm.RouterRateLimitError):
|
||||
await router.async_get_available_deployment(
|
||||
model="mixed",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={"tools": [{"type": "web_search"}]},
|
||||
)
|
||||
|
||||
|
||||
def test_invalidate_model_group_info_cache():
|
||||
"""Test that _invalidate_model_group_info_cache clears the LRU cache."""
|
||||
router = Router(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue