From 9b5da78531c0f48f977690b946be7a7f2ad3c005 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 16 Oct 2026 21:38:28 +0000 Subject: [PATCH 01/11] fix(router): drop supports_web_search: false deployments for Anthropic and Gemini web search tools Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router_utils/common_utils.py | 31 +++++++++++++---- .../test_router_utils_common_utils.py | 33 +++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index c4e4d9ccc0b..316d76e67b1 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -30,6 +30,17 @@ _V = TypeVar("_V") MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation" ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG}) _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) +_OBJECT_TUPLE_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) +_GEMINI_SEARCH_TOOL_KEYS: Final = frozenset( + { + "googleSearch", + "google_search", + "googleSearchRetrieval", + "google_search_retrieval", + "enterpriseWebSearch", + "enterprise_web_search", + } +) def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]: @@ -329,6 +340,16 @@ def _deployment_supports_web_search(deployment: dict) -> bool: return True +def _is_web_search_tool(tool: object) -> bool: + if not isinstance(tool, Mapping): + return False + tool_mapping: Final = _OBJECT_MAPPING_ADAPTER.validate_python(tool) + tool_type: Final = tool_mapping.get("type") + if isinstance(tool_type, str) and (tool_type == "web_search" or tool_type.startswith("web_search_")): + return True + return any(key in _GEMINI_SEARCH_TOOL_KEYS for key in tool_mapping if isinstance(key, str)) + + def filter_web_search_deployments( healthy_deployments: list[dict] | dict, request_kwargs: dict | None = None, @@ -343,13 +364,9 @@ def filter_web_search_deployments( if isinstance(healthy_deployments, dict): return healthy_deployments - is_web_search_request = False - tools: Final = request_kwargs.get("tools") or [] - for tool in tools: - # These are the two websearch tools for OpenAI / Azure. - if tool.get("type") == "web_search" or tool.get("type") == "web_search_preview": - is_web_search_request = True - break + tools_value: Final[object] = request_kwargs.get("tools") or [] + tools: Final = _OBJECT_TUPLE_ADAPTER.validate_python(tools_value) + is_web_search_request: Final = any(_is_web_search_tool(tool) for tool in tools) if not is_web_search_request: return healthy_deployments diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index af8fca72252..0ee7b81ec19 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -295,6 +295,39 @@ class TestFilterWebSearchDeployments: result_ids = [d["model_info"]["id"] for d in result] assert "deployment-3" not in result_ids + @pytest.mark.parametrize( + "tool", + [ + {"type": "web_search_20250305", "name": "web_search"}, + {"type": "web_search_20260209", "name": "web_search"}, + {"type": "web_search_preview_2025_03_11"}, + {"googleSearch": {}}, + {"google_search": {}}, + {"googleSearchRetrieval": {}}, + {"enterpriseWebSearch": {}}, + ], + ) + def test_native_web_search_tools_filter_unsupported( + self, sample_deployments: list[dict[str, object]], tool: dict[str, object] + ) -> None: + result = filter_web_search_deployments(sample_deployments, {"tools": [tool]}) + assert result == sample_deployments[:2] + + @pytest.mark.parametrize( + "tool", + [ + {"type": "web_fetch_20250910", "name": "web_fetch"}, + {"type": "function", "function": {"name": "web_search"}}, + {"type": "function", "name": "googleSearch"}, + {"urlContext": {}}, + ], + ) + def test_non_web_search_tools_preserve_deployments( + self, sample_deployments: list[dict[str, object]], tool: dict[str, object] + ) -> None: + result = filter_web_search_deployments(sample_deployments, {"tools": [tool]}) + assert result == sample_deployments + def test_web_search_with_other_tools(self, sample_deployments): """Web search filtering works when mixed with other tools""" request_kwargs = { From 07546d5189301f8e0225558b89a8d0193c22eb87 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 16 Oct 2026 21:54:16 +0000 Subject: [PATCH 02/11] test(router): annotate new web search filter test locals with Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/router_utils/test_router_utils_common_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 0ee7b81ec19..6c52d43dfba 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -310,7 +310,7 @@ class TestFilterWebSearchDeployments: def test_native_web_search_tools_filter_unsupported( self, sample_deployments: list[dict[str, object]], tool: dict[str, object] ) -> None: - result = filter_web_search_deployments(sample_deployments, {"tools": [tool]}) + result: Final = filter_web_search_deployments(sample_deployments, {"tools": [tool]}) assert result == sample_deployments[:2] @pytest.mark.parametrize( @@ -325,7 +325,7 @@ class TestFilterWebSearchDeployments: def test_non_web_search_tools_preserve_deployments( self, sample_deployments: list[dict[str, object]], tool: dict[str, object] ) -> None: - result = filter_web_search_deployments(sample_deployments, {"tools": [tool]}) + result: Final = filter_web_search_deployments(sample_deployments, {"tools": [tool]}) assert result == sample_deployments def test_web_search_with_other_tools(self, sample_deployments): From 85a070a39382f322d9bd228a33cbf929ebb43d33 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:12:37 -0700 Subject: [PATCH 03/11] fix(router): filter web search deployments on the sync pick, skip null Gemini search keys, and keep every deployment when none supports search --- litellm/router.py | 5 +- litellm/router_utils/common_utils.py | 34 ++++--- .../test_router_utils_common_utils.py | 93 +++++++++++++++++-- 3 files changed, 112 insertions(+), 20 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index fc888495e99..5df73364782 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -16416,7 +16416,10 @@ class Router: ) healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows self._without_deployments_that_cannot_serve_route( - healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + filter_web_search_deployments( + healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + request_kwargs, + ), model, ) ) diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 28279910876..ae381768023 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -13,9 +13,11 @@ import litellm from litellm._logging import verbose_logger, verbose_router_logger from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure +from litellm.rust_bridge.public_call import optional_sequence from litellm.types.router import ( CredentialLiteLLMParams, Deployment, @@ -35,7 +37,6 @@ _V = TypeVar("_V") MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation" ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG}) _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) -_OBJECT_TUPLE_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) _GEMINI_SEARCH_TOOL_KEYS: Final = frozenset( { "googleSearch", @@ -46,6 +47,7 @@ _GEMINI_SEARCH_TOOL_KEYS: Final = frozenset( "enterprise_web_search", } ) +_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "google_search"}) _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) @@ -439,13 +441,19 @@ def _deployment_supports_web_search(deployment: dict) -> bool: def _is_web_search_tool(tool: object) -> bool: - if not isinstance(tool, Mapping): + tool_fields: Final = as_str_mapping(tool) + if tool_fields is None: return False - tool_mapping: Final = _OBJECT_MAPPING_ADAPTER.validate_python(tool) - tool_type: Final = tool_mapping.get("type") - if isinstance(tool_type, str) and (tool_type == "web_search" or tool_type.startswith("web_search_")): + tool_type: Final = tool_fields.get("type") + if isinstance(tool_type, str) and (tool_type in _WEB_SEARCH_TOOL_TYPES or tool_type.startswith("web_search_")): return True - return any(key in _GEMINI_SEARCH_TOOL_KEYS for key in tool_mapping if isinstance(key, str)) + return not _GEMINI_SEARCH_TOOL_KEYS.isdisjoint(tool_fields) and any( + tool_fields[key] is not None for key in _GEMINI_SEARCH_TOOL_KEYS.intersection(tool_fields) + ) + + +def _is_web_search_request(tools: object, web_search_options: object) -> bool: + return web_search_options is not None or any(_is_web_search_tool(tool) for tool in optional_sequence(tools) or ()) def filter_web_search_deployments( @@ -463,17 +471,17 @@ def filter_web_search_deployments( return healthy_deployments tools: Final[object] = request_kwargs.get("tools") - is_web_search_request: Final = isinstance(tools, (list, tuple)) and any( - _is_web_search_tool(tool) for tool in _OBJECT_TUPLE_ADAPTER.validate_python(tools) - ) - if not is_web_search_request: + web_search_options: Final[object] = request_kwargs.get("web_search_options") + if not _is_web_search_request(tools, web_search_options): return healthy_deployments # Filter out deployments that don't support web search final_deployments: Final = [d for d in healthy_deployments if _deployment_supports_web_search(d)] - if len(healthy_deployments) > 0 and len(final_deployments) == 0: - verbose_logger.warning("No deployments support web search for request") - return final_deployments + if final_deployments: + return final_deployments + if healthy_deployments: + verbose_logger.warning("No deployment supports web search for this request, so every deployment stays eligible") + return healthy_deployments # Credential params that only one provider family reads, paired with the providers diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index f9e710cff46..269c9cd9bd0 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -318,7 +318,10 @@ class TestFilterWebSearchDeployments: {"googleSearch": {}}, {"google_search": {}}, {"googleSearchRetrieval": {}}, + {"google_search_retrieval": {}}, {"enterpriseWebSearch": {}}, + {"enterprise_web_search": {}}, + {"type": "google_search"}, ], ) def test_native_web_search_tools_filter_unsupported( @@ -334,6 +337,8 @@ class TestFilterWebSearchDeployments: {"type": "function", "function": {"name": "web_search"}}, {"type": "function", "name": "googleSearch"}, {"urlContext": {}}, + {"googleSearch": None}, + {"google_search": None, "urlContext": {}}, ], ) def test_non_web_search_tools_preserve_deployments( @@ -365,15 +370,27 @@ class TestFilterWebSearchDeployments: result = filter_web_search_deployments(deployments, request_kwargs) assert len(result) == 2 - def test_no_deployments_support_web_search(self): - """When no deployments support web search, all are filtered out""" - deployments = [ + def test_no_deployment_supporting_web_search_keeps_every_deployment(self) -> None: + deployments: Final = [ {"model_info": {"id": "d1", "supports_web_search": False}}, {"model_info": {"id": "d2", "supports_web_search": False}}, ] - request_kwargs = {"tools": [{"type": "web_search"}]} - result = filter_web_search_deployments(deployments, request_kwargs) - assert len(result) == 0 + result: Final = filter_web_search_deployments(deployments, {"tools": [{"type": "web_search"}]}) + assert result == deployments + + def test_a_tool_that_is_not_an_object_is_no_web_search_request( + self, sample_deployments: list[dict[str, object]] + ) -> None: + result: Final = filter_web_search_deployments(sample_deployments, {"tools": ["web_search", None]}) + assert result == sample_deployments + + def test_web_search_options_filter_unsupported(self, sample_deployments: list[dict[str, object]]) -> None: + result: Final = filter_web_search_deployments(sample_deployments, {"web_search_options": {}}) + assert result == sample_deployments[:2] + + def test_null_web_search_options_preserve_deployments(self, sample_deployments: list[dict[str, object]]) -> None: + result: Final = filter_web_search_deployments(sample_deployments, {"web_search_options": None}) + assert result == sample_deployments def test_missing_config_defaults_to_supported(self): """Deployments without supports_web_search config default to True""" @@ -408,6 +425,70 @@ class TestFilterWebSearchDeployments: assert result == deployment + +def _router_with_a_deployment_that_disables_web_search(routing_strategy: str) -> Router: + return Router( + model_list=[ + { + "model_name": "search-model", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key"}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + { + "model_name": "search-model", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key"}, + "model_info": {"id": "search"}, + }, + ], + routing_strategy=routing_strategy, + ) + + +_ANTHROPIC_WEB_SEARCH_REQUEST: Final = {"tools": [{"type": "web_search_20250305", "name": "web_search"}]} + + +@pytest.mark.parametrize("routing_strategy", ["simple-shuffle", "usage-based-routing"]) +def test_sync_pick_skips_deployments_without_web_search(routing_strategy: str) -> None: + router: Final = _router_with_a_deployment_that_disables_web_search(routing_strategy) + picks: Final = { + router.get_available_deployment(model="search-model", request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST))[ + "model_info" + ]["id"] + for _ in range(20) + } + assert picks == {"search"} + + +@pytest.mark.asyncio +async def test_web_search_request_still_routes_when_every_deployment_disables_web_search() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "search-model", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key"}, + "model_info": {"id": "no-search", "supports_web_search": False}, + } + ] + ) + deployment: Final = await router.async_get_available_deployment( + model="search-model", request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST) + ) + assert deployment["model_info"]["id"] == "no-search" + + +@pytest.mark.asyncio +async def test_legacy_usage_based_async_pick_skips_deployments_without_web_search() -> None: + router: Final = _router_with_a_deployment_that_disables_web_search("usage-based-routing") + picks: Final = { + ( + await router.async_get_available_deployment( + model="search-model", request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST) + ) + )["model_info"]["id"] + for _ in range(20) + } + assert picks == {"search"} + def test_invalidate_model_group_info_cache(): """Test that _invalidate_model_group_info_cache clears the LRU cache.""" router = Router( From 7025ed6b89eade1fcd0ba21f7c99f7412b6a0de6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:17:17 -0700 Subject: [PATCH 04/11] fix(router): keep the empty deployment list when no deployment supports web search --- litellm/router_utils/common_utils.py | 8 ++--- .../test_router_utils_common_utils.py | 36 +++++++++++++------ 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index ae381768023..4ae99518685 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -477,11 +477,9 @@ def filter_web_search_deployments( # Filter out deployments that don't support web search final_deployments: Final = [d for d in healthy_deployments if _deployment_supports_web_search(d)] - if final_deployments: - return final_deployments - if healthy_deployments: - verbose_logger.warning("No deployment supports web search for this request, so every deployment stays eligible") - return healthy_deployments + if healthy_deployments and not final_deployments: + verbose_logger.warning("No deployments support web search for request") + return final_deployments # Credential params that only one provider family reads, paired with the providers diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 269c9cd9bd0..568cb4ee9e1 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -370,13 +370,21 @@ class TestFilterWebSearchDeployments: result = filter_web_search_deployments(deployments, request_kwargs) assert len(result) == 2 - def test_no_deployment_supporting_web_search_keeps_every_deployment(self) -> None: + @pytest.mark.parametrize( + "request_kwargs", + [ + {"tools": [{"type": "web_search"}]}, + {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}, + {"tools": [{"googleSearch": {}}]}, + {"web_search_options": {}}, + ], + ) + def test_no_deployment_supporting_web_search_leaves_none(self, request_kwargs: dict[str, object]) -> None: deployments: Final = [ {"model_info": {"id": "d1", "supports_web_search": False}}, {"model_info": {"id": "d2", "supports_web_search": False}}, ] - result: Final = filter_web_search_deployments(deployments, {"tools": [{"type": "web_search"}]}) - assert result == deployments + assert filter_web_search_deployments(deployments, request_kwargs) == [] def test_a_tool_that_is_not_an_object_is_no_web_search_request( self, sample_deployments: list[dict[str, object]] @@ -460,20 +468,26 @@ def test_sync_pick_skips_deployments_without_web_search(routing_strategy: str) - @pytest.mark.asyncio -async def test_web_search_request_still_routes_when_every_deployment_disables_web_search() -> None: +async def test_web_search_request_to_a_group_without_web_search_falls_back_to_a_group_with_it() -> None: router: Final = Router( model_list=[ { - "model_name": "search-model", - "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key"}, + "model_name": "flagged-model", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "flagged"}, "model_info": {"id": "no-search", "supports_web_search": False}, - } - ] + }, + { + "model_name": "search-model", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "searched"}, + "model_info": {"id": "search"}, + }, + ], + fallbacks=[{"flagged-model": ["search-model"]}], ) - deployment: Final = await router.async_get_available_deployment( - model="search-model", request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST) + response: Final = await router.acompletion( + model="flagged-model", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST ) - assert deployment["model_info"]["id"] == "no-search" + assert response.choices[0].message.content == "searched" @pytest.mark.asyncio From 3271825b1a9cdaf73f8e8f40f0a222a203a4ed22 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:09:03 -0700 Subject: [PATCH 05/11] fix(router): keep main's routing for web search when no deployment can search --- litellm/router.py | 1 + litellm/router_utils/common_utils.py | 29 ++++++-- .../test_router_utils_common_utils.py | 69 ++++++++++++++----- 3 files changed, 76 insertions(+), 23 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5df73364782..88032b2719c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -16419,6 +16419,7 @@ class Router: filter_web_search_deployments( healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type request_kwargs, + empty_for_openai_web_search=False, ), model, ) diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 4ae99518685..f473933e7e2 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -48,6 +48,7 @@ _GEMINI_SEARCH_TOOL_KEYS: Final = frozenset( } ) _WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "google_search"}) +_OPENAI_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"}) _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) @@ -456,30 +457,48 @@ def _is_web_search_request(tools: object, web_search_options: object) -> bool: return web_search_options is not None or any(_is_web_search_tool(tool) for tool in optional_sequence(tools) or ()) +def _is_openai_web_search_tool(tool: object) -> bool: + tool_fields: Final = as_str_mapping(tool) + return ( + tool_fields is not None + and isinstance(tool_type := tool_fields.get("type"), str) + and tool_type in _OPENAI_WEB_SEARCH_TOOL_TYPES + ) + + +def _has_openai_web_search_tool(tools: object) -> bool: + tool_list: Final = optional_sequence(tools) + return tool_list is not None and any(_is_openai_web_search_tool(tool) for tool in tool_list) + + def filter_web_search_deployments( healthy_deployments: list[dict] | dict, request_kwargs: dict | None = None, + *, + empty_for_openai_web_search: bool = True, ) -> list[dict] | dict: """ If the request is websearch, filter out deployments that don't support web search """ - if request_kwargs is None: + request_fields: Final = as_str_mapping(request_kwargs) + if request_fields is None: return healthy_deployments # When a specific deployment was already chosen, it's returned as a dict # rather than a list - nothing to filter, just pass through if isinstance(healthy_deployments, dict): return healthy_deployments - tools: Final[object] = request_kwargs.get("tools") - web_search_options: Final[object] = request_kwargs.get("web_search_options") + tools: Final = request_fields.get("tools") + web_search_options: Final = request_fields.get("web_search_options") if not _is_web_search_request(tools, web_search_options): return healthy_deployments # Filter out deployments that don't support web search final_deployments: Final = [d for d in healthy_deployments if _deployment_supports_web_search(d)] - if healthy_deployments and not final_deployments: + empties_group: Final = empty_for_openai_web_search and _has_openai_web_search_tool(tools) + if healthy_deployments and not final_deployments and empties_group: verbose_logger.warning("No deployments support web search for request") - return final_deployments + return final_deployments if final_deployments or empties_group else healthy_deployments # Credential params that only one provider family reads, paired with the providers diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 568cb4ee9e1..a9c9cf629a9 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -370,21 +370,31 @@ class TestFilterWebSearchDeployments: result = filter_web_search_deployments(deployments, request_kwargs) assert len(result) == 2 - @pytest.mark.parametrize( - "request_kwargs", - [ - {"tools": [{"type": "web_search"}]}, - {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}, - {"tools": [{"googleSearch": {}}]}, - {"web_search_options": {}}, - ], - ) - def test_no_deployment_supporting_web_search_leaves_none(self, request_kwargs: dict[str, object]) -> None: + @pytest.mark.parametrize("tool_type", ["web_search", "web_search_preview"]) + def test_no_deployment_supporting_openai_web_search_leaves_none(self, tool_type: str) -> None: deployments: Final = [ {"model_info": {"id": "d1", "supports_web_search": False}}, {"model_info": {"id": "d2", "supports_web_search": False}}, ] - assert filter_web_search_deployments(deployments, request_kwargs) == [] + assert filter_web_search_deployments(deployments, {"tools": [{"type": tool_type}]}) == [] + + @pytest.mark.parametrize( + "request_kwargs", + [ + {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}, + {"tools": [{"type": "web_search_2025_08_26"}]}, + {"tools": [{"googleSearch": {}}]}, + {"web_search_options": {}}, + ], + ) + def test_no_deployment_supporting_other_web_search_keeps_every_deployment( + self, request_kwargs: dict[str, object] + ) -> None: + deployments: Final = [ + {"model_info": {"id": "d1", "supports_web_search": False}}, + {"model_info": {"id": "d2", "supports_web_search": False}}, + ] + assert filter_web_search_deployments(deployments, request_kwargs) == deployments def test_a_tool_that_is_not_an_object_is_no_web_search_request( self, sample_deployments: list[dict[str, object]] @@ -433,7 +443,6 @@ class TestFilterWebSearchDeployments: assert result == deployment - def _router_with_a_deployment_that_disables_web_search(routing_strategy: str) -> Router: return Router( model_list=[ @@ -467,9 +476,8 @@ def test_sync_pick_skips_deployments_without_web_search(routing_strategy: str) - assert picks == {"search"} -@pytest.mark.asyncio -async def test_web_search_request_to_a_group_without_web_search_falls_back_to_a_group_with_it() -> None: - router: Final = Router( +def _router_with_a_group_that_disables_web_search(routing_strategy: str = "simple-shuffle") -> Router: + return Router( model_list=[ { "model_name": "flagged-model", @@ -483,11 +491,35 @@ async def test_web_search_request_to_a_group_without_web_search_falls_back_to_a_ }, ], fallbacks=[{"flagged-model": ["search-model"]}], + routing_strategy=routing_strategy, ) - response: Final = await router.acompletion( - model="flagged-model", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tools", "expected_content"), + [ + ([{"type": "web_search"}], "searched"), + ([{"type": "web_search_20250305", "name": "web_search"}], "flagged"), + ([{"googleSearch": dict[str, object]()}], "flagged"), + ], +) +async def test_web_search_to_a_group_without_web_search_routes_like_main( + tools: list[dict[str, object]], expected_content: str +) -> None: + response: Final = await _router_with_a_group_that_disables_web_search().acompletion( + model="flagged-model", messages=[{"role": "user", "content": "hi"}], tools=tools ) - assert response.choices[0].message.content == "searched" + assert response.choices[0].message.content == expected_content + + +@pytest.mark.parametrize("routing_strategy", ["simple-shuffle", "usage-based-routing"]) +def test_sync_pick_keeps_a_group_without_web_search_for_openai_web_search(routing_strategy: str) -> None: + router: Final = _router_with_a_group_that_disables_web_search(routing_strategy) + deployment: Final = router.get_available_deployment( + model="flagged-model", request_kwargs={"tools": [{"type": "web_search"}]} + ) + assert deployment["model_info"]["id"] == "no-search" @pytest.mark.asyncio @@ -503,6 +535,7 @@ async def test_legacy_usage_based_async_pick_skips_deployments_without_web_searc } assert picks == {"search"} + def test_invalidate_model_group_info_cache(): """Test that _invalidate_model_group_info_cache clears the LRU cache.""" router = Router( From 585ccf378ee5e5a133558b65dc5c0727d95ce982 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:26:17 -0700 Subject: [PATCH 06/11] fix(router): prefer web search deployments only after the other router filters run --- litellm/router.py | 29 ++--- litellm/router_utils/common_utils.py | 61 ++++++---- .../test_router_utils_common_utils.py | 106 ++++++++++++++++-- 3 files changed, 149 insertions(+), 47 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 88032b2719c..33f53d9b852 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -245,6 +245,7 @@ from litellm.router_utils.common_utils import ( format_no_fallback_group_message, get_request_team_id, is_proxy_admin_request, + prefer_web_search_deployments, provider_for_generic_call, request_user_api_key_auth, resolve_model_group_alias, @@ -15245,10 +15246,13 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = self._without_declared_text_only_deployments( - cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts - model, - messages, + batch_candidates: Final = prefer_web_search_deployments( + self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict + ), cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( @@ -16416,11 +16420,7 @@ class Router: ) healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows self._without_deployments_that_cannot_serve_route( - filter_web_search_deployments( - healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type - request_kwargs, - empty_for_openai_web_search=False, - ), + healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type model, ) ) @@ -16481,10 +16481,13 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = self._without_declared_text_only_deployments( - cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts - model, - messages, + batch_candidates: Final = prefer_web_search_deployments( + self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict + ), cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index f473933e7e2..052bbdccb05 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -457,48 +457,61 @@ def _is_web_search_request(tools: object, web_search_options: object) -> bool: return web_search_options is not None or any(_is_web_search_tool(tool) for tool in optional_sequence(tools) or ()) -def _is_openai_web_search_tool(tool: object) -> bool: - tool_fields: Final = as_str_mapping(tool) - return ( - tool_fields is not None - and isinstance(tool_type := tool_fields.get("type"), str) - and tool_type in _OPENAI_WEB_SEARCH_TOOL_TYPES - ) - - -def _has_openai_web_search_tool(tools: object) -> bool: - tool_list: Final = optional_sequence(tools) - return tool_list is not None and any(_is_openai_web_search_tool(tool) for tool in tool_list) - - def filter_web_search_deployments( healthy_deployments: list[dict] | dict, request_kwargs: dict | None = None, - *, - empty_for_openai_web_search: bool = True, ) -> list[dict] | dict: """ If the request is websearch, filter out deployments that don't support web search """ - request_fields: Final = as_str_mapping(request_kwargs) - if request_fields is None: + if request_kwargs is None: return healthy_deployments # When a specific deployment was already chosen, it's returned as a dict # rather than a list - nothing to filter, just pass through if isinstance(healthy_deployments, dict): return healthy_deployments - tools: Final = request_fields.get("tools") - web_search_options: Final = request_fields.get("web_search_options") - if not _is_web_search_request(tools, web_search_options): + tools: Final[object] = request_kwargs.get("tools") + is_web_search_request: Final = isinstance(tools, (list, tuple)) and any( + isinstance(tool, dict) + and isinstance(tool_type := tool.get("type"), str) + and tool_type in _OPENAI_WEB_SEARCH_TOOL_TYPES + for tool in tools + ) + if not is_web_search_request: return healthy_deployments # Filter out deployments that don't support web search final_deployments: Final = [d for d in healthy_deployments if _deployment_supports_web_search(d)] - empties_group: Final = empty_for_openai_web_search and _has_openai_web_search_tool(tools) - if healthy_deployments and not final_deployments and empties_group: + if len(healthy_deployments) > 0 and len(final_deployments) == 0: verbose_logger.warning("No deployments support web search for request") - return final_deployments if final_deployments or empties_group else healthy_deployments + return final_deployments + + +def _deployment_id(deployment: Mapping[str, object]) -> object: + model_info: Final = as_str_mapping(deployment.get("model_info")) + return None if model_info is None else model_info.get("id") + + +def prefer_web_search_deployments( + deployments: list[dict[str, object]], # mutable-ok: router deployment-list contract + request_kwargs: Mapping[str, object] | None, +) -> list[dict[str, object]]: # mutable-ok: returns a mutable deployment list + if request_kwargs is None or not _is_web_search_request( + request_kwargs.get("tools"), request_kwargs.get("web_search_options") + ): + return deployments + tried: Final = frozenset( + ( + *as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")), + *as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")), + ) + ) + capable: Final = [d for d in deployments if _deployment_supports_web_search(d)] + if any(_deployment_id(d) not in tried for d in capable): + return capable + verbose_router_logger.debug("No deployment left for this web search request supports web search, so all stay") + return deployments # Credential params that only one provider family reads, paired with the providers diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index a9c9cf629a9..9387aaf487f 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -15,6 +15,7 @@ from litellm.router_utils.common_utils import ( add_model_file_id_mappings, filter_team_based_models, filter_web_search_deployments, + prefer_web_search_deployments, format_content_policy_no_fallback_note, format_fallback_outcome_message, provider_for_generic_call, @@ -327,7 +328,7 @@ class TestFilterWebSearchDeployments: def test_native_web_search_tools_filter_unsupported( self, sample_deployments: list[dict[str, object]], tool: dict[str, object] ) -> None: - result: Final = filter_web_search_deployments(sample_deployments, {"tools": [tool]}) + result: Final = prefer_web_search_deployments(sample_deployments, {"tools": [tool]}) assert result == sample_deployments[:2] @pytest.mark.parametrize( @@ -344,7 +345,7 @@ class TestFilterWebSearchDeployments: def test_non_web_search_tools_preserve_deployments( self, sample_deployments: list[dict[str, object]], tool: dict[str, object] ) -> None: - result: Final = filter_web_search_deployments(sample_deployments, {"tools": [tool]}) + result: Final = prefer_web_search_deployments(sample_deployments, {"tools": [tool]}) assert result == sample_deployments def test_web_search_with_other_tools(self, sample_deployments): @@ -384,30 +385,29 @@ class TestFilterWebSearchDeployments: {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}, {"tools": [{"type": "web_search_2025_08_26"}]}, {"tools": [{"googleSearch": {}}]}, + {"tools": [{"type": "web_search"}]}, {"web_search_options": {}}, ], ) - def test_no_deployment_supporting_other_web_search_keeps_every_deployment( - self, request_kwargs: dict[str, object] - ) -> None: - deployments: Final = [ + def test_no_deployment_supporting_web_search_keeps_every_deployment(self, request_kwargs: dict[str, object]) -> None: + deployments: Final[list[dict[str, object]]] = [ {"model_info": {"id": "d1", "supports_web_search": False}}, {"model_info": {"id": "d2", "supports_web_search": False}}, ] - assert filter_web_search_deployments(deployments, request_kwargs) == deployments + assert prefer_web_search_deployments(deployments, request_kwargs) == deployments def test_a_tool_that_is_not_an_object_is_no_web_search_request( self, sample_deployments: list[dict[str, object]] ) -> None: - result: Final = filter_web_search_deployments(sample_deployments, {"tools": ["web_search", None]}) + result: Final = prefer_web_search_deployments(sample_deployments, {"tools": ["web_search", None]}) assert result == sample_deployments def test_web_search_options_filter_unsupported(self, sample_deployments: list[dict[str, object]]) -> None: - result: Final = filter_web_search_deployments(sample_deployments, {"web_search_options": {}}) + result: Final = prefer_web_search_deployments(sample_deployments, {"web_search_options": {}}) assert result == sample_deployments[:2] def test_null_web_search_options_preserve_deployments(self, sample_deployments: list[dict[str, object]]) -> None: - result: Final = filter_web_search_deployments(sample_deployments, {"web_search_options": None}) + result: Final = prefer_web_search_deployments(sample_deployments, {"web_search_options": None}) assert result == sample_deployments def test_missing_config_defaults_to_supported(self): @@ -536,6 +536,92 @@ async def test_legacy_usage_based_async_pick_skips_deployments_without_web_searc assert picks == {"search"} +def _router_with_a_mixed_web_search_group(enable_tag_filtering: bool) -> Router: + return Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "tags": ["cheap"], "order": 1}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "tags": ["premium"], "order": 2}, + "model_info": {"id": "search"}, + }, + ], + enable_tag_filtering=enable_tag_filtering, + ) + + +def _cool_down(router: Router, model_ids: tuple[str, ...]) -> None: + for model_id in model_ids: + router.cooldown_cache.add_deployment_to_cooldown( + model_id=model_id, original_exception=Exception("rate limited"), exception_status=429, cooldown_time=300 + ) + + +_LATER_FILTER_CASES: Final = pytest.mark.parametrize( + ("enable_tag_filtering", "cooled_down", "request_kwargs", "expected_id"), + [ + (False, (), _ANTHROPIC_WEB_SEARCH_REQUEST, "search"), + (False, (), {"web_search_options": {}}, "search"), + (False, ("search",), _ANTHROPIC_WEB_SEARCH_REQUEST, "no-search"), + (False, ("search",), {"web_search_options": {}}, "no-search"), + (False, (), {**_ANTHROPIC_WEB_SEARCH_REQUEST, "_excluded_deployment_ids": ["search"]}, "no-search"), + (False, (), {**_ANTHROPIC_WEB_SEARCH_REQUEST, "_retry_skipped_deployment_ids": ("search",)}, "no-search"), + ], + ids=["order", "order-options", "cooldown", "cooldown-options", "weighted-failover", "retry-skip"], +) + + +@pytest.mark.asyncio +@_LATER_FILTER_CASES +async def test_async_pick_prefers_web_search_until_later_filters_leave_none_that_can( + enable_tag_filtering: bool, cooled_down: tuple[str, ...], request_kwargs: dict[str, object], expected_id: str +) -> None: + router: Final = _router_with_a_mixed_web_search_group(enable_tag_filtering) + _cool_down(router, cooled_down) + deployment: Final = await router.async_get_available_deployment( + model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(request_kwargs) + ) + assert deployment["model_info"]["id"] == expected_id + + +@_LATER_FILTER_CASES +def test_sync_pick_prefers_web_search_until_later_filters_leave_none_that_can( + enable_tag_filtering: bool, cooled_down: tuple[str, ...], request_kwargs: dict[str, object], expected_id: str +) -> None: + router: Final = _router_with_a_mixed_web_search_group(enable_tag_filtering) + _cool_down(router, cooled_down) + deployment: Final = router.get_available_deployment( + model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(request_kwargs) + ) + assert deployment["model_info"]["id"] == expected_id + + +@pytest.mark.asyncio +async def test_tag_routing_to_a_deployment_without_web_search_keeps_it() -> None: + deployment: Final = await _router_with_a_mixed_web_search_group(True).async_get_available_deployment( + model="mixed", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={"web_search_options": {}, "metadata": {"tags": ["cheap"]}}, + ) + assert deployment["model_info"]["id"] == "no-search" + + +@pytest.mark.asyncio +async def test_openai_web_search_still_finds_no_deployment_while_the_only_one_that_can_search_cools_down() -> None: + router: Final = _router_with_a_mixed_web_search_group(False) + _cool_down(router, ("search",)) + with pytest.raises(litellm.RouterRateLimitError): + await router.async_get_available_deployment( + model="mixed", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={"tools": [{"type": "web_search"}]}, + ) + + def test_invalidate_model_group_info_cache(): """Test that _invalidate_model_group_info_cache clears the LRU cache.""" router = Router( From 48f20a081ece3762cb670208fc6ced1130639774 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:57:58 -0700 Subject: [PATCH 07/11] fix(router): let a web search request skip a session or key pin on a deployment that cannot search --- .../deployment_affinity_check.py | 10 +- .../test_deployment_affinity_check.py | 98 +++++++++++++++++++ 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index daf7d6bb345..62272e751b0 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -31,7 +31,7 @@ from litellm.caching.dual_cache import DualCache from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger, Span from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.router_utils.common_utils import deployment_ids_failed_in_request +from litellm.router_utils.common_utils import deployment_ids_failed_in_request, prefer_web_search_deployments from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes @@ -433,6 +433,10 @@ class DeploymentAffinityCheck(CustomLogger): failed_deployment_ids: Final = deployment_ids_failed_in_request( cast(Mapping[str, object], request_kwargs) # cast-ok: request kwargs keys are param names ) + pin_candidates: Final = prefer_web_search_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + request_kwargs, + ) user_key: Final = ( self.get_user_key_from_request_kwargs(request_kwargs=request_kwargs) if (session_affinity_active or enable_user_key) @@ -464,7 +468,7 @@ class DeploymentAffinityCheck(CustomLogger): if session_model_id and session_model_id not in failed_deployment_ids: session_deployment: Final = self._find_deployment_by_model_id( - healthy_deployments=typed_healthy_deployments, + healthy_deployments=pin_candidates, model_id=session_model_id, ) if session_deployment is not None: @@ -501,7 +505,7 @@ class DeploymentAffinityCheck(CustomLogger): return typed_healthy_deployments deployment = self._find_deployment_by_model_id( - healthy_deployments=typed_healthy_deployments, + healthy_deployments=pin_candidates, model_id=model_id, ) if deployment is None: diff --git a/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py index fa75f980cc0..799acc5d5cc 100644 --- a/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py @@ -1299,3 +1299,101 @@ async def test_previous_response_id_pin_holds_even_after_its_deployment_failed() }, parent_otel_span=None, ) == [healthy_deployments[0]] + + +def _mixed_web_search_router(optional_pre_call_checks: list[str]) -> litellm.Router: # mutable-ok: Router signature + return litellm.Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi"}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi"}, + "model_info": {"id": "search"}, + }, + ], + optional_pre_call_checks=optional_pre_call_checks, + ) + + +_CALLER_METADATA: Final = {"user_api_key_hash": "caller-1", "session_id": "session-1"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "web_search_request", + [{"web_search_options": {}}, {"tools": [{"type": "web_search_20250305", "name": "web_search"}]}], + ids=["web-search-options", "web-search-tool"], +) +@pytest.mark.parametrize( + ("optional_pre_call_checks", "pin_key"), + [ + ( + ["deployment_affinity"], + DeploymentAffinityCheck.get_affinity_cache_key(model_group="mixed", user_key="caller-1"), + ), + ( + ["session_affinity"], + DeploymentAffinityCheck.get_session_affinity_cache_key( + model_group="mixed", session_id="session-1", user_key="caller-1" + ), + ), + ], + ids=["user-key-pin", "session-pin"], +) +async def test_pin_on_a_deployment_without_web_search_yields_to_a_web_search_request( + optional_pre_call_checks: list[str], # mutable-ok: Router signature + pin_key: str, + web_search_request: dict[str, object], # mutable-ok: request kwargs +): + router: Final = _mixed_web_search_router(optional_pre_call_checks) + await router.cache.async_set_cache(pin_key, {"model_id": "no-search"}, ttl=3600) + messages: Final = [{"role": "user", "content": "hi"}] + + plain: Final = await router.acompletion(model="mixed", messages=messages, metadata=dict(_CALLER_METADATA)) + searching: Final = await router.acompletion( + model="mixed", messages=messages, metadata=dict(_CALLER_METADATA), **web_search_request + ) + + assert plain._hidden_params["model_id"] == "no-search" + assert searching._hidden_params["model_id"] == "search" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("search_supported", "tried"), + [((False, False), ()), ((False, True), ("search",))], + ids=["no-deployment-can-search", "capable-deployment-already-tried"], +) +async def test_pin_holds_for_a_web_search_request_when_no_untried_deployment_can_search( + search_supported: tuple[bool, bool], tried: tuple[str, ...] +): + cache: Final = AsyncMock() + cache.async_get_cache = AsyncMock(return_value={"model_id": "no-search"}) + callback: Final = DeploymentAffinityCheck( + cache=cache, ttl_seconds=60, enable_user_key_affinity=True, enable_responses_api_affinity=False + ) + healthy_deployments: Final = [ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x"}, + "model_info": {"id": model_id, "supports_web_search": supported}, + } + for model_id, supported in zip(("no-search", "search"), search_supported) + ] + + filtered: Final = await callback.async_filter_deployments( + model="mixed", + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs={ + "metadata": {"user_api_key_hash": "caller-1"}, + "web_search_options": {}, + "_excluded_deployment_ids": list(tried), + }, + ) + + assert [d["model_info"]["id"] for d in filtered] == ["no-search"] From 5b2ccc8e585abf6c59629ac51df4a51dfb0262af Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:51:01 -0700 Subject: [PATCH 08/11] fix(router): keep the web search preference inside each order tier --- litellm/router_utils/common_utils.py | 18 ++- .../test_router_utils_common_utils.py | 136 ++++++++++++++++-- 2 files changed, 134 insertions(+), 20 deletions(-) diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 052bbdccb05..dd82f4e7862 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -507,11 +507,19 @@ def prefer_web_search_deployments( *as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")), ) ) - capable: Final = [d for d in deployments if _deployment_supports_web_search(d)] - if any(_deployment_id(d) not in tried for d in capable): - return capable - verbose_router_logger.debug("No deployment left for this web search request supports web search, so all stay") - return deployments + untried: Final = (d for d in deployments if _deployment_id(d) not in tried) + orders_with_an_untried_capable_deployment: Final = frozenset( + litellm.utils.get_deployment_order(d) for d in untried if _deployment_supports_web_search(d) + ) + if not orders_with_an_untried_capable_deployment: + verbose_router_logger.debug("No deployment left for this web search request supports web search, so all stay") + return deployments + return [ + d + for d in deployments + if _deployment_supports_web_search(d) + or litellm.utils.get_deployment_order(d) not in orders_with_an_untried_capable_deployment + ] # Credential params that only one provider family reads, paired with the providers diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 9387aaf487f..273973c4847 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -536,17 +536,33 @@ async def test_legacy_usage_based_async_pick_skips_deployments_without_web_searc assert picks == {"search"} -def _router_with_a_mixed_web_search_group(enable_tag_filtering: bool) -> Router: +def _order_param(order: int | None) -> dict[str, int]: + return {} if order is None else {"order": order} + + +def _router_with_a_mixed_web_search_group( + enable_tag_filtering: bool, orders: tuple[int | None, int | None] = (None, None) +) -> Router: return Router( model_list=[ { "model_name": "mixed", - "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "tags": ["cheap"], "order": 1}, + "litellm_params": { + "model": "openai/gpt-x", + "api_key": "fake-key", + "tags": ["cheap"], + **_order_param(orders[0]), + }, "model_info": {"id": "no-search", "supports_web_search": False}, }, { "model_name": "mixed", - "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "tags": ["premium"], "order": 2}, + "litellm_params": { + "model": "openai/gpt-y", + "api_key": "fake-key", + "tags": ["premium"], + **_order_param(orders[1]), + }, "model_info": {"id": "search"}, }, ], @@ -562,25 +578,39 @@ def _cool_down(router: Router, model_ids: tuple[str, ...]) -> None: _LATER_FILTER_CASES: Final = pytest.mark.parametrize( - ("enable_tag_filtering", "cooled_down", "request_kwargs", "expected_id"), + ("orders", "cooled_down", "request_kwargs", "expected_id"), [ - (False, (), _ANTHROPIC_WEB_SEARCH_REQUEST, "search"), - (False, (), {"web_search_options": {}}, "search"), - (False, ("search",), _ANTHROPIC_WEB_SEARCH_REQUEST, "no-search"), - (False, ("search",), {"web_search_options": {}}, "no-search"), - (False, (), {**_ANTHROPIC_WEB_SEARCH_REQUEST, "_excluded_deployment_ids": ["search"]}, "no-search"), - (False, (), {**_ANTHROPIC_WEB_SEARCH_REQUEST, "_retry_skipped_deployment_ids": ("search",)}, "no-search"), + ((None, None), (), _ANTHROPIC_WEB_SEARCH_REQUEST, "search"), + ((None, None), (), {"web_search_options": {}}, "search"), + ((1, 2), (), _ANTHROPIC_WEB_SEARCH_REQUEST, "no-search"), + ((1, 2), (), {"web_search_options": {}}, "no-search"), + ((None, None), ("search",), _ANTHROPIC_WEB_SEARCH_REQUEST, "no-search"), + ((None, None), ("search",), {"web_search_options": {}}, "no-search"), + ((None, None), (), {**_ANTHROPIC_WEB_SEARCH_REQUEST, "_excluded_deployment_ids": ["search"]}, "no-search"), + ((None, None), (), {**_ANTHROPIC_WEB_SEARCH_REQUEST, "_retry_skipped_deployment_ids": ("search",)}, "no-search"), + ], + ids=[ + "same-tier", + "same-tier-options", + "lower-order-tier", + "lower-order-tier-options", + "cooldown", + "cooldown-options", + "weighted-failover", + "retry-skip", ], - ids=["order", "order-options", "cooldown", "cooldown-options", "weighted-failover", "retry-skip"], ) @pytest.mark.asyncio @_LATER_FILTER_CASES async def test_async_pick_prefers_web_search_until_later_filters_leave_none_that_can( - enable_tag_filtering: bool, cooled_down: tuple[str, ...], request_kwargs: dict[str, object], expected_id: str + orders: tuple[int | None, int | None], + cooled_down: tuple[str, ...], + request_kwargs: dict[str, object], + expected_id: str, ) -> None: - router: Final = _router_with_a_mixed_web_search_group(enable_tag_filtering) + router: Final = _router_with_a_mixed_web_search_group(False, orders) _cool_down(router, cooled_down) deployment: Final = await router.async_get_available_deployment( model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(request_kwargs) @@ -590,9 +620,12 @@ async def test_async_pick_prefers_web_search_until_later_filters_leave_none_that @_LATER_FILTER_CASES def test_sync_pick_prefers_web_search_until_later_filters_leave_none_that_can( - enable_tag_filtering: bool, cooled_down: tuple[str, ...], request_kwargs: dict[str, object], expected_id: str + orders: tuple[int | None, int | None], + cooled_down: tuple[str, ...], + request_kwargs: dict[str, object], + expected_id: str, ) -> None: - router: Final = _router_with_a_mixed_web_search_group(enable_tag_filtering) + router: Final = _router_with_a_mixed_web_search_group(False, orders) _cool_down(router, cooled_down) deployment: Final = router.get_available_deployment( model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(request_kwargs) @@ -600,6 +633,79 @@ def test_sync_pick_prefers_web_search_until_later_filters_leave_none_that_can( assert deployment["model_info"]["id"] == expected_id +_WEB_SEARCH_REQUESTS: Final = pytest.mark.parametrize( + "web_search_request", + [_ANTHROPIC_WEB_SEARCH_REQUEST, {"web_search_options": {}}], + ids=["web-search-tool", "web-search-options"], +) + + +@pytest.mark.asyncio +@_WEB_SEARCH_REQUESTS +async def test_order_fallback_answers_a_web_search_request_from_the_next_tier_when_the_capable_one_fails( + web_search_request: dict[str, object], +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "ordered", + "litellm_params": { + "model": "openai/gpt-x", + "api_key": "fake-key", + "mock_response": Exception("connection error"), + "order": 1, + }, + "model_info": {"id": "search"}, + }, + { + "model_name": "ordered", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "order": 2}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + num_retries=0, + ) + response: Final = await router.acompletion( + model="ordered", messages=[{"role": "user", "content": "hi"}], **web_search_request + ) + assert response._hidden_params["model_id"] == "no-search" + + +@pytest.mark.asyncio +@_WEB_SEARCH_REQUESTS +async def test_web_search_request_prefers_the_capable_deployment_inside_its_order_tier( + web_search_request: dict[str, object], +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "ordered", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "order": 1}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + { + "model_name": "ordered", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "order": 1}, + "model_info": {"id": "search"}, + }, + { + "model_name": "ordered", + "litellm_params": {"model": "openai/gpt-z", "api_key": "fake-key", "order": 2}, + "model_info": {"id": "later-no-search", "supports_web_search": False}, + }, + ], + ) + picks: Final = { + ( + await router.async_get_available_deployment( + model="ordered", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(web_search_request) + ) + )["model_info"]["id"] + for _ in range(20) + } + assert picks == {"search"} + + @pytest.mark.asyncio async def test_tag_routing_to_a_deployment_without_web_search_keeps_it() -> None: deployment: Final = await _router_with_a_mixed_web_search_group(True).async_get_available_deployment( From a179118b3eb0e62f3251ba11544e2b9bf64c78ea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:13:54 -0700 Subject: [PATCH 09/11] fix(router): prefer web search deployments at the strategy pick and fall back when none has capacity --- litellm/router.py | 73 +++++---- litellm/router_utils/common_utils.py | 65 +++++++- .../test_router_utils_common_utils.py | 154 +++++++++++++++++- 3 files changed, 252 insertions(+), 40 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 33f53d9b852..16eadfe159f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -235,6 +235,7 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.router_utils.common_utils import ( MID_STREAM_CONTINUATION_KWARG, + apick_preferring_web_search, as_retry_skipped_deployment_ids, deployments_the_caller_may_use, filter_team_based_models, @@ -245,6 +246,7 @@ from litellm.router_utils.common_utils import ( format_no_fallback_group_message, get_request_team_id, is_proxy_admin_request, + pick_preferring_web_search, prefer_web_search_deployments, provider_for_generic_call, request_user_api_key_auth, @@ -15246,13 +15248,10 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = prefer_web_search_deployments( - self._without_declared_text_only_deployments( - cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts - model, - messages, - cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict - ), + batch_candidates: Final = self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( @@ -15535,26 +15534,31 @@ class Router: return healthy_deployments[0] start_time: Final = time.time() + routing_kwargs: Final = cast(Mapping[str, object], request_kwargs) # cast-ok: str-keyed request kwargs if strategy == "simple-shuffle": shuffled: Final = simple_shuffle( resolve_model_alias=self.get_model_from_alias, - healthy_deployments=healthy_deployments, + healthy_deployments=prefer_web_search_deployments(healthy_deployments, routing_kwargs), model=model, - request_kwargs=request_kwargs, + request_kwargs=routing_kwargs, ) phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return shuffled with PrefetchedUsage.scoped( routing_read_batch.prefetched_usage if routing_read_batch is not None else None ): - deployment: Final = await self._select_deployment_async( - strategy=strategy, - selector=strategy_selector, - model=model, - healthy_deployments=healthy_deployments, - messages=messages, - input=input, - request_kwargs=request_kwargs, + deployment: Final = await apick_preferring_web_search( + lambda candidates: self._select_deployment_async( + strategy=strategy, + selector=strategy_selector, + model=model, + healthy_deployments=candidates, + messages=messages, + input=input, + request_kwargs=request_kwargs, + ), + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + routing_kwargs, ) if deployment is None: exception: Final = await async_raise_no_deployment_exception( @@ -16481,13 +16485,10 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - batch_candidates: Final = prefer_web_search_deployments( - self._without_declared_text_only_deployments( - cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts - model, - messages, - cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict - ), + batch_candidates: Final = self._without_declared_text_only_deployments( + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + model, + messages, cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict ) embedding_batch_admitting: Final = deployments_admitting_embedding_batch( @@ -16575,20 +16576,24 @@ class Router: ############## Check 'weight' param set for weighted pick ################# shuffled: Final = simple_shuffle( resolve_model_alias=self.get_model_from_alias, - healthy_deployments=healthy_deployments, + healthy_deployments=prefer_web_search_deployments(healthy_deployments, routing_kwargs), model=model, - request_kwargs=request_kwargs, + request_kwargs=routing_kwargs, ) phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return shuffled - deployment: Final = self._select_deployment_sync( - strategy=strategy, - selector=strategy_selector, - model=model, - healthy_deployments=healthy_deployments, - messages=messages, - input=input, - request_kwargs=request_kwargs, + deployment: Final = pick_preferring_web_search( + lambda candidates: self._select_deployment_sync( + strategy=strategy, + selector=strategy_selector, + model=model, + healthy_deployments=candidates, + messages=messages, + input=input, + request_kwargs=request_kwargs, + ), + cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts + routing_kwargs, ) if deployment is None: diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index dd82f4e7862..5b3f923d936 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -1,6 +1,6 @@ import hashlib import json -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar @@ -33,6 +33,8 @@ from litellm.types.utils import ( ) _V = TypeVar("_V") +_Picked: Final = TypeVar("_Picked") +_DeploymentT: Final = TypeVar("_DeploymentT", bound=dict[str, object]) MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation" ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG}) @@ -53,6 +55,9 @@ _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = tuple[Mapping[str, object], ...] ) _DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...]) +_RETRY_BREADCRUMBS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( + tuple[Mapping[str, object], ...] +) _FALLBACK_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) @@ -493,10 +498,31 @@ def _deployment_id(deployment: Mapping[str, object]) -> object: return None if model_info is None else model_info.get("id") +def _retry_breadcrumbs(metadata: object) -> tuple[Mapping[str, object], ...]: + fields: Final = as_str_mapping(metadata) + breadcrumbs: Final = None if fields is None else fields.get("previous_models") + if breadcrumbs is None: + return () + try: + return _RETRY_BREADCRUMBS_ADAPTER.validate_python(breadcrumbs) + except ValidationError: + return () + + +def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) -> tuple[str, ...]: + breadcrumbs: Final = ( + *_retry_breadcrumbs(request_kwargs.get("metadata")), + *_retry_breadcrumbs(request_kwargs.get("litellm_metadata")), + ) + return tuple( + deployment_id for breadcrumb in breadcrumbs if isinstance(deployment_id := breadcrumb.get("deployment_id"), str) + ) + + def prefer_web_search_deployments( - deployments: list[dict[str, object]], # mutable-ok: router deployment-list contract + deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract request_kwargs: Mapping[str, object] | None, -) -> list[dict[str, object]]: # mutable-ok: returns a mutable deployment list +) -> list[_DeploymentT]: # mutable-ok: returns a mutable deployment list if request_kwargs is None or not _is_web_search_request( request_kwargs.get("tools"), request_kwargs.get("web_search_options") ): @@ -505,10 +531,11 @@ def prefer_web_search_deployments( ( *as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")), *as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")), + *_deployment_ids_retried_in_request(request_kwargs), ) ) untried: Final = (d for d in deployments if _deployment_id(d) not in tried) - orders_with_an_untried_capable_deployment: Final = frozenset( + orders_with_an_untried_capable_deployment: Final = tuple( litellm.utils.get_deployment_order(d) for d in untried if _deployment_supports_web_search(d) ) if not orders_with_an_untried_capable_deployment: @@ -522,6 +549,36 @@ def prefer_web_search_deployments( ] +def pick_preferring_web_search( + pick: Callable[[list[_DeploymentT]], _Picked | None], # mutable-ok: router deployment-list contract + deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract + request_kwargs: Mapping[str, object] | None, +) -> _Picked | None: + preferred: Final = prefer_web_search_deployments(deployments, request_kwargs) + if len(preferred) == len(deployments): + return pick(deployments) + try: + from_preferred: Final = pick(preferred) + except (litellm.RateLimitError, ValueError): + return pick(deployments) + return from_preferred if from_preferred is not None else pick(deployments) + + +async def apick_preferring_web_search( + pick: Callable[[list[_DeploymentT]], Awaitable[_Picked | None]], # mutable-ok: router deployment-list contract + deployments: list[_DeploymentT], # mutable-ok: router deployment-list contract + request_kwargs: Mapping[str, object] | None, +) -> _Picked | None: + preferred: Final = prefer_web_search_deployments(deployments, request_kwargs) + if len(preferred) == len(deployments): + return await pick(deployments) + try: + from_preferred: Final = await pick(preferred) + except (litellm.RateLimitError, ValueError): + return await pick(deployments) + return from_preferred if from_preferred is not None else await pick(deployments) + + # Credential params that only one provider family reads, paired with the providers # that read them. A deployment carrying them while resolving elsewhere is almost # always a missing route prefix: `model: claude-sonnet-5` with `aws_region_name` diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 273973c4847..0049793a354 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -1,3 +1,4 @@ +import asyncio import logging from itertools import chain from typing import Dict, Final, List, Optional, Union @@ -15,6 +16,7 @@ from litellm.router_utils.common_utils import ( add_model_file_id_mappings, filter_team_based_models, filter_web_search_deployments, + pick_preferring_web_search, prefer_web_search_deployments, format_content_policy_no_fallback_note, format_fallback_outcome_message, @@ -523,8 +525,11 @@ def test_sync_pick_keeps_a_group_without_web_search_for_openai_web_search(routin @pytest.mark.asyncio -async def test_legacy_usage_based_async_pick_skips_deployments_without_web_search() -> None: - router: Final = _router_with_a_deployment_that_disables_web_search("usage-based-routing") +@pytest.mark.parametrize( + "routing_strategy", ["usage-based-routing", "usage-based-routing-v2", "least-busy", "latency-based-routing"] +) +async def test_async_strategy_pick_skips_deployments_without_web_search(routing_strategy: str) -> None: + router: Final = _router_with_a_deployment_that_disables_web_search(routing_strategy) picks: Final = { ( await router.async_get_available_deployment( @@ -706,6 +711,151 @@ async def test_web_search_request_prefers_the_capable_deployment_inside_its_orde assert picks == {"search"} +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +async def test_web_search_preference_gives_way_once_the_capable_deployment_failed_this_request( + metadata_field: str, +) -> None: + router: Final = _router_with_a_mixed_web_search_group(False) + picks: Final = { + ( + await router.async_get_available_deployment( + model="mixed", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={ + **_ANTHROPIC_WEB_SEARCH_REQUEST, + metadata_field: {"previous_models": ({"deployment_id": "search", "exception_type": "Timeout"},)}, + }, + ) + )["model_info"]["id"] + for _ in range(40) + } + assert picks == {"search", "no-search"} + + +@pytest.mark.asyncio +@_WEB_SEARCH_REQUESTS +async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches_the_other_one( + web_search_request: dict[str, object], +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-x", + "api_key": "fake-key", + "mock_response": Exception("connection error"), + "weight": 1, + }, + "model_info": {"id": "search"}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "weight": 10**6}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + num_retries=1, + disable_cooldowns=True, + ) + response: Final = await router.acompletion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request + ) + assert response._hidden_params["model_id"] == "no-search" + + +def _rate_limited_web_search_group(routing_strategy: str) -> Router: + return Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi", "rpm": 1}, + "model_info": {"id": "search"}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "rpm": 1000}, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + routing_strategy=routing_strategy, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("routing_strategy", ["usage-based-routing-v2", "cost-based-routing"]) +async def test_web_search_request_goes_to_the_other_deployment_once_the_capable_one_is_at_its_rpm_limit( + routing_strategy: str, +) -> None: + router: Final = _rate_limited_web_search_group(routing_strategy) + served_by: Final = [] # mutable-ok: collects one id per sequential request + for _ in range(3): + response = await router.acompletion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST + ) + served_by.append(response._hidden_params["model_id"]) + await asyncio.sleep(0.2) + assert served_by == ["search", "no-search", "no-search"] + + +def test_sync_web_search_request_goes_to_the_other_deployment_once_the_capable_one_is_at_its_rpm_limit() -> None: + router: Final = _rate_limited_web_search_group("usage-based-routing-v2") + served_by: Final = tuple( + router.completion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST + )._hidden_params["model_id"] + for _ in range(3) + ) + assert served_by == ("search", "no-search", "no-search") + + +def test_sync_pick_falls_back_to_every_deployment_when_the_strategy_finds_none_that_can_search() -> None: + deployments: Final = [ + {"model_info": {"id": "search"}}, + {"model_info": {"id": "no-search", "supports_web_search": False}}, + ] + picked: Final = pick_preferring_web_search( + lambda candidates: next((d for d in candidates if d["model_info"]["id"] == "no-search"), None), + deployments, + _ANTHROPIC_WEB_SEARCH_REQUEST, + ) + assert picked == deployments[1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("search_order", "no_search_order"), + [([1], [1]), ({"tier": 1}, None)], + ids=["list-order", "dict-order"], +) +async def test_web_search_request_routes_on_a_group_whose_order_is_not_a_number( + search_order: object, no_search_order: object +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key"}, + "model_info": {"id": "search", "order": search_order}, + }, + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key"}, + "model_info": { + "id": "no-search", + "supports_web_search": False, + **({} if no_search_order is None else {"order": no_search_order}), + }, + }, + ], + ) + deployment: Final = await router.async_get_available_deployment( + model="mixed", messages=[{"role": "user", "content": "hi"}], request_kwargs=dict(_ANTHROPIC_WEB_SEARCH_REQUEST) + ) + assert deployment["model_info"]["id"] == "search" + + @pytest.mark.asyncio async def test_tag_routing_to_a_deployment_without_web_search_keeps_it() -> None: deployment: Final = await _router_with_a_mixed_web_search_group(True).async_get_available_deployment( From 27ec8d2adf840aaa2e9c7451a372053b03b64273 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:04:26 -0700 Subject: [PATCH 10/11] fix(router): count a failed deployment as tried when the caller supplies the key With caller-supplied credentials the retry record's deployment_id is the generated id, so the web search preference never saw the configured deployment that failed and sent every retry back to it. Retry records now also carry the configured id. --- litellm/router.py | 13 ++++++++++--- litellm/router_utils/common_utils.py | 4 +++- litellm/types/router.py | 3 ++- .../test_router_utils_common_utils.py | 15 ++++++++++++--- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 16eadfe159f..ec0692a937f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -119,6 +119,7 @@ from litellm.litellm_core_utils.batch_utils import ( ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, coerce_token_limit, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -9880,14 +9881,15 @@ class Router: _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs model_group: Final = kwargs.get("model") - model_info: Final = request_metadata.get("model_info") - deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None + model_info: Final = as_str_mapping(request_metadata.get("model_info")) or {} + deployment_id: Final = model_info.get("id") + configured_deployment_id: Final = model_info.get("original_model_id") base_model: Final = get_base_model_from_litellm_call_metadata( metadata=request_metadata ) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None) deployment_model: Final = request_metadata.get("deployment") attempted_retries: Final = request_metadata.get("attempted_retries") - attempt_record: Final[RetryAttemptRecord] = { + attempt_fields: Final[RetryAttemptRecord] = { "model_group": model_group if isinstance(model_group, str) else None, "deployment_id": deployment_id if isinstance(deployment_id, str) else None, "model": deployment_model if isinstance(deployment_model, str) else None, @@ -9896,6 +9898,11 @@ class Router: "exception_string": str(e), "attempted_retries": attempted_retries if type(attempted_retries) is int else None, } + attempt_record: Final[RetryAttemptRecord] = ( + {**attempt_fields, "configured_deployment_id": configured_deployment_id} + if isinstance(configured_deployment_id, str) + else attempt_fields + ) earlier_breadcrumbs: Final = request_metadata.get("previous_models") kept_breadcrumbs: Final[tuple[object, ...]] = ( tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :] diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 5b3f923d936..34e12c1f751 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -515,7 +515,9 @@ def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) -> *_retry_breadcrumbs(request_kwargs.get("litellm_metadata")), ) return tuple( - deployment_id for breadcrumb in breadcrumbs if isinstance(deployment_id := breadcrumb.get("deployment_id"), str) + deployment_id + for breadcrumb in breadcrumbs + if isinstance(deployment_id := breadcrumb.get("configured_deployment_id", breadcrumb.get("deployment_id")), str) ) diff --git a/litellm/types/router.py b/litellm/types/router.py index f8de2128a80..75c4eea9547 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -23,7 +23,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from openai import Timeout as SDKTimeout from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator -from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable +from typing_extensions import NotRequired, Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -1197,6 +1197,7 @@ class RetryAttemptRecord(TypedDict): exception_type: ReadOnly[str] exception_string: ReadOnly[str] attempted_retries: ReadOnly[int | None] + configured_deployment_id: ReadOnly[NotRequired[str]] VALID_LITELLM_ENVIRONMENTS = [ diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 0049793a354..a28731afad1 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -735,8 +735,12 @@ async def test_web_search_preference_gives_way_once_the_capable_deployment_faile @pytest.mark.asyncio @_WEB_SEARCH_REQUESTS +@pytest.mark.parametrize( + "caller_credentials", [{}, {"api_key": "caller-key"}], ids=["configured-credentials", "caller-credentials"] +) async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches_the_other_one( web_search_request: dict[str, object], + caller_credentials: dict[str, str], ) -> None: router: Final = Router( model_list=[ @@ -752,7 +756,12 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches }, { "model_name": "mixed", - "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "weight": 10**6}, + "litellm_params": { + "model": "openai/gpt-y", + "api_key": "fake-key", + "mock_response": "from no-search", + "weight": 10**6, + }, "model_info": {"id": "no-search", "supports_web_search": False}, }, ], @@ -760,9 +769,9 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches disable_cooldowns=True, ) response: Final = await router.acompletion( - model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request + model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request, **caller_credentials ) - assert response._hidden_params["model_id"] == "no-search" + assert response.choices[0].message.content == "from no-search" def _rate_limited_web_search_group(routing_strategy: str) -> Router: From 46aed6565f746a82d854b7c423ae071ea9da2fea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:12:10 -0700 Subject: [PATCH 11/11] fix(router): let the web search preference step aside once retry history was trimmed --- litellm/router_utils/common_utils.py | 11 +++ .../test_router_utils_common_utils.py | 84 +++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 34e12c1f751..c2f70503cc2 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -509,6 +509,12 @@ def _retry_breadcrumbs(metadata: object) -> tuple[Mapping[str, object], ...]: return () +def _retry_history_was_trimmed(metadata: object) -> bool: + fields: Final = as_str_mapping(metadata) + retry_count: Final = None if fields is None else fields.get("request_retry_count") + return type(retry_count) is int and retry_count > len(_retry_breadcrumbs(metadata)) + + def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) -> tuple[str, ...]: breadcrumbs: Final = ( *_retry_breadcrumbs(request_kwargs.get("metadata")), @@ -529,6 +535,11 @@ def prefer_web_search_deployments( request_kwargs.get("tools"), request_kwargs.get("web_search_options") ): return deployments + if _retry_history_was_trimmed(request_kwargs.get("metadata")) or _retry_history_was_trimmed( + request_kwargs.get("litellm_metadata") + ): + verbose_router_logger.debug("This request's retry history was trimmed, so every deployment stays") + return deployments tried: Final = frozenset( ( *as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")), diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index a28731afad1..4e76fff5aad 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -774,6 +774,90 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches assert response.choices[0].message.content == "from no-search" +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize(("request_retry_count", "flagged_reachable"), [(1, False), (2, True)], ids=["kept", "trimmed"]) +async def test_web_search_preference_steps_aside_only_once_the_retry_history_was_trimmed( + metadata_field: str, request_retry_count: int, flagged_reachable: bool +) -> None: + router: Final = Router( + model_list=[ + *( + { + "model_name": "mixed", + "litellm_params": {"model": "openai/gpt-x", "api_key": "fake-key", "mock_response": "hi"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("search-a", "search-b") + ), + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-y", + "api_key": "fake-key", + "mock_response": "hi", + "weight": 10**6, + }, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + ) + picks: Final = { + ( + await router.async_get_available_deployment( + model="mixed", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={ + **_ANTHROPIC_WEB_SEARCH_REQUEST, + metadata_field: { + "previous_models": ({"deployment_id": "search-a", "exception_type": "Timeout"},), + "request_retry_count": request_retry_count, + }, + }, + ) + )["model_info"]["id"] + for _ in range(20) + } + assert ("no-search" in picks) is flagged_reachable + + +@pytest.mark.asyncio +async def test_retries_past_the_kept_retry_history_reach_the_deployment_that_cannot_search() -> None: + capable_deployments: Final = [ + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-x", + "api_key": "fake-key", + "mock_response": Exception("connection error"), + }, + "model_info": {"id": f"search-{index}"}, + } + for index in range(5) + ] + router: Final = Router( + model_list=[ + *capable_deployments, + { + "model_name": "mixed", + "litellm_params": { + "model": "openai/gpt-y", + "api_key": "fake-key", + "mock_response": "from no-search", + "weight": 10**6, + }, + "model_info": {"id": "no-search", "supports_web_search": False}, + }, + ], + num_retries=5, + disable_cooldowns=True, + ) + response: Final = await router.acompletion( + model="mixed", messages=[{"role": "user", "content": "hi"}], **_ANTHROPIC_WEB_SEARCH_REQUEST + ) + assert response.choices[0].message.content == "from no-search" + + def _rate_limited_web_search_group(routing_strategy: str) -> Router: return Router( model_list=[