diff --git a/litellm/router.py b/litellm/router.py index f0bb6c3a7a9..97b473d176a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -215,6 +215,7 @@ from litellm.router_utils.common_utils import ( as_retry_skipped_deployment_ids, deployments_the_caller_may_use, filter_team_based_models, + filter_tool_protocol_deployments, filter_web_search_deployments, format_fallback_outcome_message, format_no_fallback_group_message, @@ -15032,6 +15033,10 @@ class Router: model, ) ) + healthy_deployments = filter_tool_protocol_deployments( # rebind-ok: progressive narrowing of the candidate list through the router filter pipeline; rebinding is the established pattern in this function + healthy_deployments=healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any] + request_kwargs=request_kwargs, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any] + ) if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: @@ -16346,11 +16351,16 @@ class Router: self._filter_blocked_deployments(healthy_deployments), routing_kwargs ) + healthy_deployments = filter_tool_protocol_deployments( # rebind-ok: progressive narrowing of the candidate list through the router filter pipeline; rebinding is the established pattern in this function + healthy_deployments=healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any] + request_kwargs=request_kwargs, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any] + ) + # filter pre-call checks if self.enable_pre_call_checks and (messages is not None or input is not None): healthy_deployments = self._pre_call_checks( model=model, - healthy_deployments=healthy_deployments, + healthy_deployments=healthy_deployments, # pyright: ignore[reportArgumentType] # filtered result stays a bare-dict list for the untyped callee messages=messages, input=input, request_kwargs=request_kwargs, diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 91b9e5cb114..4f5eb50308e 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -2,7 +2,7 @@ import hashlib import json from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeVar +from typing import TYPE_CHECKING, Any, Final, TypeVar, cast # noqa: TID251 # Any for basedpyright; cast is validated from pydantic import TypeAdapter, ValidationError @@ -424,6 +424,60 @@ def filter_web_search_deployments( return final_deployments +# Providers that natively speak the OpenAI chat-completions wire protocol, so an +# OpenAI-format tool-calling contract reaches them without cross-protocol +# translation. Same family litellm uses for OpenAI-protocol param handling in +# `add_provider_specific_params_to_optional_params` (litellm/utils.py). +_OPENAI_NATIVE_PROTOCOL_PROVIDERS: Final = frozenset( + ["openai", "azure", "text-completion-openai"] + + cast("list[str]", litellm.openai_compatible_providers) # cast-ok: provider names are strings +) + + +def _deployment_speaks_openai_protocol(deployment: dict[str, Any]) -> bool: # mutable-ok: read-only, never mutated + litellm_params: Final[dict[str, Any]] = deployment.get("litellm_params") or {} # mutable-ok: read-only + provider: Final = provider_for_generic_call(litellm_params) + if provider is None: + return True + return provider in _OPENAI_NATIVE_PROTOCOL_PROVIDERS + + +def filter_tool_protocol_deployments( + healthy_deployments: list[dict[str, Any]] | dict[str, Any], # mutable-ok: router candidate type + request_kwargs: dict[str, Any] | None = None, # mutable-ok: OptionalParams dict +) -> list[dict[str, Any]] | dict[str, Any]: # mutable-ok: returns input or narrowed list + """ + Keep only OpenAI-protocol deployments for OpenAI-format tool-calling requests. + + In a mixed-protocol model group (e.g. an `openai/...` and an `anthropic/...` + deployment behind one model name), sending OpenAI-format `tools` to a + non-OpenAI-protocol deployment forces a cross-protocol translation of the + tool contract that third-party upstreams can reject, causing intermittent + tool-calling failures (BerriAI/litellm#43685). + + The tool shape identifies the inbound protocol: only + `{"type": "function", "function": {...}}` triggers filtering, so + Anthropic-protocol inbound requests (proxy `/v1/messages`) are unaffected. + + When no deployment speaks the OpenAI protocol, candidates are left + untouched so requests that succeed today via provider translation keep + working; filtering only narrows selection when a same-protocol alternative + exists. Deployments whose provider cannot be resolved are kept. + """ + if request_kwargs is None or isinstance(healthy_deployments, dict): + return healthy_deployments + tools: Final[list[dict[str, Any]]] = request_kwargs.get("tools") or [] # mutable-ok: read-only + is_openai_tool_request: Final = any( + tool.get("type") == "function" and isinstance(tool.get("function"), dict) for tool in tools + ) + if not is_openai_tool_request: + return healthy_deployments + compatible: Final = [d for d in healthy_deployments if _deployment_speaks_openai_protocol(d)] + if not compatible: + return healthy_deployments + return compatible + + # Credential params that only one provider family reads, paired with the providers # that read them. A deployment carrying them while resolving elsewhere is almost # always a missing route prefix: `model: claude-sonnet-5` with `aws_region_name` diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 3532870df1c..38ff1bf6540 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -14,6 +14,7 @@ from litellm.router_utils.common_utils import ( _deployment_supports_web_search, add_model_file_id_mappings, filter_team_based_models, + filter_tool_protocol_deployments, filter_web_search_deployments, format_fallback_outcome_message, provider_for_generic_call, @@ -374,6 +375,69 @@ class TestFilterWebSearchDeployments: assert result == deployment +class TestFilterToolProtocolDeployments: + """Regression tests for BerriAI/litellm#43685: an OpenAI-format tool-calling + request must not be routed to a non-OpenAI-protocol deployment when an + OpenAI-protocol deployment is available in the same model group.""" + + @pytest.fixture + def mixed_protocol_deployments(self) -> list[dict]: + return [ + { + "model_name": "claude-4.6-sonnet", + "litellm_params": {"model": "openai/claude-4.6-sonnet"}, + "model_info": {"id": "openai-deployment"}, + }, + { + "model_name": "claude-4.6-sonnet", + "litellm_params": {"model": "anthropic/claude-4.6-sonnet"}, + "model_info": {"id": "anthropic-deployment"}, + }, + ] + + @staticmethod + def _openai_tool_kwargs() -> dict: + return {"tools": [{"type": "function", "function": {"name": "get_weather"}}]} + + def test_openai_tool_request_keeps_only_openai_protocol(self, mixed_protocol_deployments): + result = filter_tool_protocol_deployments(mixed_protocol_deployments, self._openai_tool_kwargs()) + assert [d["model_info"]["id"] for d in result] == ["openai-deployment"] + + def test_no_tools_keeps_all(self, mixed_protocol_deployments): + assert filter_tool_protocol_deployments(mixed_protocol_deployments, {}) == mixed_protocol_deployments + + def test_empty_tools_keeps_all(self, mixed_protocol_deployments): + assert filter_tool_protocol_deployments(mixed_protocol_deployments, {"tools": []}) == mixed_protocol_deployments + + def test_anthropic_format_tools_keeps_all(self, mixed_protocol_deployments): + request_kwargs = {"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}]} + assert ( + filter_tool_protocol_deployments(mixed_protocol_deployments, request_kwargs) == mixed_protocol_deployments + ) + + def test_no_openai_protocol_deployment_keeps_all_for_translation(self): + deployments = [ + {"litellm_params": {"model": "anthropic/claude"}, "model_info": {"id": "a1"}}, + {"litellm_params": {"model": "bedrock/anthropic.claude"}, "model_info": {"id": "b1"}}, + ] + result = filter_tool_protocol_deployments(deployments, self._openai_tool_kwargs()) + assert result == deployments + + def test_unresolvable_provider_is_kept(self, mixed_protocol_deployments): + deployments = mixed_protocol_deployments + [ + {"litellm_params": {"model": "no-provider-knows-this-model"}, "model_info": {"id": "mystery"}} + ] + result = filter_tool_protocol_deployments(deployments, self._openai_tool_kwargs()) + assert [d["model_info"]["id"] for d in result] == ["openai-deployment", "mystery"] + + def test_dict_deployment_passthrough(self): + deployment = {"litellm_params": {"model": "anthropic/x"}, "model_info": {"id": "d1"}} + assert filter_tool_protocol_deployments(deployment, self._openai_tool_kwargs()) is deployment + + def test_none_kwargs_passthrough(self, mixed_protocol_deployments): + assert filter_tool_protocol_deployments(mixed_protocol_deployments, None) == mixed_protocol_deployments + + def test_invalidate_model_group_info_cache(): """Test that _invalidate_model_group_info_cache clears the LRU cache.""" router = Router( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index c6d02dddae0..765e970bf52 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -30599,3 +30599,42 @@ async def test_anthropic_caller_rate_limit_does_not_block_other_credentials( finally: await GLOBAL_LOGGING_WORKER.flush() await client.close() +def _mixed_protocol_model_list(): + return [ + { + "model_name": "claude-4.6-sonnet", + "litellm_params": {"model": "openai/claude-4.6-sonnet", "api_key": "fake-key"}, + }, + { + "model_name": "claude-4.6-sonnet", + "litellm_params": {"model": "anthropic/claude-4.6-sonnet", "api_key": "fake-key"}, + }, + ] + + +def test_router_openai_tool_request_selects_openai_protocol_deployment(): + """BerriAI/litellm#43685: in a mixed-protocol model group, an OpenAI-format + tool-calling request must only select the OpenAI-protocol deployment.""" + router = litellm.Router(model_list=_mixed_protocol_model_list(), routing_strategy="simple-shuffle") + tools = [{"type": "function", "function": {"name": "get_weather"}}] + for _ in range(20): + deployment = router.get_available_deployment( + model="claude-4.6-sonnet", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={"tools": tools}, + ) + assert deployment["litellm_params"]["model"] == "openai/claude-4.6-sonnet" + + +@pytest.mark.asyncio +async def test_router_openai_tool_request_selects_openai_protocol_deployment_async(): + """BerriAI/litellm#43685: same guarantee on the async deployment-selection path.""" + router = litellm.Router(model_list=_mixed_protocol_model_list(), routing_strategy="simple-shuffle") + tools = [{"type": "function", "function": {"name": "get_weather"}}] + for _ in range(20): + deployment = await router.async_get_available_deployment( + model="claude-4.6-sonnet", + messages=[{"role": "user", "content": "hi"}], + request_kwargs={"tools": tools}, + ) + assert deployment["litellm_params"]["model"] == "openai/claude-4.6-sonnet"