This commit is contained in:
Junyi Yao 2026-10-28 06:26:59 -07:00 • committed by GitHub
commit fb7ccfed3d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 169 additions and 2 deletions

View file

@ -215,6 +215,7 @@ from litellm.router_utils.common_utils import (
as_retry_skipped_deployment_ids,
deployments_the_caller_may_use,
filter_team_based_models,
filter_tool_protocol_deployments,
filter_web_search_deployments,
format_fallback_outcome_message,
format_no_fallback_group_message,
@ -15032,6 +15033,10 @@ class Router:
model,
)
)
healthy_deployments = filter_tool_protocol_deployments( # rebind-ok: progressive narrowing of the candidate list through the router filter pipeline; rebinding is the established pattern in this function
healthy_deployments=healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any]
request_kwargs=request_kwargs, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any]
)
if isinstance(healthy_deployments, dict):
if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
@ -16346,11 +16351,16 @@ class Router:
self._filter_blocked_deployments(healthy_deployments), routing_kwargs
)
healthy_deployments = filter_tool_protocol_deployments( # rebind-ok: progressive narrowing of the candidate list through the router filter pipeline; rebinding is the established pattern in this function
healthy_deployments=healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any]
request_kwargs=request_kwargs, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any]
)
# filter pre-call checks
if self.enable_pre_call_checks and (messages is not None or input is not None):
healthy_deployments = self._pre_call_checks(
model=model,
healthy_deployments=healthy_deployments,
healthy_deployments=healthy_deployments, # pyright: ignore[reportArgumentType] # filtered result stays a bare-dict list for the untyped callee
messages=messages,
input=input,
request_kwargs=request_kwargs,

View file

@ -2,7 +2,7 @@ import hashlib
import json
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, TypeVar
from typing import TYPE_CHECKING, Any, Final, TypeVar, cast # noqa: TID251 # Any for basedpyright; cast is validated
from pydantic import TypeAdapter, ValidationError
@ -424,6 +424,60 @@ def filter_web_search_deployments(
return final_deployments
# Providers that natively speak the OpenAI chat-completions wire protocol, so an
# OpenAI-format tool-calling contract reaches them without cross-protocol
# translation. Same family litellm uses for OpenAI-protocol param handling in
# `add_provider_specific_params_to_optional_params` (litellm/utils.py).
_OPENAI_NATIVE_PROTOCOL_PROVIDERS: Final = frozenset(
["openai", "azure", "text-completion-openai"]
+ cast("list[str]", litellm.openai_compatible_providers) # cast-ok: provider names are strings
)
def _deployment_speaks_openai_protocol(deployment: dict[str, Any]) -> bool: # mutable-ok: read-only, never mutated
litellm_params: Final[dict[str, Any]] = deployment.get("litellm_params") or {} # mutable-ok: read-only
provider: Final = provider_for_generic_call(litellm_params)
if provider is None:
return True
return provider in _OPENAI_NATIVE_PROTOCOL_PROVIDERS
def filter_tool_protocol_deployments(
healthy_deployments: list[dict[str, Any]] | dict[str, Any], # mutable-ok: router candidate type
request_kwargs: dict[str, Any] | None = None, # mutable-ok: OptionalParams dict
) -> list[dict[str, Any]] | dict[str, Any]: # mutable-ok: returns input or narrowed list
"""
Keep only OpenAI-protocol deployments for OpenAI-format tool-calling requests.
In a mixed-protocol model group (e.g. an `openai/...` and an `anthropic/...`
deployment behind one model name), sending OpenAI-format `tools` to a
non-OpenAI-protocol deployment forces a cross-protocol translation of the
tool contract that third-party upstreams can reject, causing intermittent
tool-calling failures (BerriAI/litellm#43685).
The tool shape identifies the inbound protocol: only
`{"type": "function", "function": {...}}` triggers filtering, so
Anthropic-protocol inbound requests (proxy `/v1/messages`) are unaffected.
When no deployment speaks the OpenAI protocol, candidates are left
untouched so requests that succeed today via provider translation keep
working; filtering only narrows selection when a same-protocol alternative
exists. Deployments whose provider cannot be resolved are kept.
"""
if request_kwargs is None or isinstance(healthy_deployments, dict):
return healthy_deployments
tools: Final[list[dict[str, Any]]] = request_kwargs.get("tools") or [] # mutable-ok: read-only
is_openai_tool_request: Final = any(
tool.get("type") == "function" and isinstance(tool.get("function"), dict) for tool in tools
)
if not is_openai_tool_request:
return healthy_deployments
compatible: Final = [d for d in healthy_deployments if _deployment_speaks_openai_protocol(d)]
if not compatible:
return healthy_deployments
return compatible
# Credential params that only one provider family reads, paired with the providers
# that read them. A deployment carrying them while resolving elsewhere is almost
# always a missing route prefix: `model: claude-sonnet-5` with `aws_region_name`

View file

@ -14,6 +14,7 @@ from litellm.router_utils.common_utils import (
_deployment_supports_web_search,
add_model_file_id_mappings,
filter_team_based_models,
filter_tool_protocol_deployments,
filter_web_search_deployments,
format_fallback_outcome_message,
provider_for_generic_call,
@ -374,6 +375,69 @@ class TestFilterWebSearchDeployments:
assert result == deployment
class TestFilterToolProtocolDeployments:
"""Regression tests for BerriAI/litellm#43685: an OpenAI-format tool-calling
request must not be routed to a non-OpenAI-protocol deployment when an
OpenAI-protocol deployment is available in the same model group."""
@pytest.fixture
def mixed_protocol_deployments(self) -> list[dict]:
return [
{
"model_name": "claude-4.6-sonnet",
"litellm_params": {"model": "openai/claude-4.6-sonnet"},
"model_info": {"id": "openai-deployment"},
},
{
"model_name": "claude-4.6-sonnet",
"litellm_params": {"model": "anthropic/claude-4.6-sonnet"},
"model_info": {"id": "anthropic-deployment"},
},
]
@staticmethod
def _openai_tool_kwargs() -> dict:
return {"tools": [{"type": "function", "function": {"name": "get_weather"}}]}
def test_openai_tool_request_keeps_only_openai_protocol(self, mixed_protocol_deployments):
result = filter_tool_protocol_deployments(mixed_protocol_deployments, self._openai_tool_kwargs())
assert [d["model_info"]["id"] for d in result] == ["openai-deployment"]
def test_no_tools_keeps_all(self, mixed_protocol_deployments):
assert filter_tool_protocol_deployments(mixed_protocol_deployments, {}) == mixed_protocol_deployments
def test_empty_tools_keeps_all(self, mixed_protocol_deployments):
assert filter_tool_protocol_deployments(mixed_protocol_deployments, {"tools": []}) == mixed_protocol_deployments
def test_anthropic_format_tools_keeps_all(self, mixed_protocol_deployments):
request_kwargs = {"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}]}
assert (
filter_tool_protocol_deployments(mixed_protocol_deployments, request_kwargs) == mixed_protocol_deployments
)
def test_no_openai_protocol_deployment_keeps_all_for_translation(self):
deployments = [
{"litellm_params": {"model": "anthropic/claude"}, "model_info": {"id": "a1"}},
{"litellm_params": {"model": "bedrock/anthropic.claude"}, "model_info": {"id": "b1"}},
]
result = filter_tool_protocol_deployments(deployments, self._openai_tool_kwargs())
assert result == deployments
def test_unresolvable_provider_is_kept(self, mixed_protocol_deployments):
deployments = mixed_protocol_deployments + [
{"litellm_params": {"model": "no-provider-knows-this-model"}, "model_info": {"id": "mystery"}}
]
result = filter_tool_protocol_deployments(deployments, self._openai_tool_kwargs())
assert [d["model_info"]["id"] for d in result] == ["openai-deployment", "mystery"]
def test_dict_deployment_passthrough(self):
deployment = {"litellm_params": {"model": "anthropic/x"}, "model_info": {"id": "d1"}}
assert filter_tool_protocol_deployments(deployment, self._openai_tool_kwargs()) is deployment
def test_none_kwargs_passthrough(self, mixed_protocol_deployments):
assert filter_tool_protocol_deployments(mixed_protocol_deployments, None) == mixed_protocol_deployments
def test_invalidate_model_group_info_cache():
"""Test that _invalidate_model_group_info_cache clears the LRU cache."""
router = Router(

View file

@ -30599,3 +30599,42 @@ async def test_anthropic_caller_rate_limit_does_not_block_other_credentials(
finally:
await GLOBAL_LOGGING_WORKER.flush()
await client.close()
def _mixed_protocol_model_list():
return [
{
"model_name": "claude-4.6-sonnet",
"litellm_params": {"model": "openai/claude-4.6-sonnet", "api_key": "fake-key"},
},
{
"model_name": "claude-4.6-sonnet",
"litellm_params": {"model": "anthropic/claude-4.6-sonnet", "api_key": "fake-key"},
},
]
def test_router_openai_tool_request_selects_openai_protocol_deployment():
"""BerriAI/litellm#43685: in a mixed-protocol model group, an OpenAI-format
tool-calling request must only select the OpenAI-protocol deployment."""
router = litellm.Router(model_list=_mixed_protocol_model_list(), routing_strategy="simple-shuffle")
tools = [{"type": "function", "function": {"name": "get_weather"}}]
for _ in range(20):
deployment = router.get_available_deployment(
model="claude-4.6-sonnet",
messages=[{"role": "user", "content": "hi"}],
request_kwargs={"tools": tools},
)
assert deployment["litellm_params"]["model"] == "openai/claude-4.6-sonnet"
@pytest.mark.asyncio
async def test_router_openai_tool_request_selects_openai_protocol_deployment_async():
"""BerriAI/litellm#43685: same guarantee on the async deployment-selection path."""
router = litellm.Router(model_list=_mixed_protocol_model_list(), routing_strategy="simple-shuffle")
tools = [{"type": "function", "function": {"name": "get_weather"}}]
for _ in range(20):
deployment = await router.async_get_available_deployment(
model="claude-4.6-sonnet",
messages=[{"role": "user", "content": "hi"}],
request_kwargs={"tools": tools},
)
assert deployment["litellm_params"]["model"] == "openai/claude-4.6-sonnet"