mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 8b8c9e0f47 into 38b4d31447
This commit is contained in:
commit
fb7ccfed3d
4 changed files with 169 additions and 2 deletions
|
|
@ -215,6 +215,7 @@ from litellm.router_utils.common_utils import (
|
|||
as_retry_skipped_deployment_ids,
|
||||
deployments_the_caller_may_use,
|
||||
filter_team_based_models,
|
||||
filter_tool_protocol_deployments,
|
||||
filter_web_search_deployments,
|
||||
format_fallback_outcome_message,
|
||||
format_no_fallback_group_message,
|
||||
|
|
@ -15032,6 +15033,10 @@ class Router:
|
|||
model,
|
||||
)
|
||||
)
|
||||
healthy_deployments = filter_tool_protocol_deployments( # rebind-ok: progressive narrowing of the candidate list through the router filter pipeline; rebinding is the established pattern in this function
|
||||
healthy_deployments=healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any]
|
||||
request_kwargs=request_kwargs, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any]
|
||||
)
|
||||
|
||||
if isinstance(healthy_deployments, dict):
|
||||
if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
|
||||
|
|
@ -16346,11 +16351,16 @@ class Router:
|
|||
self._filter_blocked_deployments(healthy_deployments), routing_kwargs
|
||||
)
|
||||
|
||||
healthy_deployments = filter_tool_protocol_deployments( # rebind-ok: progressive narrowing of the candidate list through the router filter pipeline; rebinding is the established pattern in this function
|
||||
healthy_deployments=healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any]
|
||||
request_kwargs=request_kwargs, # pyright: ignore[reportUnknownArgumentType] # router passes bare dicts; callee accepts dict[str, Any]
|
||||
)
|
||||
|
||||
# filter pre-call checks
|
||||
if self.enable_pre_call_checks and (messages is not None or input is not None):
|
||||
healthy_deployments = self._pre_call_checks(
|
||||
model=model,
|
||||
healthy_deployments=healthy_deployments,
|
||||
healthy_deployments=healthy_deployments, # pyright: ignore[reportArgumentType] # filtered result stays a bare-dict list for the untyped callee
|
||||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import hashlib
|
|||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeVar
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeVar, cast # noqa: TID251 # Any for basedpyright; cast is validated
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -424,6 +424,60 @@ def filter_web_search_deployments(
|
|||
return final_deployments
|
||||
|
||||
|
||||
# Providers that natively speak the OpenAI chat-completions wire protocol, so an
|
||||
# OpenAI-format tool-calling contract reaches them without cross-protocol
|
||||
# translation. Same family litellm uses for OpenAI-protocol param handling in
|
||||
# `add_provider_specific_params_to_optional_params` (litellm/utils.py).
|
||||
_OPENAI_NATIVE_PROTOCOL_PROVIDERS: Final = frozenset(
|
||||
["openai", "azure", "text-completion-openai"]
|
||||
+ cast("list[str]", litellm.openai_compatible_providers) # cast-ok: provider names are strings
|
||||
)
|
||||
|
||||
|
||||
def _deployment_speaks_openai_protocol(deployment: dict[str, Any]) -> bool: # mutable-ok: read-only, never mutated
|
||||
litellm_params: Final[dict[str, Any]] = deployment.get("litellm_params") or {} # mutable-ok: read-only
|
||||
provider: Final = provider_for_generic_call(litellm_params)
|
||||
if provider is None:
|
||||
return True
|
||||
return provider in _OPENAI_NATIVE_PROTOCOL_PROVIDERS
|
||||
|
||||
|
||||
def filter_tool_protocol_deployments(
|
||||
healthy_deployments: list[dict[str, Any]] | dict[str, Any], # mutable-ok: router candidate type
|
||||
request_kwargs: dict[str, Any] | None = None, # mutable-ok: OptionalParams dict
|
||||
) -> list[dict[str, Any]] | dict[str, Any]: # mutable-ok: returns input or narrowed list
|
||||
"""
|
||||
Keep only OpenAI-protocol deployments for OpenAI-format tool-calling requests.
|
||||
|
||||
In a mixed-protocol model group (e.g. an `openai/...` and an `anthropic/...`
|
||||
deployment behind one model name), sending OpenAI-format `tools` to a
|
||||
non-OpenAI-protocol deployment forces a cross-protocol translation of the
|
||||
tool contract that third-party upstreams can reject, causing intermittent
|
||||
tool-calling failures (BerriAI/litellm#43685).
|
||||
|
||||
The tool shape identifies the inbound protocol: only
|
||||
`{"type": "function", "function": {...}}` triggers filtering, so
|
||||
Anthropic-protocol inbound requests (proxy `/v1/messages`) are unaffected.
|
||||
|
||||
When no deployment speaks the OpenAI protocol, candidates are left
|
||||
untouched so requests that succeed today via provider translation keep
|
||||
working; filtering only narrows selection when a same-protocol alternative
|
||||
exists. Deployments whose provider cannot be resolved are kept.
|
||||
"""
|
||||
if request_kwargs is None or isinstance(healthy_deployments, dict):
|
||||
return healthy_deployments
|
||||
tools: Final[list[dict[str, Any]]] = request_kwargs.get("tools") or [] # mutable-ok: read-only
|
||||
is_openai_tool_request: Final = any(
|
||||
tool.get("type") == "function" and isinstance(tool.get("function"), dict) for tool in tools
|
||||
)
|
||||
if not is_openai_tool_request:
|
||||
return healthy_deployments
|
||||
compatible: Final = [d for d in healthy_deployments if _deployment_speaks_openai_protocol(d)]
|
||||
if not compatible:
|
||||
return healthy_deployments
|
||||
return compatible
|
||||
|
||||
|
||||
# Credential params that only one provider family reads, paired with the providers
|
||||
# that read them. A deployment carrying them while resolving elsewhere is almost
|
||||
# always a missing route prefix: `model: claude-sonnet-5` with `aws_region_name`
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from litellm.router_utils.common_utils import (
|
|||
_deployment_supports_web_search,
|
||||
add_model_file_id_mappings,
|
||||
filter_team_based_models,
|
||||
filter_tool_protocol_deployments,
|
||||
filter_web_search_deployments,
|
||||
format_fallback_outcome_message,
|
||||
provider_for_generic_call,
|
||||
|
|
@ -374,6 +375,69 @@ class TestFilterWebSearchDeployments:
|
|||
assert result == deployment
|
||||
|
||||
|
||||
class TestFilterToolProtocolDeployments:
|
||||
"""Regression tests for BerriAI/litellm#43685: an OpenAI-format tool-calling
|
||||
request must not be routed to a non-OpenAI-protocol deployment when an
|
||||
OpenAI-protocol deployment is available in the same model group."""
|
||||
|
||||
@pytest.fixture
|
||||
def mixed_protocol_deployments(self) -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"model_name": "claude-4.6-sonnet",
|
||||
"litellm_params": {"model": "openai/claude-4.6-sonnet"},
|
||||
"model_info": {"id": "openai-deployment"},
|
||||
},
|
||||
{
|
||||
"model_name": "claude-4.6-sonnet",
|
||||
"litellm_params": {"model": "anthropic/claude-4.6-sonnet"},
|
||||
"model_info": {"id": "anthropic-deployment"},
|
||||
},
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _openai_tool_kwargs() -> dict:
|
||||
return {"tools": [{"type": "function", "function": {"name": "get_weather"}}]}
|
||||
|
||||
def test_openai_tool_request_keeps_only_openai_protocol(self, mixed_protocol_deployments):
|
||||
result = filter_tool_protocol_deployments(mixed_protocol_deployments, self._openai_tool_kwargs())
|
||||
assert [d["model_info"]["id"] for d in result] == ["openai-deployment"]
|
||||
|
||||
def test_no_tools_keeps_all(self, mixed_protocol_deployments):
|
||||
assert filter_tool_protocol_deployments(mixed_protocol_deployments, {}) == mixed_protocol_deployments
|
||||
|
||||
def test_empty_tools_keeps_all(self, mixed_protocol_deployments):
|
||||
assert filter_tool_protocol_deployments(mixed_protocol_deployments, {"tools": []}) == mixed_protocol_deployments
|
||||
|
||||
def test_anthropic_format_tools_keeps_all(self, mixed_protocol_deployments):
|
||||
request_kwargs = {"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}]}
|
||||
assert (
|
||||
filter_tool_protocol_deployments(mixed_protocol_deployments, request_kwargs) == mixed_protocol_deployments
|
||||
)
|
||||
|
||||
def test_no_openai_protocol_deployment_keeps_all_for_translation(self):
|
||||
deployments = [
|
||||
{"litellm_params": {"model": "anthropic/claude"}, "model_info": {"id": "a1"}},
|
||||
{"litellm_params": {"model": "bedrock/anthropic.claude"}, "model_info": {"id": "b1"}},
|
||||
]
|
||||
result = filter_tool_protocol_deployments(deployments, self._openai_tool_kwargs())
|
||||
assert result == deployments
|
||||
|
||||
def test_unresolvable_provider_is_kept(self, mixed_protocol_deployments):
|
||||
deployments = mixed_protocol_deployments + [
|
||||
{"litellm_params": {"model": "no-provider-knows-this-model"}, "model_info": {"id": "mystery"}}
|
||||
]
|
||||
result = filter_tool_protocol_deployments(deployments, self._openai_tool_kwargs())
|
||||
assert [d["model_info"]["id"] for d in result] == ["openai-deployment", "mystery"]
|
||||
|
||||
def test_dict_deployment_passthrough(self):
|
||||
deployment = {"litellm_params": {"model": "anthropic/x"}, "model_info": {"id": "d1"}}
|
||||
assert filter_tool_protocol_deployments(deployment, self._openai_tool_kwargs()) is deployment
|
||||
|
||||
def test_none_kwargs_passthrough(self, mixed_protocol_deployments):
|
||||
assert filter_tool_protocol_deployments(mixed_protocol_deployments, None) == mixed_protocol_deployments
|
||||
|
||||
|
||||
def test_invalidate_model_group_info_cache():
|
||||
"""Test that _invalidate_model_group_info_cache clears the LRU cache."""
|
||||
router = Router(
|
||||
|
|
|
|||
|
|
@ -30599,3 +30599,42 @@ async def test_anthropic_caller_rate_limit_does_not_block_other_credentials(
|
|||
finally:
|
||||
await GLOBAL_LOGGING_WORKER.flush()
|
||||
await client.close()
|
||||
def _mixed_protocol_model_list():
|
||||
return [
|
||||
{
|
||||
"model_name": "claude-4.6-sonnet",
|
||||
"litellm_params": {"model": "openai/claude-4.6-sonnet", "api_key": "fake-key"},
|
||||
},
|
||||
{
|
||||
"model_name": "claude-4.6-sonnet",
|
||||
"litellm_params": {"model": "anthropic/claude-4.6-sonnet", "api_key": "fake-key"},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_router_openai_tool_request_selects_openai_protocol_deployment():
|
||||
"""BerriAI/litellm#43685: in a mixed-protocol model group, an OpenAI-format
|
||||
tool-calling request must only select the OpenAI-protocol deployment."""
|
||||
router = litellm.Router(model_list=_mixed_protocol_model_list(), routing_strategy="simple-shuffle")
|
||||
tools = [{"type": "function", "function": {"name": "get_weather"}}]
|
||||
for _ in range(20):
|
||||
deployment = router.get_available_deployment(
|
||||
model="claude-4.6-sonnet",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={"tools": tools},
|
||||
)
|
||||
assert deployment["litellm_params"]["model"] == "openai/claude-4.6-sonnet"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_openai_tool_request_selects_openai_protocol_deployment_async():
|
||||
"""BerriAI/litellm#43685: same guarantee on the async deployment-selection path."""
|
||||
router = litellm.Router(model_list=_mixed_protocol_model_list(), routing_strategy="simple-shuffle")
|
||||
tools = [{"type": "function", "function": {"name": "get_weather"}}]
|
||||
for _ in range(20):
|
||||
deployment = await router.async_get_available_deployment(
|
||||
model="claude-4.6-sonnet",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
request_kwargs={"tools": tools},
|
||||
)
|
||||
assert deployment["litellm_params"]["model"] == "openai/claude-4.6-sonnet"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue