mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(decisions): keep deployments that drop or cannot resolve clips in the media filter
This commit is contained in:
parent
e978f589c0
commit
2edef875a5
4 changed files with 52 additions and 17 deletions
|
|
@ -311,14 +311,22 @@ def _takes_videos(
|
|||
)
|
||||
|
||||
|
||||
def deployment_takes_media(
|
||||
model: str, custom_llm_provider: str, model_info: Mapping[str, object], *, audio: bool, videos: bool
|
||||
def deployment_refuses_media(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
model_info: Mapping[str, object],
|
||||
call_kwargs: Mapping[str, object],
|
||||
*,
|
||||
audio: bool,
|
||||
videos: bool,
|
||||
) -> bool:
|
||||
if _drops_params(call_kwargs):
|
||||
return False
|
||||
provider_config: Final = get_decisions_provider_config(model, custom_llm_provider)
|
||||
canonical_model: Final = provider_config.canonical_model(model)
|
||||
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
|
||||
return (not audio or _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) and (
|
||||
not videos or _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
|
||||
return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or (
|
||||
videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14886,15 +14886,31 @@ class Router(DeprecatedRouterMembers):
|
|||
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _decisions_deployment_takes_media(
|
||||
deployment: DeploymentTypedDict, group: str, *, audio: bool, videos: bool
|
||||
def _decisions_deployment_refuses_media(
|
||||
deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool
|
||||
) -> bool:
|
||||
from litellm.decisions.call import deployment_takes_media
|
||||
from litellm.decisions.call import deployment_refuses_media
|
||||
|
||||
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
|
||||
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
|
||||
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
|
||||
return deployment_takes_media(model, custom_llm_provider, model_info, audio=audio, videos=videos)
|
||||
deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict
|
||||
try:
|
||||
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
|
||||
return deployment_refuses_media(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
model_info,
|
||||
{**deployment_params, **request_kwargs},
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself
|
||||
verbose_router_logger.debug(
|
||||
"litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s",
|
||||
group,
|
||||
e,
|
||||
)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _without_decisions_deployments_refusing_media(
|
||||
|
|
@ -14910,19 +14926,20 @@ class Router(DeprecatedRouterMembers):
|
|||
):
|
||||
return healthy_deployments
|
||||
|
||||
media_capable_deployments: Final = [
|
||||
deployments_taking_media: Final = [
|
||||
deployment
|
||||
for deployment in healthy_deployments
|
||||
if Router._decisions_deployment_takes_media(
|
||||
if not Router._decisions_deployment_refuses_media(
|
||||
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
|
||||
group,
|
||||
call_kwargs,
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
]
|
||||
if not media_capable_deployments:
|
||||
if not deployments_taking_media:
|
||||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, media_capable_deployments) # cast-ok: same list shape as the input candidates
|
||||
return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _without_declared_text_only_deployments(
|
||||
|
|
|
|||
|
|
@ -184,7 +184,7 @@ ignored_function_names = [
|
|||
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
|
||||
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
|
||||
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
|
||||
"_decisions_deployment_takes_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
|
||||
"_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3870,10 +3870,12 @@ _MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
def _group_member(group: str, model: str, key: str, weight: int, model_info: Mapping[str, object]) -> dict[str, object]:
|
||||
def _group_member(
|
||||
group: str, model: str, key: str, weight: int, model_info: Mapping[str, object], **params: object
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": group,
|
||||
"litellm_params": {"model": model, "api_key": key, "weight": weight},
|
||||
"litellm_params": {"model": model, "api_key": key, "weight": weight, **params},
|
||||
"model_info": {"id": f"{group}-{key}", **model_info},
|
||||
}
|
||||
|
||||
|
|
@ -3905,6 +3907,10 @@ async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_o
|
|||
_group_member("listed", listed, "takes", 0, {}),
|
||||
_group_member("none", unlisted, "refuses", 1, {}),
|
||||
_group_member("none", listed, "refuses", 1, {"supports_audio_input": False}),
|
||||
_group_member("drops", unlisted, "drops", 1, {}, drop_params=True),
|
||||
_group_member("drops", listed, "takes", 0, {}),
|
||||
_group_member("unresolvable", "databricks/not a serving endpoint", "unresolvable", 0, {}),
|
||||
_group_member("unresolvable", listed, "takes", 1, {}),
|
||||
]
|
||||
)
|
||||
call: Final = getattr(router, surface)
|
||||
|
|
@ -3913,11 +3919,15 @@ async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_o
|
|||
response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media)
|
||||
return await response if asyncio.iscoroutine(response) else response
|
||||
|
||||
for group in ("unlisted", "listed"):
|
||||
for group in ("unlisted", "listed", "unresolvable"):
|
||||
await send(group, audio=_AUDIO_CLIPS)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
|
||||
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
|
||||
|
||||
await send("drops", audio=_AUDIO_CLIPS)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
|
||||
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
|
||||
|
||||
await send("unlisted")
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue