fix(decisions): keep deployments that drop or cannot resolve clips in the media filter

This commit is contained in:
mateo-berri 2026-10-30 19:08:58 -07:00
parent e978f589c0
commit 2edef875a5
4 changed files with 52 additions and 17 deletions

View file

@ -311,14 +311,22 @@ def _takes_videos(
)
def deployment_takes_media(
model: str, custom_llm_provider: str, model_info: Mapping[str, object], *, audio: bool, videos: bool
def deployment_refuses_media(
model: str,
custom_llm_provider: str,
model_info: Mapping[str, object],
call_kwargs: Mapping[str, object],
*,
audio: bool,
videos: bool,
) -> bool:
if _drops_params(call_kwargs):
return False
provider_config: Final = get_decisions_provider_config(model, custom_llm_provider)
canonical_model: Final = provider_config.canonical_model(model)
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
return (not audio or _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) and (
not videos or _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or (
videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
)

View file

@ -14886,15 +14886,31 @@ class Router(DeprecatedRouterMembers):
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
@staticmethod
def _decisions_deployment_takes_media(
deployment: DeploymentTypedDict, group: str, *, audio: bool, videos: bool
def _decisions_deployment_refuses_media(
deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool
) -> bool:
from litellm.decisions.call import deployment_takes_media
from litellm.decisions.call import deployment_refuses_media
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
return deployment_takes_media(model, custom_llm_provider, model_info, audio=audio, videos=videos)
deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict
try:
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
return deployment_refuses_media(
model,
custom_llm_provider,
model_info,
{**deployment_params, **request_kwargs},
audio=audio,
videos=videos,
)
except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself
verbose_router_logger.debug(
"litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s",
group,
e,
)
return False
@staticmethod
def _without_decisions_deployments_refusing_media(
@ -14910,19 +14926,20 @@ class Router(DeprecatedRouterMembers):
):
return healthy_deployments
media_capable_deployments: Final = [
deployments_taking_media: Final = [
deployment
for deployment in healthy_deployments
if Router._decisions_deployment_takes_media(
if not Router._decisions_deployment_refuses_media(
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
group,
call_kwargs,
audio=audio,
videos=videos,
)
]
if not media_capable_deployments:
if not deployments_taking_media:
return healthy_deployments
return cast(_RouteDeploymentsT, media_capable_deployments) # cast-ok: same list shape as the input candidates
return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates
@staticmethod
def _without_declared_text_only_deployments(

View file

@ -184,7 +184,7 @@ ignored_function_names = [
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
"_decisions_deployment_takes_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
"_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
]

View file

@ -3870,10 +3870,12 @@ _MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType(
)
def _group_member(group: str, model: str, key: str, weight: int, model_info: Mapping[str, object]) -> dict[str, object]:
def _group_member(
group: str, model: str, key: str, weight: int, model_info: Mapping[str, object], **params: object
) -> dict[str, object]:
return {
"model_name": group,
"litellm_params": {"model": model, "api_key": key, "weight": weight},
"litellm_params": {"model": model, "api_key": key, "weight": weight, **params},
"model_info": {"id": f"{group}-{key}", **model_info},
}
@ -3905,6 +3907,10 @@ async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_o
_group_member("listed", listed, "takes", 0, {}),
_group_member("none", unlisted, "refuses", 1, {}),
_group_member("none", listed, "refuses", 1, {"supports_audio_input": False}),
_group_member("drops", unlisted, "drops", 1, {}, drop_params=True),
_group_member("drops", listed, "takes", 0, {}),
_group_member("unresolvable", "databricks/not a serving endpoint", "unresolvable", 0, {}),
_group_member("unresolvable", listed, "takes", 1, {}),
]
)
call: Final = getattr(router, surface)
@ -3913,11 +3919,15 @@ async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_o
response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media)
return await response if asyncio.iscoroutine(response) else response
for group in ("unlisted", "listed"):
for group in ("unlisted", "listed", "unresolvable"):
await send(group, audio=_AUDIO_CLIPS)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
await send("drops", audio=_AUDIO_CLIPS)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops"
assert "audio" not in json.loads(respx_mock.calls.last.request.content)
await send("unlisted")
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses"