From 2edef875a5a4289541dac4c0af96d20b4689aaa5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:08:58 -0700 Subject: [PATCH] fix(decisions): keep deployments that drop or cannot resolve clips in the media filter --- litellm/decisions/call.py | 16 ++++++--- litellm/router.py | 35 ++++++++++++++----- .../router_code_coverage.py | 2 +- tests/unit/decisions/test_main.py | 16 +++++++-- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 0137ed809b7..871fa8227ea 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -311,14 +311,22 @@ def _takes_videos( ) -def deployment_takes_media( - model: str, custom_llm_provider: str, model_info: Mapping[str, object], *, audio: bool, videos: bool +def deployment_refuses_media( + model: str, + custom_llm_provider: str, + model_info: Mapping[str, object], + call_kwargs: Mapping[str, object], + *, + audio: bool, + videos: bool, ) -> bool: + if _drops_params(call_kwargs): + return False provider_config: Final = get_decisions_provider_config(model, custom_llm_provider) canonical_model: Final = provider_config.canonical_model(model) deployment: Final = _DeploymentCapabilities.model_validate(model_info) - return (not audio or _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) and ( - not videos or _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment) + return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or ( + videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment) ) diff --git a/litellm/router.py b/litellm/router.py index 99e09055c17..c8201a88449 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14886,15 +14886,31 @@ class Router(DeprecatedRouterMembers): return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates @staticmethod - def _decisions_deployment_takes_media( - deployment: DeploymentTypedDict, group: str, *, audio: bool, videos: bool + def _decisions_deployment_refuses_media( + deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool ) -> bool: - from litellm.decisions.call import deployment_takes_media + from litellm.decisions.call import deployment_refuses_media plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict - model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group) - return deployment_takes_media(model, custom_llm_provider, model_info, audio=audio, videos=videos) + deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict + try: + model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group) + return deployment_refuses_media( + model, + custom_llm_provider, + model_info, + {**deployment_params, **request_kwargs}, + audio=audio, + videos=videos, + ) + except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself + verbose_router_logger.debug( + "litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s", + group, + e, + ) + return False @staticmethod def _without_decisions_deployments_refusing_media( @@ -14910,19 +14926,20 @@ class Router(DeprecatedRouterMembers): ): return healthy_deployments - media_capable_deployments: Final = [ + deployments_taking_media: Final = [ deployment for deployment in healthy_deployments - if Router._decisions_deployment_takes_media( + if not Router._decisions_deployment_refuses_media( cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape group, + call_kwargs, audio=audio, videos=videos, ) ] - if not media_capable_deployments: + if not deployments_taking_media: return healthy_deployments - return cast(_RouteDeploymentsT, media_capable_deployments) # cast-ok: same list shape as the input candidates + return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates @staticmethod def _without_declared_text_only_deployments( diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index ce64708a874..1bb67cc60bf 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -184,7 +184,7 @@ ignored_function_names = [ "_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py "_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests "_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py - "_decisions_deployment_takes_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests + "_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests ] diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index a2be68fe4ff..1ecd84c505c 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3870,10 +3870,12 @@ _MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType( ) -def _group_member(group: str, model: str, key: str, weight: int, model_info: Mapping[str, object]) -> dict[str, object]: +def _group_member( + group: str, model: str, key: str, weight: int, model_info: Mapping[str, object], **params: object +) -> dict[str, object]: return { "model_name": group, - "litellm_params": {"model": model, "api_key": key, "weight": weight}, + "litellm_params": {"model": model, "api_key": key, "weight": weight, **params}, "model_info": {"id": f"{group}-{key}", **model_info}, } @@ -3905,6 +3907,10 @@ async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_o _group_member("listed", listed, "takes", 0, {}), _group_member("none", unlisted, "refuses", 1, {}), _group_member("none", listed, "refuses", 1, {"supports_audio_input": False}), + _group_member("drops", unlisted, "drops", 1, {}, drop_params=True), + _group_member("drops", listed, "takes", 0, {}), + _group_member("unresolvable", "databricks/not a serving endpoint", "unresolvable", 0, {}), + _group_member("unresolvable", listed, "takes", 1, {}), ] ) call: Final = getattr(router, surface) @@ -3913,11 +3919,15 @@ async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_o response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media) return await response if asyncio.iscoroutine(response) else response - for group in ("unlisted", "listed"): + for group in ("unlisted", "listed", "unresolvable"): await send(group, audio=_AUDIO_CLIPS) assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes" assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS) + await send("drops", audio=_AUDIO_CLIPS) + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer drops" + assert "audio" not in json.loads(respx_mock.calls.last.request.content) + await send("unlisted") assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses"