diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 119b524703e..6d5b63e4007 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -291,6 +291,37 @@ class _ResolvedSupport: videos: bool +def _takes_audio( + provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities +) -> bool: + return provider_config.supports_audio_input( + model, + custom_llm_provider, + deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider), + ) + + +def _takes_videos( + provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities +) -> bool: + return provider_config.supports_video_input( + model, + custom_llm_provider, + deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider), + ) + + +def deployment_takes_media( + model: str, custom_llm_provider: str, model_info: Mapping[str, object], *, audio: bool, videos: bool +) -> bool: + provider_config: Final = _provider_config(model, custom_llm_provider) + canonical_model: Final = provider_config.canonical_model(model) + deployment: Final = _DeploymentCapabilities.model_validate(model_info) + return (not audio or _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) and ( + not videos or _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment) + ) + + def _resolved_support( provider_config: BaseDecisionsConfig, model: str, @@ -304,18 +335,8 @@ def _resolved_support( box_questions=provider_config.supports_box_questions( model, custom_llm_provider, headers, deployment.supports_box_questions ), - audio=bool(fields.audio) - and provider_config.supports_audio_input( - model, - custom_llm_provider, - deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider), - ), - videos=bool(fields.videos) - and provider_config.supports_video_input( - model, - custom_llm_provider, - deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider), - ), + audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment), + videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment), ) diff --git a/litellm/router.py b/litellm/router.py index b63130c3313..392b70023b3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -606,6 +606,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"}) _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"}) _video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None) _routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None) +_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"}) _ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType( { "adecisions": ProviderConfigManager.get_provider_decisions_config, @@ -14531,6 +14532,45 @@ class Router: return healthy_deployments return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates + @staticmethod + def _decisions_deployment_takes_media( + deployment: DeploymentTypedDict, group: str, *, audio: bool, videos: bool + ) -> bool: + from litellm.decisions.call import deployment_takes_media + + plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts + model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict + model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group) + return deployment_takes_media(model, custom_llm_provider, model_info, audio=audio, videos=videos) + + @staticmethod + def _without_decisions_deployments_refusing_media( + healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None + ) -> _RouteDeploymentsT: + call_kwargs: Final = request_kwargs or {} + audio: Final = bool(call_kwargs.get("audio")) + videos: Final = bool(call_kwargs.get("videos")) + if ( + _routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES + or not (audio or videos) + or isinstance(healthy_deployments, Mapping) + ): + return healthy_deployments + + media_capable_deployments: Final = [ + deployment + for deployment in healthy_deployments + if Router._decisions_deployment_takes_media( + cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape + group, + audio=audio, + videos=videos, + ) + ] + if not media_capable_deployments: + return healthy_deployments + return cast(_RouteDeploymentsT, media_capable_deployments) # cast-ok: same list shape as the input candidates + @staticmethod def _without_declared_text_only_deployments( healthy_deployments: _RouteDeploymentsT, @@ -15142,9 +15182,13 @@ class Router: verbose_router_logger.debug("healthy_deployments after web search filter: %s", healthy_deployments) healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows - self._without_deployments_that_cannot_serve_route( - healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + self._without_decisions_deployments_refusing_media( + self._without_deployments_that_cannot_serve_route( + healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + model, + ), model, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: request kwargs are a plain dict ) ) @@ -16412,9 +16456,13 @@ class Router: request_kwargs=request_kwargs, ) healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows - self._without_deployments_that_cannot_serve_route( - healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + self._without_decisions_deployments_refusing_media( + self._without_deployments_that_cannot_serve_route( + healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + model, + ), model, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: request kwargs are a plain dict ) ) strategy, strategy_selector = self._get_routing_context(model, request_kwargs) diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 9fb3633d53c..89bd5c193ac 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -181,6 +181,8 @@ ignored_function_names = [ "_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py "_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py "_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests + "_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py + "_decisions_deployment_takes_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests ] diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index e90b6520f90..e63e713535b 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3860,6 +3860,72 @@ async def test_another_deployments_mode_never_switches_off_a_deployments_own_med assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS)) +_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType( + { + "adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS}, + "asystemone": {"state": "review", "questions": _QUESTIONS}, + "systemone": {"state": "review", "questions": _QUESTIONS}, + } +) + + +def _group_member(group: str, model: str, key: str, weight: int, model_info: Mapping[str, object]) -> dict[str, object]: + return { + "model_name": group, + "litellm_params": {"model": model, "api_key": key, "weight": weight}, + "model_info": {"id": f"{group}-{key}", **model_info}, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES)) +async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + monkeypatch.setitem( + litellm.model_cost, + "cloudflare/@cf/cloudflare/clef-omni", + {"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True}, + ) + respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond( + json=_RESPONSE + ) + unlisted: Final = "openrouter/example/omni-preview" + listed: Final = "cloudflare/clef-omni" + router: Final = litellm.Router( + model_list=[ + _group_member("unlisted", unlisted, "refuses", 1, {}), + _group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}), + _group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}), + _group_member("listed", listed, "takes", 0, {}), + _group_member("none", unlisted, "refuses", 1, {}), + _group_member("none", listed, "refuses", 1, {"supports_audio_input": False}), + ] + ) + call: Final = getattr(router, surface) + + async def send(group: str, **media: object) -> object: + response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media) + return await response if asyncio.iscoroutine(response) else response + + for group in ("unlisted", "listed"): + await send(group, audio=_AUDIO_CLIPS) + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes" + assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS) + + await send("unlisted") + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses" + + sent_before_refusal: Final = len(respx_mock.calls) + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")): + await send("none", audio=_AUDIO_CLIPS) + assert len(respx_mock.calls) == sent_before_refusal + + @pytest.mark.asyncio async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None