From 670c6392eb3573bc765d9b62f164ecc50e364f75 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:18:35 -0700 Subject: [PATCH] fix(decisions): route clips in a load-balanced group to a deployment that takes them A Decisions or SystemOne request carrying audio or video now skips group deployments whose own flags or the cost map refuse that media, and keeps the whole group when none takes it, so the 400 still names the media. --- litellm/decisions/call.py | 45 +++++++++---- litellm/router.py | 56 ++++++++++++++-- .../router_code_coverage.py | 2 + tests/unit/decisions/test_main.py | 66 +++++++++++++++++++ 4 files changed, 153 insertions(+), 16 deletions(-) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 119b524703e..6d5b63e4007 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -291,6 +291,37 @@ class _ResolvedSupport: videos: bool +def _takes_audio( + provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities +) -> bool: + return provider_config.supports_audio_input( + model, + custom_llm_provider, + deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider), + ) + + +def _takes_videos( + provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities +) -> bool: + return provider_config.supports_video_input( + model, + custom_llm_provider, + deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider), + ) + + +def deployment_takes_media( + model: str, custom_llm_provider: str, model_info: Mapping[str, object], *, audio: bool, videos: bool +) -> bool: + provider_config: Final = _provider_config(model, custom_llm_provider) + canonical_model: Final = provider_config.canonical_model(model) + deployment: Final = _DeploymentCapabilities.model_validate(model_info) + return (not audio or _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) and ( + not videos or _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment) + ) + + def _resolved_support( provider_config: BaseDecisionsConfig, model: str, @@ -304,18 +335,8 @@ def _resolved_support( box_questions=provider_config.supports_box_questions( model, custom_llm_provider, headers, deployment.supports_box_questions ), - audio=bool(fields.audio) - and provider_config.supports_audio_input( - model, - custom_llm_provider, - deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider), - ), - videos=bool(fields.videos) - and provider_config.supports_video_input( - model, - custom_llm_provider, - deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider), - ), + audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment), + videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment), ) diff --git a/litellm/router.py b/litellm/router.py index b63130c3313..392b70023b3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -606,6 +606,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"}) _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"}) _video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None) _routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None) +_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"}) _ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType( { "adecisions": ProviderConfigManager.get_provider_decisions_config, @@ -14531,6 +14532,45 @@ class Router: return healthy_deployments return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates + @staticmethod + def _decisions_deployment_takes_media( + deployment: DeploymentTypedDict, group: str, *, audio: bool, videos: bool + ) -> bool: + from litellm.decisions.call import deployment_takes_media + + plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts + model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict + model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group) + return deployment_takes_media(model, custom_llm_provider, model_info, audio=audio, videos=videos) + + @staticmethod + def _without_decisions_deployments_refusing_media( + healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None + ) -> _RouteDeploymentsT: + call_kwargs: Final = request_kwargs or {} + audio: Final = bool(call_kwargs.get("audio")) + videos: Final = bool(call_kwargs.get("videos")) + if ( + _routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES + or not (audio or videos) + or isinstance(healthy_deployments, Mapping) + ): + return healthy_deployments + + media_capable_deployments: Final = [ + deployment + for deployment in healthy_deployments + if Router._decisions_deployment_takes_media( + cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape + group, + audio=audio, + videos=videos, + ) + ] + if not media_capable_deployments: + return healthy_deployments + return cast(_RouteDeploymentsT, media_capable_deployments) # cast-ok: same list shape as the input candidates + @staticmethod def _without_declared_text_only_deployments( healthy_deployments: _RouteDeploymentsT, @@ -15142,9 +15182,13 @@ class Router: verbose_router_logger.debug("healthy_deployments after web search filter: %s", healthy_deployments) healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows - self._without_deployments_that_cannot_serve_route( - healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + self._without_decisions_deployments_refusing_media( + self._without_deployments_that_cannot_serve_route( + healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + model, + ), model, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: request kwargs are a plain dict ) ) @@ -16412,9 +16456,13 @@ class Router: request_kwargs=request_kwargs, ) healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows - self._without_deployments_that_cannot_serve_route( - healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + self._without_decisions_deployments_refusing_media( + self._without_deployments_that_cannot_serve_route( + healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type + model, + ), model, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: request kwargs are a plain dict ) ) strategy, strategy_selector = self._get_routing_context(model, request_kwargs) diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 9fb3633d53c..89bd5c193ac 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -181,6 +181,8 @@ ignored_function_names = [ "_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py "_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py "_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests + "_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py + "_decisions_deployment_takes_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests ] diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index e90b6520f90..e63e713535b 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3860,6 +3860,72 @@ async def test_another_deployments_mode_never_switches_off_a_deployments_own_med assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS)) +_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType( + { + "adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS}, + "asystemone": {"state": "review", "questions": _QUESTIONS}, + "systemone": {"state": "review", "questions": _QUESTIONS}, + } +) + + +def _group_member(group: str, model: str, key: str, weight: int, model_info: Mapping[str, object]) -> dict[str, object]: + return { + "model_name": group, + "litellm_params": {"model": model, "api_key": key, "weight": weight}, + "model_info": {"id": f"{group}-{key}", **model_info}, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES)) +async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + monkeypatch.setitem( + litellm.model_cost, + "cloudflare/@cf/cloudflare/clef-omni", + {"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True}, + ) + respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE) + respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond( + json=_RESPONSE + ) + unlisted: Final = "openrouter/example/omni-preview" + listed: Final = "cloudflare/clef-omni" + router: Final = litellm.Router( + model_list=[ + _group_member("unlisted", unlisted, "refuses", 1, {}), + _group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}), + _group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}), + _group_member("listed", listed, "takes", 0, {}), + _group_member("none", unlisted, "refuses", 1, {}), + _group_member("none", listed, "refuses", 1, {"supports_audio_input": False}), + ] + ) + call: Final = getattr(router, surface) + + async def send(group: str, **media: object) -> object: + response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media) + return await response if asyncio.iscoroutine(response) else response + + for group in ("unlisted", "listed"): + await send(group, audio=_AUDIO_CLIPS) + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes" + assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS) + + await send("unlisted") + assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses" + + sent_before_refusal: Final = len(respx_mock.calls) + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")): + await send("none", audio=_AUDIO_CLIPS) + assert len(respx_mock.calls) == sent_before_refusal + + @pytest.mark.asyncio async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None