fix(decisions): route clips in a load-balanced group to a deployment that takes them

A Decisions or SystemOne request carrying audio or video now skips group deployments whose own flags or the cost map refuse that media, and keeps the whole group when none takes it, so the 400 still names the media.
This commit is contained in:
mateo-berri 2026-10-30 09:18:35 -07:00
parent c32f224461
commit 670c6392eb
4 changed files with 153 additions and 16 deletions

View file

@ -291,6 +291,37 @@ class _ResolvedSupport:
videos: bool
def _takes_audio(
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
) -> bool:
return provider_config.supports_audio_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
)
def _takes_videos(
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
) -> bool:
return provider_config.supports_video_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
)
def deployment_takes_media(
model: str, custom_llm_provider: str, model_info: Mapping[str, object], *, audio: bool, videos: bool
) -> bool:
provider_config: Final = _provider_config(model, custom_llm_provider)
canonical_model: Final = provider_config.canonical_model(model)
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
return (not audio or _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) and (
not videos or _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
)
def _resolved_support(
provider_config: BaseDecisionsConfig,
model: str,
@ -304,18 +335,8 @@ def _resolved_support(
box_questions=provider_config.supports_box_questions(
model, custom_llm_provider, headers, deployment.supports_box_questions
),
audio=bool(fields.audio)
and provider_config.supports_audio_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
),
videos=bool(fields.videos)
and provider_config.supports_video_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
),
audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment),
videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment),
)

View file

@ -606,6 +606,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"})
_STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None)
_routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None)
_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"})
_ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType(
{
"adecisions": ProviderConfigManager.get_provider_decisions_config,
@ -14531,6 +14532,45 @@ class Router:
return healthy_deployments
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
@staticmethod
def _decisions_deployment_takes_media(
deployment: DeploymentTypedDict, group: str, *, audio: bool, videos: bool
) -> bool:
from litellm.decisions.call import deployment_takes_media
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
return deployment_takes_media(model, custom_llm_provider, model_info, audio=audio, videos=videos)
@staticmethod
def _without_decisions_deployments_refusing_media(
healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None
) -> _RouteDeploymentsT:
call_kwargs: Final = request_kwargs or {}
audio: Final = bool(call_kwargs.get("audio"))
videos: Final = bool(call_kwargs.get("videos"))
if (
_routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES
or not (audio or videos)
or isinstance(healthy_deployments, Mapping)
):
return healthy_deployments
media_capable_deployments: Final = [
deployment
for deployment in healthy_deployments
if Router._decisions_deployment_takes_media(
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
group,
audio=audio,
videos=videos,
)
]
if not media_capable_deployments:
return healthy_deployments
return cast(_RouteDeploymentsT, media_capable_deployments) # cast-ok: same list shape as the input candidates
@staticmethod
def _without_declared_text_only_deployments(
healthy_deployments: _RouteDeploymentsT,
@ -15142,9 +15182,13 @@ class Router:
verbose_router_logger.debug("healthy_deployments after web search filter: %s", healthy_deployments)
healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows
self._without_deployments_that_cannot_serve_route(
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
self._without_decisions_deployments_refusing_media(
self._without_deployments_that_cannot_serve_route(
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
model,
),
model,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: request kwargs are a plain dict
)
)
@ -16412,9 +16456,13 @@ class Router:
request_kwargs=request_kwargs,
)
healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows
self._without_deployments_that_cannot_serve_route(
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
self._without_decisions_deployments_refusing_media(
self._without_deployments_that_cannot_serve_route(
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
model,
),
model,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: request kwargs are a plain dict
)
)
strategy, strategy_selector = self._get_routing_context(model, request_kwargs)

View file

@ -181,6 +181,8 @@ ignored_function_names = [
"_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
"_decisions_deployment_takes_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
]

View file

@ -3860,6 +3860,72 @@ async def test_another_deployments_mode_never_switches_off_a_deployments_own_med
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType(
{
"adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS},
"asystemone": {"state": "review", "questions": _QUESTIONS},
"systemone": {"state": "review", "questions": _QUESTIONS},
}
)
def _group_member(group: str, model: str, key: str, weight: int, model_info: Mapping[str, object]) -> dict[str, object]:
return {
"model_name": group,
"litellm_params": {"model": model, "api_key": key, "weight": weight},
"model_info": {"id": f"{group}-{key}", **model_info},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
monkeypatch.setitem(
litellm.model_cost,
"cloudflare/@cf/cloudflare/clef-omni",
{"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True},
)
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond(
json=_RESPONSE
)
unlisted: Final = "openrouter/example/omni-preview"
listed: Final = "cloudflare/clef-omni"
router: Final = litellm.Router(
model_list=[
_group_member("unlisted", unlisted, "refuses", 1, {}),
_group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}),
_group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}),
_group_member("listed", listed, "takes", 0, {}),
_group_member("none", unlisted, "refuses", 1, {}),
_group_member("none", listed, "refuses", 1, {"supports_audio_input": False}),
]
)
call: Final = getattr(router, surface)
async def send(group: str, **media: object) -> object:
response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media)
return await response if asyncio.iscoroutine(response) else response
for group in ("unlisted", "listed"):
await send(group, audio=_AUDIO_CLIPS)
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
await send("unlisted")
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses"
sent_before_refusal: Final = len(respx_mock.calls)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
await send("none", audio=_AUDIO_CLIPS)
assert len(respx_mock.calls) == sent_before_refusal
@pytest.mark.asyncio
async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None