mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(decisions): route clips in a load-balanced group to a deployment that takes them
A Decisions or SystemOne request carrying audio or video now skips group deployments whose own flags or the cost map refuse that media, and keeps the whole group when none takes it, so the 400 still names the media.
This commit is contained in:
parent
c32f224461
commit
670c6392eb
4 changed files with 153 additions and 16 deletions
|
|
@ -291,6 +291,37 @@ class _ResolvedSupport:
|
|||
videos: bool
|
||||
|
||||
|
||||
def _takes_audio(
|
||||
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
|
||||
) -> bool:
|
||||
return provider_config.supports_audio_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
|
||||
)
|
||||
|
||||
|
||||
def _takes_videos(
|
||||
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
|
||||
) -> bool:
|
||||
return provider_config.supports_video_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
|
||||
)
|
||||
|
||||
|
||||
def deployment_takes_media(
|
||||
model: str, custom_llm_provider: str, model_info: Mapping[str, object], *, audio: bool, videos: bool
|
||||
) -> bool:
|
||||
provider_config: Final = _provider_config(model, custom_llm_provider)
|
||||
canonical_model: Final = provider_config.canonical_model(model)
|
||||
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
|
||||
return (not audio or _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) and (
|
||||
not videos or _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
|
||||
)
|
||||
|
||||
|
||||
def _resolved_support(
|
||||
provider_config: BaseDecisionsConfig,
|
||||
model: str,
|
||||
|
|
@ -304,18 +335,8 @@ def _resolved_support(
|
|||
box_questions=provider_config.supports_box_questions(
|
||||
model, custom_llm_provider, headers, deployment.supports_box_questions
|
||||
),
|
||||
audio=bool(fields.audio)
|
||||
and provider_config.supports_audio_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
|
||||
),
|
||||
videos=bool(fields.videos)
|
||||
and provider_config.supports_video_input(
|
||||
model,
|
||||
custom_llm_provider,
|
||||
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
|
||||
),
|
||||
audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment),
|
||||
videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -606,6 +606,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"})
|
|||
_STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
|
||||
_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None)
|
||||
_routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None)
|
||||
_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"})
|
||||
_ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType(
|
||||
{
|
||||
"adecisions": ProviderConfigManager.get_provider_decisions_config,
|
||||
|
|
@ -14531,6 +14532,45 @@ class Router:
|
|||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _decisions_deployment_takes_media(
|
||||
deployment: DeploymentTypedDict, group: str, *, audio: bool, videos: bool
|
||||
) -> bool:
|
||||
from litellm.decisions.call import deployment_takes_media
|
||||
|
||||
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
|
||||
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
|
||||
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
|
||||
return deployment_takes_media(model, custom_llm_provider, model_info, audio=audio, videos=videos)
|
||||
|
||||
@staticmethod
|
||||
def _without_decisions_deployments_refusing_media(
|
||||
healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None
|
||||
) -> _RouteDeploymentsT:
|
||||
call_kwargs: Final = request_kwargs or {}
|
||||
audio: Final = bool(call_kwargs.get("audio"))
|
||||
videos: Final = bool(call_kwargs.get("videos"))
|
||||
if (
|
||||
_routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES
|
||||
or not (audio or videos)
|
||||
or isinstance(healthy_deployments, Mapping)
|
||||
):
|
||||
return healthy_deployments
|
||||
|
||||
media_capable_deployments: Final = [
|
||||
deployment
|
||||
for deployment in healthy_deployments
|
||||
if Router._decisions_deployment_takes_media(
|
||||
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
|
||||
group,
|
||||
audio=audio,
|
||||
videos=videos,
|
||||
)
|
||||
]
|
||||
if not media_capable_deployments:
|
||||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, media_capable_deployments) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _without_declared_text_only_deployments(
|
||||
healthy_deployments: _RouteDeploymentsT,
|
||||
|
|
@ -15142,9 +15182,13 @@ class Router:
|
|||
verbose_router_logger.debug("healthy_deployments after web search filter: %s", healthy_deployments)
|
||||
|
||||
healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows
|
||||
self._without_deployments_that_cannot_serve_route(
|
||||
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
|
||||
self._without_decisions_deployments_refusing_media(
|
||||
self._without_deployments_that_cannot_serve_route(
|
||||
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
|
||||
model,
|
||||
),
|
||||
model,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: request kwargs are a plain dict
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -16412,9 +16456,13 @@ class Router:
|
|||
request_kwargs=request_kwargs,
|
||||
)
|
||||
healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows
|
||||
self._without_deployments_that_cannot_serve_route(
|
||||
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
|
||||
self._without_decisions_deployments_refusing_media(
|
||||
self._without_deployments_that_cannot_serve_route(
|
||||
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
|
||||
model,
|
||||
),
|
||||
model,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: request kwargs are a plain dict
|
||||
)
|
||||
)
|
||||
strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
|
||||
|
|
|
|||
|
|
@ -181,6 +181,8 @@ ignored_function_names = [
|
|||
"_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py
|
||||
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
|
||||
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
|
||||
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
|
||||
"_decisions_deployment_takes_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3860,6 +3860,72 @@ async def test_another_deployments_mode_never_switches_off_a_deployments_own_med
|
|||
assert (sent["audio"], sent["videos"]) == (list(_AUDIO_CLIPS), list(_VIDEO_CLIPS))
|
||||
|
||||
|
||||
_MEDIA_SURFACES: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType(
|
||||
{
|
||||
"adecisions": {"input": "review", "questions": _PREDICATE_QUESTIONS},
|
||||
"asystemone": {"state": "review", "questions": _QUESTIONS},
|
||||
"systemone": {"state": "review", "questions": _QUESTIONS},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _group_member(group: str, model: str, key: str, weight: int, model_info: Mapping[str, object]) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": group,
|
||||
"litellm_params": {"model": model, "api_key": key, "weight": weight},
|
||||
"model_info": {"id": f"{group}-{key}", **model_info},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("surface", tuple(_MEDIA_SURFACES))
|
||||
async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_only_when_none_does(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None, surface: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct")
|
||||
monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False)
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"cloudflare/@cf/cloudflare/clef-omni",
|
||||
{"litellm_provider": "cloudflare", "mode": "evaluation", "supports_audio_input": True},
|
||||
)
|
||||
respx_mock.post("https://openrouter.ai/api/alpha/decisions").respond(json=_RESPONSE)
|
||||
respx_mock.post("https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef-omni").respond(
|
||||
json=_RESPONSE
|
||||
)
|
||||
unlisted: Final = "openrouter/example/omni-preview"
|
||||
listed: Final = "cloudflare/clef-omni"
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_group_member("unlisted", unlisted, "refuses", 1, {}),
|
||||
_group_member("unlisted", unlisted, "takes", 0, {"supports_audio_input": True}),
|
||||
_group_member("listed", listed, "refuses", 1, {"supports_audio_input": False}),
|
||||
_group_member("listed", listed, "takes", 0, {}),
|
||||
_group_member("none", unlisted, "refuses", 1, {}),
|
||||
_group_member("none", listed, "refuses", 1, {"supports_audio_input": False}),
|
||||
]
|
||||
)
|
||||
call: Final = getattr(router, surface)
|
||||
|
||||
async def send(group: str, **media: object) -> object:
|
||||
response: Final = call(model=group, **_MEDIA_SURFACES[surface], **media)
|
||||
return await response if asyncio.iscoroutine(response) else response
|
||||
|
||||
for group in ("unlisted", "listed"):
|
||||
await send(group, audio=_AUDIO_CLIPS)
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer takes"
|
||||
assert json.loads(respx_mock.calls.last.request.content)["audio"] == list(_AUDIO_CLIPS)
|
||||
|
||||
await send("unlisted")
|
||||
assert respx_mock.calls.last.request.headers["authorization"] == "Bearer refuses"
|
||||
|
||||
sent_before_refusal: Final = len(respx_mock.calls)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
|
||||
await send("none", audio=_AUDIO_CLIPS)
|
||||
assert len(respx_mock.calls) == sent_before_refusal
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue