fix(router): skip clip-refusing Decisions deployments after cooldown and tag filtering

Running the media filter before cooldown turned the 400 a group answered
when its only clip-taking deployment was cooling down into a 429. It now
runs next to the image filter, just before order filtering, in both the
async and sync selection paths.
This commit is contained in:
mateo-berri 2026-10-30 20:28:41 -07:00
parent 2edef875a5
commit 4ff23fbde9
2 changed files with 54 additions and 18 deletions

View file

@ -15561,13 +15561,9 @@ class Router(DeprecatedRouterMembers):
verbose_router_logger.debug("healthy_deployments after web search filter: %s", healthy_deployments)
healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows
self._without_decisions_deployments_refusing_media(
self._without_deployments_that_cannot_serve_route(
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
model,
),
self._without_deployments_that_cannot_serve_route(
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
model,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: request kwargs are a plain dict
)
)
@ -15669,10 +15665,14 @@ class Router(DeprecatedRouterMembers):
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
batch_candidates: Final = self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
),
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
)
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
@ -16848,13 +16848,9 @@ class Router(DeprecatedRouterMembers):
request_kwargs=request_kwargs,
)
healthy_deployments = ( # pyright: ignore[reportUnknownVariableType] # rebind-ok: route filter narrows
self._without_decisions_deployments_refusing_media(
self._without_deployments_that_cannot_serve_route(
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
model,
),
self._without_deployments_that_cannot_serve_route(
healthy_deployments, # pyright: ignore[reportUnknownArgumentType] # legacy selector type
model,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: request kwargs are a plain dict
)
)
strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
@ -16918,10 +16914,14 @@ class Router(DeprecatedRouterMembers):
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
batch_candidates: Final = self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
),
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
)
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(

View file

@ -3937,6 +3937,42 @@ async def test_a_group_sends_clips_to_a_deployment_that_takes_them_and_refuses_o
assert len(respx_mock.calls) == sent_before_refusal
def _answer_unless_key_is(rejected_key: str) -> Callable[[httpx.Request], httpx.Response]:
def answer(request: httpx.Request) -> httpx.Response:
if request.headers["authorization"] == f"Bearer {rejected_key}":
return httpx.Response(401, json={"error": {"message": "invalid key"}})
return httpx.Response(200, json=_RESPONSE)
return answer
@pytest.mark.asyncio
async def test_clips_to_a_group_whose_only_taker_is_cooling_down_are_refused_rather_than_rate_limited(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
route: Final = respx_mock.post("https://openrouter.ai/api/alpha/decisions").mock(
side_effect=_answer_unless_key_is("takes")
)
unlisted: Final = "openrouter/example/omni-preview"
router: Final = litellm.Router(
model_list=[
_group_member("g", unlisted, "refuses", 1, {}),
_group_member("g", unlisted, "takes", 1, {"supports_audio_input": True}),
],
num_retries=0,
cooldown_time=120,
)
with pytest.raises(litellm.AuthenticationError):
await router.adecisions(model="g-takes", input="review", questions=_PREDICATE_QUESTIONS)
sent_before_clips: Final = len(route.calls)
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("['audio']")):
await router.adecisions(model="g", input="review", questions=_PREDICATE_QUESTIONS, audio=_AUDIO_CLIPS)
assert len(route.calls) == sent_before_clips
@pytest.mark.asyncio
async def test_a_deployment_that_turns_audio_off_is_refused_or_dropped_even_when_the_cost_map_allows_it(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None