fix(proxy): let the rate-limit fallback probe try a fallback this worker has not loaded yet

The probe now skips a fallback only when the endpoint check removed every deployment it has, so a model added on another worker still reaches route_request and its database lookup
This commit is contained in:
mateo-berri 2026-10-29 00:00:08 -07:00
parent 0f83ec33db
commit 7ef9d0d3cb
2 changed files with 28 additions and 1 deletions

View file

@ -448,7 +448,9 @@ def endpoint_unsupported_by_model(
"user_api_key_team_id": team_id,
},
}
return not llm_router.get_routable_deployments(model, request_kwargs, call_type=route_type)
return bool(llm_router.get_routable_deployments(model, request_kwargs)) and not llm_router.get_routable_deployments(
model, request_kwargs, call_type=route_type
)
class MockTestingParamsDisabledError(HTTPException):

View file

@ -28,6 +28,7 @@ from litellm.proxy.route_llm_request import (
JSON_OBJECT_ADAPTER,
ProxyModelNotFoundError,
UserConfigRouters,
endpoint_unsupported_by_model,
route_request,
user_config_routers,
)
@ -3027,3 +3028,27 @@ async def test_user_config_routers_discard_the_least_recently_used_router_past_c
assert first_handler() is not None
assert second_handler() is None
routers.clear()
def test_fallback_probe_skips_only_a_loaded_model_that_refuses_the_route() -> None:
router: Final = litellm.Router(
model_list=[
{
"model_name": "chat-fallback",
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "unit-provider-key"},
},
{
"model_name": "embed-only",
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "unit-provider-key"},
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
},
],
num_retries=0,
)
def skipped(model: str) -> bool:
return endpoint_unsupported_by_model("acompletion", {"model": model}, router, None)
assert skipped("embed-only") is True
assert skipped("chat-fallback") is False
assert skipped("added-on-another-worker") is False