mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): let the rate-limit fallback probe try a fallback this worker has not loaded yet
The probe now skips a fallback only when the endpoint check removed every deployment it has, so a model added on another worker still reaches route_request and its database lookup
This commit is contained in:
parent
0f83ec33db
commit
7ef9d0d3cb
2 changed files with 28 additions and 1 deletions
|
|
@ -448,7 +448,9 @@ def endpoint_unsupported_by_model(
|
|||
"user_api_key_team_id": team_id,
|
||||
},
|
||||
}
|
||||
return not llm_router.get_routable_deployments(model, request_kwargs, call_type=route_type)
|
||||
return bool(llm_router.get_routable_deployments(model, request_kwargs)) and not llm_router.get_routable_deployments(
|
||||
model, request_kwargs, call_type=route_type
|
||||
)
|
||||
|
||||
|
||||
class MockTestingParamsDisabledError(HTTPException):
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from litellm.proxy.route_llm_request import (
|
|||
JSON_OBJECT_ADAPTER,
|
||||
ProxyModelNotFoundError,
|
||||
UserConfigRouters,
|
||||
endpoint_unsupported_by_model,
|
||||
route_request,
|
||||
user_config_routers,
|
||||
)
|
||||
|
|
@ -3027,3 +3028,27 @@ async def test_user_config_routers_discard_the_least_recently_used_router_past_c
|
|||
assert first_handler() is not None
|
||||
assert second_handler() is None
|
||||
routers.clear()
|
||||
|
||||
|
||||
def test_fallback_probe_skips_only_a_loaded_model_that_refuses_the_route() -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "chat-fallback",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "unit-provider-key"},
|
||||
},
|
||||
{
|
||||
"model_name": "embed-only",
|
||||
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "unit-provider-key"},
|
||||
"model_info": {"enforce_supported_endpoints": True, "supported_endpoints": ["/v1/embeddings"]},
|
||||
},
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
def skipped(model: str) -> bool:
|
||||
return endpoint_unsupported_by_model("acompletion", {"model": model}, router, None)
|
||||
|
||||
assert skipped("embed-only") is True
|
||||
assert skipped("chat-fallback") is False
|
||||
assert skipped("added-on-another-worker") is False
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue