From 8368fbeac38437e689efcc0fb1bc26eb69821ec1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:45:35 -0700 Subject: [PATCH] test(router): cover a fallback group staying up after the primary's selection fails --- .../router_utils/test_cooldown_handlers.py | 74 +++++++++++++------ 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 07fcfe2e983..4299b61c703 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -2987,6 +2987,24 @@ async def _complete_like_the_proxy(router: Router, model: str) -> str: return model_id +_CHAT_COMPLETION_OK: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _openai_deployment(model_name: str, host: str, deployment_id: str) -> Mapping[str, object]: + return { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": f"https://{host}/v1"}, + "model_info": {"id": deployment_id}, + } + + @pytest.mark.asyncio async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown( respx_mock: respx.MockRouter, @@ -2997,30 +3015,12 @@ async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown( return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}}) ) backup_route: Final = respx_mock.post("https://backup.test/v1/chat/completions").mock( - return_value=httpx.Response( - 200, - json={ - "id": "chatcmpl-1", - "object": "chat.completion", - "created": 1, - "model": "gpt-4o", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - }, - ) + return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK) ) router: Final = litellm.Router( model_list=[ - { - "model_name": "primary", - "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "https://primary.test/v1"}, - "model_info": {"id": "primary-dep"}, - }, - { - "model_name": "backup", - "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": "https://backup.test/v1"}, - "model_info": {"id": "backup-dep"}, - }, + _openai_deployment("primary", "primary.test", "primary-dep"), + _openai_deployment("backup", "backup.test", "backup-dep"), ], fallbacks=[{"primary": ["backup"]}], allowed_fails=litellm.allowed_fails, @@ -3038,6 +3038,38 @@ async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown( assert tuple(cooldown_deployments) == ("primary-dep",) +@pytest.mark.asyncio +async def test_fallback_group_stays_up_after_a_bad_key_cools_the_primary_under_the_rate_based_default( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://primary.test/v1/chat/completions").mock( + return_value=httpx.Response(401, json={"error": {"message": "bad key", "type": "invalid_request_error"}}) + ) + respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK) + ) + router: Final = litellm.Router( + model_list=[ + _openai_deployment("primary", "primary.test", "primary-dep"), + _openai_deployment("backup", "backup.test", "backup-1"), + _openai_deployment("backup", "backup.test", "backup-2"), + ], + fallbacks=[{"primary": ["backup"]}], + num_retries=0, + ) + + served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(12)] + cooldown_deployments: Final = await async_get_cooldown_deployments( + litellm_router_instance=router, + parent_otel_span=None, + ) + + assert set(served_by) <= {"backup-1", "backup-2"} + assert tuple(cooldown_deployments) == ("primary-dep",) + + TIMED_OUT_DEPLOYMENT: Final = { "model_name": "test-model", "litellm_params": {"model": "openai/test-model", "api_key": "test-key"},