diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1f31597c010..08f7a013b12 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6491,10 +6491,27 @@ class ProxyConfig: router_model_ids: Final = llm_router.get_model_ids() # Check for model IDs in llm_router not present in combined_id_list and delete them + kept_config_ids: Final[frozenset[str]] = ( + frozenset( + model_id + for model_id in router_model_ids + if (deployment := llm_router.get_deployment(model_id=model_id)) is not None + and deployment.model_info.db_model is False + ) + if not model_list + else frozenset() + ) + if kept_config_ids: + verbose_proxy_logger.warning( + "Config read in _delete_deployment returned no model_list. " + "Keeping %d config-defined deployments to avoid removing valid models.", + len(kept_config_ids), + ) + for model_id in router_model_ids: - if model_id not in combined_id_list: + if model_id not in combined_id_list and model_id not in kept_config_ids: llm_router.delete_deployment(id=model_id) - return frozenset(combined_id_list) + return frozenset(combined_id_list) | kept_config_ids def _resolve_db_litellm_param(self, key: str, value: object) -> object: if not isinstance(value, str): diff --git a/tests/test_litellm/proxy/test_update_llm_router_resilience.py b/tests/test_litellm/proxy/test_update_llm_router_resilience.py index d6ebfde1091..1c7370b8fb8 100644 --- a/tests/test_litellm/proxy/test_update_llm_router_resilience.py +++ b/tests/test_litellm/proxy/test_update_llm_router_resilience.py @@ -290,3 +290,96 @@ class TestDeleteDeploymentKeepsPluginConfigModels: entry = {"model_name": "gpt-4o-mini", "litellm_params": {"model": "gpt-4o-mini"}} pin_complexity_router_model_id(entry) assert "model_info" not in entry + + +class TestDeleteDeploymentKeepsConfigModelsOnEmptyConfigRead: + """Regression: a config read that succeeds but returns no model_list (e.g. a + partially written file) must not evict config-sourced deployments, because + nothing re-adds config models at runtime. DB-sourced deployments missing from + db_models must still be evicted.""" + + @staticmethod + def _router(model_list): + from litellm import Router + from litellm.types.router import RouterGeneralSettings + + return Router( + model_list=model_list, + router_general_settings=RouterGeneralSettings(async_only_mode=True), + ) + + @pytest.mark.asyncio + async def test_delete_deployment_keeps_config_models_when_config_read_has_no_model_list(self, tmp_path): + config_file_path = str(tmp_path / "config.yaml") + (tmp_path / "config.yaml").write_text("general_settings:\n master_key: sk-1234\n") + + router = self._router( + [ + { + "model_name": "config-model", + "litellm_params": {"model": "gpt-4o-mini"}, + "model_info": {"id": "config-model-1"}, + }, + { + "model_name": "db-model", + "litellm_params": {"model": "gpt-4o-mini"}, + "model_info": {"id": "db-model-1", "db_model": True}, + }, + ] + ) + proxy_config = ProxyConfig() + with ( + patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: reads module global + patch( # test-quality-ok: reads module global + "litellm.proxy.proxy_server.user_config_file_path", + config_file_path, + ), + ): + result = await proxy_config._delete_deployment(db_models=[]) + + model_ids = router.get_model_ids() + assert "config-model-1" in model_ids + assert "db-model-1" not in model_ids + assert result is not None + assert "config-model-1" in result + + @pytest.mark.asyncio + async def test_delete_deployment_still_evicts_config_model_removed_from_non_empty_model_list(self, tmp_path): + config_file_path = str(tmp_path / "config.yaml") + (tmp_path / "config.yaml").write_text( + "model_list:\n" + " - model_name: model-a\n" + " litellm_params:\n" + " model: gpt-4o-mini\n" + " model_info:\n" + " id: model-a-id\n" + ) + + router = self._router( + [ + { + "model_name": "model-a", + "litellm_params": {"model": "gpt-4o-mini"}, + "model_info": {"id": "model-a-id"}, + }, + { + "model_name": "model-b", + "litellm_params": {"model": "gpt-4o-mini"}, + "model_info": {"id": "model-b-id"}, + }, + ] + ) + proxy_config = ProxyConfig() + with ( + patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: reads module global + patch( # test-quality-ok: reads module global + "litellm.proxy.proxy_server.user_config_file_path", + config_file_path, + ), + ): + result = await proxy_config._delete_deployment(db_models=[]) + + model_ids = router.get_model_ids() + assert "model-a-id" in model_ids + assert "model-b-id" not in model_ids + assert result == frozenset({"model-a-id"})