From 28969ffe07598f5d482af83f57bb3ca191cf7ace Mon Sep 17 00:00:00 2001 From: Ayush Arora Date: Sat, 12 Sep 2026 01:29:56 +0530 Subject: [PATCH] fix(proxy): reconcile config-defined models in _add_deployment during router update --- litellm/proxy/proxy_server.py | 56 +++++++++++++++++- .../test_update_llm_router_resilience.py | 58 +++++++++++++++++++ 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0f24acb8bb4..b7b899d0c10 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6423,9 +6423,9 @@ class ProxyConfig: return get_secret(decrypted_value) return decrypted_value - def _add_deployment(self, db_models: list) -> int: + def _add_deployment(self, db_models: list, config_models: list | None = None) -> int: """ - Iterate through db models + Iterate through db models and config models for any not in router - add them. @@ -6463,6 +6463,58 @@ class ProxyConfig: if added is not None: added_models += 1 + + ## ADD CONFIG MODEL LOGIC + if config_models is None: + config_state = self.get_config_state() + if isinstance(config_state, dict): + config_models = config_state.get("model_list", None) + if not config_models and user_config_file_path and os.path.exists(user_config_file_path): + try: + with open(user_config_file_path, "r") as f: + yaml_cfg = yaml.safe_load(f) + if isinstance(yaml_cfg, dict): + config_models = yaml_cfg.get("model_list", None) + except Exception: + pass + + if config_models: + for model in config_models: + try: + raw_litellm_params = copy.deepcopy(model.get("litellm_params", {})) + for k, v in raw_litellm_params.items(): + if isinstance(v, str) and v.startswith("os.environ/"): + raw_litellm_params[k] = get_secret(v) + + model_info_dict = copy.deepcopy(model.get("model_info", {})) + model_id = model_info_dict.get("id", None) + if model_id is None: + model_id = llm_router.generate_model_id( + model_group=model["model_name"], + litellm_params=raw_litellm_params, + ) + else: + model_id = str(model_id) + model_info_dict["id"] = model_id + model_info_dict["db_model"] = False + + _model_info = RouterModelInfo(**model_info_dict) + _litellm_params = LiteLLM_Params.model_validate(raw_litellm_params) + + added = llm_router.upsert_deployment( + deployment=Deployment( + model_name=model["model_name"], + litellm_params=_litellm_params, + model_info=_model_info, + ) + ) + if added is not None: + added_models += 1 + except Exception as e: + verbose_proxy_logger.error( + "Error adding config model to llm_router: %s. model=%s", e, model + ) + return added_models def decrypt_model_list_from_db(self, new_models: list) -> list: diff --git a/tests/test_litellm/proxy/test_update_llm_router_resilience.py b/tests/test_litellm/proxy/test_update_llm_router_resilience.py index d6ebfde1091..1be05c4e66b 100644 --- a/tests/test_litellm/proxy/test_update_llm_router_resilience.py +++ b/tests/test_litellm/proxy/test_update_llm_router_resilience.py @@ -290,3 +290,61 @@ class TestDeleteDeploymentKeepsPluginConfigModels: entry = {"model_name": "gpt-4o-mini", "litellm_params": {"model": "gpt-4o-mini"}} pin_complexity_router_model_id(entry) assert "model_info" not in entry + + @pytest.mark.asyncio + async def test_config_model_updated_params_reconciles_successfully(self, tmp_path): + import yaml + import litellm + from litellm.router import Router + + initial_config = { + "model_list": [ + { + "model_name": "gpt-4-test", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "sk-1234", + "timeout": 30, + }, + } + ] + } + cfg_file = tmp_path / "config.yaml" + cfg_file.write_text(yaml.safe_dump(initial_config)) + + router = Router(model_list=initial_config["model_list"]) + assert "gpt-4-test" in router.model_names + initial_deployments = [d for d in router.model_list if d.get("model_name") == "gpt-4-test"] + assert len(initial_deployments) == 1 + assert initial_deployments[0]["litellm_params"]["timeout"] == 30 + + updated_config = { + "model_list": [ + { + "model_name": "gpt-4-test", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "sk-1234", + "timeout": 60, + }, + } + ] + } + cfg_file.write_text(yaml.safe_dump(updated_config)) + + proxy_config = ProxyConfig() + proxy_config.update_config_state(config=updated_config) + + with ( + patch.object(proxy_config, "get_config", new_callable=AsyncMock, return_value=updated_config), + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.user_config_file_path", str(cfg_file)), + patch("litellm.proxy.proxy_server.master_key", "sk-test"), + patch("litellm.proxy.proxy_server.premium_user", False), + ): + await proxy_config._update_llm_router(new_models=[], proxy_logging_obj=MagicMock()) + + assert "gpt-4-test" in router.model_names + reconciled_deployments = [d for d in router.model_list if d.get("model_name") == "gpt-4-test"] + assert len(reconciled_deployments) == 1 + assert reconciled_deployments[0]["litellm_params"]["timeout"] == 60