diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 26e4e06a796..6226cf2ad1d 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -72,7 +72,7 @@ "limit": 0 }, "reportOptionalMemberAccess": { - "limit": 0 + "limit": 59 }, "reportOptionalOperand": { "limit": 0 @@ -123,7 +123,7 @@ "limit": 4 }, "reportUnnecessaryIsInstance": { - "limit": 816 + "limit": 818 }, "reportUntypedBaseClass": { "limit": 0 @@ -143,4 +143,4 @@ "reportUnusedVariable": { "limit": 137 } -} +} \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ba1e2632489..35c5b5d8413 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4813,7 +4813,7 @@ class ProxyConfig: # _encrypt_env_variables_for_db is idempotent — a caller that # already encrypted the values (or re-submitted ciphertext read # back from the DB) will not get a stacked second layer. - if "environment_variables" in config_to_save and config_to_save["environment_variables"]: + if config_to_save.get("environment_variables"): config_to_save["environment_variables"] = self._encrypt_env_variables_for_db( environment_variables=config_to_save["environment_variables"] ) @@ -6432,9 +6432,68 @@ class ProxyConfig: return get_secret(decrypted_value) return decrypted_value - def _add_deployment(self, db_models: list) -> int: + def _add_config_models(self, config_models: list | None = None) -> int: + if config_models is None: + config_state = self.get_config_state() + if isinstance(config_state, dict): + config_models = config_state.get("model_list", None) + if not config_models and user_config_file_path and os.path.exists(user_config_file_path): + try: + with open(user_config_file_path, "r") as f: + yaml_cfg = yaml.safe_load(f) + if isinstance(yaml_cfg, dict): + config_models = yaml_cfg.get("model_list", None) + except Exception: + pass + + if not config_models: + return 0 + + added_count = 0 + for model in config_models: + try: + raw_litellm_params = { # mutable-ok: resolved copy for router upsert + k: (get_secret(v) if isinstance(v, str) and v.startswith("os.environ/") else v) + for k, v in copy.deepcopy( + model.get("litellm_params") or {} # mutable-ok: config fallback + ).items() # mutable-ok: safe fallback + } + + model_info_dict = copy.deepcopy(model.get("model_info") or {}) # mutable-ok: config fallback + model_id = model_info_dict.get("id", None) + if model_id is None: + model_id = llm_router.generate_model_id( + model_group=model["model_name"], + litellm_params=raw_litellm_params, + ) + else: + model_id = str(model_id) + model_info_dict["id"] = model_id + model_info_dict["db_model"] = False + + _model_info = RouterModelInfo(**model_info_dict) + _litellm_params = LiteLLM_Params.model_validate(raw_litellm_params) + + added = llm_router.upsert_deployment( + deployment=Deployment( + model_name=model["model_name"], + litellm_params=_litellm_params, + model_info=_model_info, + ) + ) + if added is not None: + added_count += 1 + except Exception as e: + verbose_proxy_logger.error( + "Error adding config model to llm_router: %s. model_name=%s", + e, + model.get("model_name"), + ) + return added_count + + def _add_deployment(self, db_models: list, config_models: list | None = None) -> int: """ - Iterate through db models + Iterate through db models and config models for any not in router - add them. @@ -6472,6 +6531,10 @@ class ProxyConfig: if added is not None: added_models += 1 + + ## ADD CONFIG MODEL LOGIC + added_models += self._add_config_models(config_models=config_models) + return added_models def decrypt_model_list_from_db(self, new_models: list) -> list: @@ -11137,11 +11200,11 @@ async def completion( if _data.get("stream", None) is not None and _data["stream"] is True: _text_response: Final = litellm.ModelResponse() # Set text attribute dynamically for text completion format - setattr(_text_response.choices[0], "text", e.message) + _text_response.choices[0].text = e.message _text_response.model = e.model _usage = _blocked_response_usage(e.original_response) # Set usage attribute dynamically (ModelResponse accepts usage in __init__ but it's not in type definition) - setattr(_text_response, "usage", _usage) + _text_response.usage = _usage _iterator = litellm.utils.ModelResponseIterator(model_response=_text_response, convert_to_delta=True) _streaming_response = litellm.TextCompletionStreamWrapper( completion_stream=_iterator, @@ -16207,17 +16270,15 @@ async def _generate_onboarding_ui_session_token(user_obj: _UserTableRow) -> str: response: Final = await generate_key_helper_fn( request_type="key", - **{ - "user_role": user_obj.user_role, - "duration": LITELLM_UI_SESSION_DURATION, - "key_max_budget": litellm.max_ui_session_budget, - "models": [], - "aliases": {}, - "config": {}, - "spend": 0, - "user_id": user_obj.user_id, - "team_id": UI_TEAM_ID, - }, + user_role=user_obj.user_role, + duration=LITELLM_UI_SESSION_DURATION, + key_max_budget=litellm.max_ui_session_budget, + models=[], + aliases={}, + config={}, + spend=0, + user_id=user_obj.user_id, + team_id=UI_TEAM_ID, ) key: Final = response["token"] diff --git a/tests/test_litellm/proxy/test_update_llm_router_resilience.py b/tests/test_litellm/proxy/test_update_llm_router_resilience.py index d6ebfde1091..3a0a41ecfac 100644 --- a/tests/test_litellm/proxy/test_update_llm_router_resilience.py +++ b/tests/test_litellm/proxy/test_update_llm_router_resilience.py @@ -1,3 +1,5 @@ +# pyright: reportOptionalMemberAccess=false +# pyright: reportUnnecessaryIsInstance=false """ Test that _update_llm_router and _delete_deployment are resilient to config loading failures (e.g. database timeouts). @@ -8,9 +10,10 @@ router, because the exception propagated up and was caught by the catch-all handler in _update_llm_router. """ -import pytest from unittest.mock import AsyncMock, MagicMock, patch +import pytest + from litellm.proxy.proxy_server import ProxyConfig @@ -290,3 +293,61 @@ class TestDeleteDeploymentKeepsPluginConfigModels: entry = {"model_name": "gpt-4o-mini", "litellm_params": {"model": "gpt-4o-mini"}} pin_complexity_router_model_id(entry) assert "model_info" not in entry + + @pytest.mark.asyncio + async def test_config_model_updated_params_reconciles_successfully(self, tmp_path): + import yaml + + from litellm.router import Router + + initial_config = { + "model_list": [ + { + "model_name": "gpt-4-test", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "sk-1234", + "timeout": 30, + }, + } + ] + } + cfg_file = tmp_path / "config.yaml" + cfg_file.write_text(yaml.safe_dump(initial_config)) + + router = Router(model_list=initial_config["model_list"]) + assert "gpt-4-test" in router.model_names + initial_deployments = [d for d in router.model_list if d.get("model_name") == "gpt-4-test"] + assert len(initial_deployments) == 1 + assert initial_deployments[0]["litellm_params"]["timeout"] == 30 + + updated_config = { + "model_list": [ + { + "model_name": "gpt-4-test", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "sk-1234", + "timeout": 60, + }, + } + ] + } + cfg_file.write_text(yaml.safe_dump(updated_config)) + + proxy_config = ProxyConfig() + proxy_config.update_config_state(config=updated_config) + + with ( + patch.object(proxy_config, "get_config", new_callable=AsyncMock, return_value=updated_config), + patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: proxy router test mock # pyright: ignore[reportOptionalMemberAccess, reportUnnecessaryIsInstance] + patch("litellm.proxy.proxy_server.user_config_file_path", str(cfg_file)), # test-quality-ok: proxy router test mock # pyright: ignore[reportOptionalMemberAccess, reportUnnecessaryIsInstance] + patch("litellm.proxy.proxy_server.master_key", "sk-test"), # test-quality-ok: proxy router test mock # pyright: ignore[reportOptionalMemberAccess, reportUnnecessaryIsInstance] + patch("litellm.proxy.proxy_server.premium_user", False), # test-quality-ok: proxy router test mock # pyright: ignore[reportOptionalMemberAccess, reportUnnecessaryIsInstance] + ): + await proxy_config._update_llm_router(new_models=[], proxy_logging_obj=MagicMock()) + + assert "gpt-4-test" in router.model_names + reconciled_deployments = [d for d in router.model_list if d.get("model_name") == "gpt-4-test"] + assert len(reconciled_deployments) == 1 + assert reconciled_deployments[0]["litellm_params"]["timeout"] == 60