diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 70c2d600ba3..8f05d6f5a55 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -229,6 +229,8 @@ graph LR | `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) | | `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) | +With `store_model_in_db` enabled, deployment reconciliation removes Router entries absent from both the database and `model_list`. Plugins that own runtime-injected deployments can set a non-empty `model_info.managed_by` marker, such as `"plugin"`, and leave `model_info.db_model` as `False`. These deployments survive reconciliation even when `model_list` is populated or explicitly empty, and their IDs remain in the returned desired set. The plugin is responsible for removing its deployments when they are no longer needed. DB deployments missing from the database are still removed, even if they carry an owner marker. The existing protection for non-DB deployments when `model_list` is missing or null remains unchanged + **Cost Attribution Flow:** 1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes 2. `update_response_metadata()` (`litellm_core_utils/llm_response_utils/response_metadata.py`) is called diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ca0f8892b9d..68d283a69a5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7613,12 +7613,12 @@ class ProxyConfig: - Create all up list of model id's (db + config) - Compare all up list to router model id's - - Remove any that are missing + - Remove missing deployments unless a non-DB deployment declares managed_by Return: - - frozenset[str] - the ids the db + config say should be served after this - reconcile, so a caller can tell an id this evicted on purpose from one that - went missing. None when no reconcile ran and that set is therefore unknown. + - frozenset[str] - the ids the db + config or an external owner say should be + served after this reconcile, so a caller can distinguish intentional eviction + from a missing deployment. None when no reconcile ran and that set is unknown. """ global user_config_file_path, llm_router combined_id_list: Final = [] @@ -7687,10 +7687,19 @@ class ProxyConfig: len(kept_config_ids), ) + externally_managed_ids: Final[frozenset[str]] = frozenset( + model_id + for model_id in router_model_ids + if (managed_deployment := llm_router.get_deployment(model_id=model_id)) is not None + and managed_deployment.model_info.db_model is False + and managed_deployment.model_info.get("managed_by") + ) + desired_ids: Final = frozenset(combined_id_list) | kept_config_ids | externally_managed_ids + for model_id in router_model_ids: - if model_id not in combined_id_list and model_id not in kept_config_ids: + if model_id not in desired_ids: llm_router.delete_deployment(id=model_id) - return frozenset(combined_id_list) | kept_config_ids + return desired_ids def _resolve_db_litellm_param(self, key: str, value: object) -> object: if not isinstance(value, str): diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 745488bf93f..526f2586b92 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -3150,6 +3150,62 @@ async def test_delete_deployment_type_mismatch(): ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("config_shape", "expected_ids"), + [ + ("populated", {"config-1", "plugin-1"}), + ("empty", {"plugin-1"}), + ("null", {"config-1", "plugin-1", "unowned-1", "empty-owner-1", "null-owner-1"}), + ("missing", {"config-1", "plugin-1", "unowned-1", "empty-owner-1", "null-owner-1"}), + ], +) +async def test_delete_deployment_preserves_external_owners_without_preserving_db_orphans( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config_shape: str, expected_ids: set[str] +) -> None: + from litellm import Router + from litellm.proxy.proxy_server import ProxyConfig + from litellm.types.router import DeploymentTypedDict + + def deployment(model_id: str, info: dict[str, str | bool | None]) -> DeploymentTypedDict: + return { + "model_name": model_id, + "litellm_params": {"model": "openai/runtime-test", "api_key": "sk-fake"}, + "model_info": {"id": model_id, **info}, + } + + config_model: Final = deployment("config-1", {}) + router: Final = Router( + model_list=[ + config_model, + deployment("plugin-1", {"managed_by": "plugin"}), + deployment("unowned-1", {}), + deployment("empty-owner-1", {"managed_by": ""}), + deployment("null-owner-1", {"managed_by": None}), + deployment("db-orphan-1", {"db_model": True}), + deployment("marked-db-orphan-1", {"db_model": True, "managed_by": "plugin"}), + ] + ) + config: Final = { + "populated": {"model_list": [config_model]}, + "empty": {"model_list": []}, + "null": {"model_list": None}, + "missing": {}, + }[config_shape] + config_path: Final = tmp_path / "config.yaml" + config_path.write_text(yaml.safe_dump(config)) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + monkeypatch.setattr(proxy_server_module, "llm_router", router) + monkeypatch.setattr(proxy_server_module, "user_config_file_path", str(config_path)) + monkeypatch.setattr(proxy_server_module, "prisma_client", None) + monkeypatch.setattr(proxy_server_module, "store_model_in_db", True) + + desired_ids: Final = await ProxyConfig()._delete_deployment(db_models=[]) + + assert set(router.get_model_ids()) == expected_ids + assert desired_ids == frozenset(expected_ids) + + @pytest.mark.asyncio async def test_get_config_from_file(tmp_path, monkeypatch): """