Merge 638310a1f8 into e16d79d635

ARCHITECTURE.md

@@ -229,6 +229,8 @@ graph LR
 | `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
 | `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
 
+With `store_model_in_db` enabled, deployment reconciliation removes Router entries absent from both the database and `model_list`. Plugins that own runtime-injected deployments can set a non-empty `model_info.managed_by` marker, such as `"plugin"`, and leave `model_info.db_model` as `False`. These deployments survive reconciliation even when `model_list` is populated or explicitly empty, and their IDs remain in the returned desired set. The plugin is responsible for removing its deployments when they are no longer needed. DB deployments missing from the database are still removed, even if they carry an owner marker. The existing protection for non-DB deployments when `model_list` is missing or null remains unchanged
+
 **Cost Attribution Flow:**
 1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes
 2. `update_response_metadata()` (`litellm_core_utils/llm_response_utils/response_metadata.py`) is called

litellm/proxy/proxy_server.py

@@ -7613,12 +7613,12 @@ class ProxyConfig:
 
         - Create all up list of model id's (db + config)
         - Compare all up list to router model id's
-        - Remove any that are missing
+        - Remove missing deployments unless a non-DB deployment declares managed_by
 
         Return:
-        - frozenset[str] - the ids the db + config say should be served after this
-          reconcile, so a caller can tell an id this evicted on purpose from one that
-          went missing. None when no reconcile ran and that set is therefore unknown.
+        - frozenset[str] - the ids the db + config or an external owner say should be
+          served after this reconcile, so a caller can distinguish intentional eviction
+          from a missing deployment. None when no reconcile ran and that set is unknown.
         """
         global user_config_file_path, llm_router
         combined_id_list: Final = []
@@ -7687,10 +7687,19 @@ class ProxyConfig:
                 len(kept_config_ids),
             )
 
+        externally_managed_ids: Final[frozenset[str]] = frozenset(
+            model_id
+            for model_id in router_model_ids
+            if (managed_deployment := llm_router.get_deployment(model_id=model_id)) is not None
+            and managed_deployment.model_info.db_model is False
+            and managed_deployment.model_info.get("managed_by")
+        )
+        desired_ids: Final = frozenset(combined_id_list) | kept_config_ids | externally_managed_ids
+
         for model_id in router_model_ids:
-            if model_id not in combined_id_list and model_id not in kept_config_ids:
+            if model_id not in desired_ids:
                 llm_router.delete_deployment(id=model_id)
-        return frozenset(combined_id_list) | kept_config_ids
+        return desired_ids
 
     def _resolve_db_litellm_param(self, key: str, value: object) -> object:
         if not isinstance(value, str):

tests/unit/proxy/test_proxy_server_endpoints_and_startup.py

@@ -3150,6 +3150,62 @@ async def test_delete_deployment_type_mismatch():
     )
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("config_shape", "expected_ids"),
+    [
+        ("populated", {"config-1", "plugin-1"}),
+        ("empty", {"plugin-1"}),
+        ("null", {"config-1", "plugin-1", "unowned-1", "empty-owner-1", "null-owner-1"}),
+        ("missing", {"config-1", "plugin-1", "unowned-1", "empty-owner-1", "null-owner-1"}),
+    ],
+)
+async def test_delete_deployment_preserves_external_owners_without_preserving_db_orphans(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config_shape: str, expected_ids: set[str]
+) -> None:
+    from litellm import Router
+    from litellm.proxy.proxy_server import ProxyConfig
+    from litellm.types.router import DeploymentTypedDict
+
+    def deployment(model_id: str, info: dict[str, str | bool | None]) -> DeploymentTypedDict:
+        return {
+            "model_name": model_id,
+            "litellm_params": {"model": "openai/runtime-test", "api_key": "sk-fake"},
+            "model_info": {"id": model_id, **info},
+        }
+
+    config_model: Final = deployment("config-1", {})
+    router: Final = Router(
+        model_list=[
+            config_model,
+            deployment("plugin-1", {"managed_by": "plugin"}),
+            deployment("unowned-1", {}),
+            deployment("empty-owner-1", {"managed_by": ""}),
+            deployment("null-owner-1", {"managed_by": None}),
+            deployment("db-orphan-1", {"db_model": True}),
+            deployment("marked-db-orphan-1", {"db_model": True, "managed_by": "plugin"}),
+        ]
+    )
+    config: Final = {
+        "populated": {"model_list": [config_model]},
+        "empty": {"model_list": []},
+        "null": {"model_list": None},
+        "missing": {},
+    }[config_shape]
+    config_path: Final = tmp_path / "config.yaml"
+    config_path.write_text(yaml.safe_dump(config))
+    monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
+    monkeypatch.setattr(proxy_server_module, "llm_router", router)
+    monkeypatch.setattr(proxy_server_module, "user_config_file_path", str(config_path))
+    monkeypatch.setattr(proxy_server_module, "prisma_client", None)
+    monkeypatch.setattr(proxy_server_module, "store_model_in_db", True)
+
+    desired_ids: Final = await ProxyConfig()._delete_deployment(db_models=[])
+
+    assert set(router.get_model_ids()) == expected_ids
+    assert desired_ids == frozenset(expected_ids)
+
+
 @pytest.mark.asyncio
 async def test_get_config_from_file(tmp_path, monkeypatch):
     """