From 7cdc7bfaca9d21b711e8c96590ab59a37f0a1fb2 Mon Sep 17 00:00:00 2001 From: Eduardo Pessin Date: Tue, 20 Oct 2026 09:43:30 +0100 Subject: [PATCH 1/2] fix(proxy): preserve externally managed runtime deployments --- ARCHITECTURE.md | 2 + litellm/proxy/proxy_server.py | 21 +++++-- ...test_proxy_server_endpoints_and_startup.py | 56 +++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8e88c0ea15b..bd4a100790e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -229,6 +229,8 @@ graph LR | `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) | | `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) | +With `store_model_in_db` enabled, deployment reconciliation removes Router entries absent from both the database and `model_list`. Plugins that own runtime-injected deployments can set a non-empty `model_info.managed_by` marker, such as `"plugin"`, and leave `model_info.db_model` as `False`. These deployments survive reconciliation even when `model_list` is populated or explicitly empty, and their IDs remain in the returned desired set. The plugin is responsible for removing its deployments when they are no longer needed. DB deployments missing from the database are still removed, even if they carry an owner marker. The existing protection for non-DB deployments when `model_list` is missing or null remains unchanged + **Cost Attribution Flow:** 1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes 2. `update_response_metadata()` (`litellm_core_utils/llm_response_utils/response_metadata.py`) is called diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ee8f20be243..0e54e596fa8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7756,12 +7756,12 @@ class ProxyConfig: - Create all up list of model id's (db + config) - Compare all up list to router model id's - - Remove any that are missing + - Remove missing deployments unless a non-DB deployment declares managed_by Return: - - frozenset[str] - the ids the db + config say should be served after this - reconcile, so a caller can tell an id this evicted on purpose from one that - went missing. None when no reconcile ran and that set is therefore unknown. + - frozenset[str] - the ids the db + config or an external owner say should be + served after this reconcile, so a caller can distinguish intentional eviction + from a missing deployment. None when no reconcile ran and that set is unknown. """ global user_config_file_path, llm_router combined_id_list: Final = [] @@ -7830,10 +7830,19 @@ class ProxyConfig: len(kept_config_ids), ) + externally_managed_ids: Final[frozenset[str]] = frozenset( + model_id + for model_id in router_model_ids + if (deployment := llm_router.get_deployment(model_id=model_id)) is not None + and deployment.model_info.db_model is False + and deployment.model_info.get("managed_by") + ) + desired_ids: Final = frozenset(combined_id_list) | kept_config_ids | externally_managed_ids + for model_id in router_model_ids: - if model_id not in combined_id_list and model_id not in kept_config_ids: + if model_id not in desired_ids: llm_router.delete_deployment(id=model_id) - return frozenset(combined_id_list) | kept_config_ids + return desired_ids def _resolve_db_litellm_param(self, key: str, value: object) -> object: if not isinstance(value, str): diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 0dbf3451c3b..e6ea5f3199b 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -3150,6 +3150,62 @@ async def test_delete_deployment_type_mismatch(): ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("config_shape", "expected_ids"), + [ + ("populated", {"config-1", "plugin-1"}), + ("empty", {"plugin-1"}), + ("null", {"config-1", "plugin-1", "unowned-1", "empty-owner-1", "null-owner-1"}), + ("missing", {"config-1", "plugin-1", "unowned-1", "empty-owner-1", "null-owner-1"}), + ], +) +async def test_delete_deployment_preserves_external_owners_without_preserving_db_orphans( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config_shape: str, expected_ids: set[str] +) -> None: + from litellm import Router + from litellm.proxy.proxy_server import ProxyConfig + from litellm.types.router import DeploymentTypedDict + + def deployment(model_id: str, info: dict[str, str | bool | None]) -> DeploymentTypedDict: + return { + "model_name": model_id, + "litellm_params": {"model": "openai/runtime-test", "api_key": "sk-fake"}, + "model_info": {"id": model_id, **info}, + } + + config_model: Final = deployment("config-1", {}) + router: Final = Router( + model_list=[ + config_model, + deployment("plugin-1", {"managed_by": "plugin"}), + deployment("unowned-1", {}), + deployment("empty-owner-1", {"managed_by": ""}), + deployment("null-owner-1", {"managed_by": None}), + deployment("db-orphan-1", {"db_model": True}), + deployment("marked-db-orphan-1", {"db_model": True, "managed_by": "plugin"}), + ] + ) + config: Final = { + "populated": {"model_list": [config_model]}, + "empty": {"model_list": []}, + "null": {"model_list": None}, + "missing": {}, + }[config_shape] + config_path: Final = tmp_path / "config.yaml" + config_path.write_text(yaml.safe_dump(config)) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + monkeypatch.setattr(proxy_server_module, "llm_router", router) + monkeypatch.setattr(proxy_server_module, "user_config_file_path", str(config_path)) + monkeypatch.setattr(proxy_server_module, "prisma_client", None) + monkeypatch.setattr(proxy_server_module, "store_model_in_db", True) + + desired_ids: Final = await ProxyConfig()._delete_deployment(db_models=[]) + + assert set(router.get_model_ids()) == expected_ids + assert desired_ids == frozenset(expected_ids) + + @pytest.mark.asyncio async def test_get_config_from_file(tmp_path, monkeypatch): """ From 638310a1f875b8e56983f2e3fffa2e2b2ffc6fb3 Mon Sep 17 00:00:00 2001 From: Eduardo Pessin Date: Tue, 20 Oct 2026 10:17:33 +0100 Subject: [PATCH 2/2] fix(proxy): avoid rebinding the managed deployment variable --- litellm/proxy/proxy_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0e54e596fa8..ec0f64899bb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7833,9 +7833,9 @@ class ProxyConfig: externally_managed_ids: Final[frozenset[str]] = frozenset( model_id for model_id in router_model_ids - if (deployment := llm_router.get_deployment(model_id=model_id)) is not None - and deployment.model_info.db_model is False - and deployment.model_info.get("managed_by") + if (managed_deployment := llm_router.get_deployment(model_id=model_id)) is not None + and managed_deployment.model_info.db_model is False + and managed_deployment.model_info.get("managed_by") ) desired_ids: Final = frozenset(combined_id_list) | kept_config_ids | externally_managed_ids