From b7154bf4aae9278b4bde86e463023e7f6af38c43 Mon Sep 17 00:00:00 2001 From: pranavyk10 <144428009+pranavyk10@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:01:22 +0530 Subject: [PATCH 1/2] fix(proxy): persist workers ai model updates A Workers AI id was rejected on reload, so the update returned success while the router kept the previous model, api base, and api key. --- .../get_llm_provider_logic.py | 4 + .../model_management_endpoints.py | 37 ++++ .../test_get_llm_provider_logic.py | 8 + .../test_model_management_endpoints.py | 158 ++++++++++++++++++ tests/unit/test_router/test_router.py | 37 ++++ 5 files changed, 244 insertions(+) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 3130d49a5e2..9cd3e976470 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -537,6 +537,10 @@ def get_llm_provider( custom_llm_provider = "amazon_nova" elif model.startswith("sap/"): custom_llm_provider = "sap" + elif model.startswith(("@cf/", "@hf/")): + # Workers AI ids are not `cloudflare/`. Stripping or rejecting them makes a + # model update roll back to the previous deployment while PATCH still returns 200. + custom_llm_provider = "cloudflare" # Last resort for an otherwise-unknown model: a declarative # fallback-generalization routing rule (e.g. routes future claude-* to anthropic). diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index fc0868a5fc9..2fc0e1e74b6 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1467,6 +1467,7 @@ async def patch_model( still_desired=reload_outcome.still_desired, live_after=reload_outcome.live_after, ) + raise_if_written_params_not_served(model_id=model_id, patch_data=patch_data) return _model_write_response(updated_model, member_write) @@ -3042,6 +3043,7 @@ async def update_model( still_desired=reload_outcome.still_desired, live_after=reload_outcome.live_after, ) + raise_if_written_params_not_served(model_id=_model_id, patch_data=model_params) return None if model_response is None else _model_write_response(model_response, member_write) except Exception as e: @@ -3465,6 +3467,41 @@ def reload_serving_verdict( return (missing, collateral) +def raise_if_written_params_not_served(model_id: str, patch_data: updateDeployment) -> None: + """Refuse a 200 when the reload restored the previous model, api_base, or api_key. + + A rejected deployment is put back under the same id, so the id-liveness check still passes. + """ + from litellm.proxy.proxy_server import llm_router + + params: Final = patch_data.litellm_params + if llm_router is None or params is None: + return + deployment: Final = llm_router.get_deployment(model_id=model_id) + if not isinstance(deployment, Deployment): + return + requested: Final = tuple( + (field, incoming) + for field in ("model", "api_base", "api_key") + if (incoming := getattr(params, field)) is not None + ) + unserved: Final = tuple( + field for field, incoming in requested if getattr(deployment.litellm_params, field, None) != incoming + ) + if not unserved: + return + raise ProxyException( + message=( + "Model update was saved to the database, but this pod is still serving the previous " + f"{', '.join(unserved)}. The reload kept the last working deployment. " + "Check server logs for 'Error upserting deployment'." + ), + type=ProxyErrorTypes.internal_server_error, + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + param=unserved[0], + ) + + def raise_if_reload_degraded_serving( before: frozenset[str], written_models: Sequence[tuple[str, object]], diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index 57e66805678..1954f40417e 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -57,6 +57,14 @@ def test_get_llm_provider_still_rejects_unregistered_prefix(registered_custom_pr get_llm_provider(model="not-registered-llm/my-model") +@pytest.mark.parametrize("model", ["@cf/cloudflare/clef-flash", "@hf/thebloke/model"]) +def test_get_llm_provider_keeps_workers_ai_model_id(model: str) -> None: + resolved_model, provider, _, _ = get_llm_provider(model=model) + + assert provider == "cloudflare" + assert resolved_model == model + + @pytest.mark.parametrize( ("candidate", "expected"), [(CUSTOM_PROVIDER, True), ("not-registered-llm", False), (None, False), ("", False)], diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 303b8e060ed..f15bd0be309 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -3672,6 +3672,164 @@ class TestUpdateDBModelKeepsLegacyDropParams: assert decrypt_value_helper(value=stored["drop_params"], key="drop_params") == "true" +def _clef_router(*, model: str, api_base: str, api_key: str) -> Router: + return Router( + model_list=[ + { + "model_name": "cloudflare/clef-flash", + "litellm_params": {"model": model, "api_base": api_base, "api_key": api_key}, + "model_info": {"id": "clef-1", "db_model": True}, + } + ], + ignore_invalid_deployments=True, + ) + + +class TestPatchModelServesWrittenParams: + """PATCH must not report success while the router keeps the previous model, api_base, or api_key.""" + + @staticmethod + def _deployment() -> Deployment: + return Deployment( + model_name="cloudflare/clef-flash", + litellm_params=LiteLLM_Params( + model="cloudflare/clef-flash", + api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run", + api_key="old-key", + ), + model_info=ModelInfo(id="clef-1", db_model=True), + ) + + @staticmethod + def _patch() -> updateDeployment: + return updateDeployment( + litellm_params=updateLiteLLMParams( + model="@cf/cloudflare/clef-flash", + api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1", + api_key="new-key", + ) + ) + + @pytest.mark.asyncio + async def test_patch_model_refuses_success_when_reload_keeps_previous_params(self): + router: Final = _clef_router( + model="cloudflare/clef-flash", + api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run", + api_key="old-key", + ) + updated_row: Final = MagicMock() + updated_row.model_dump_json.return_value = "{}" + updated_row.model_info = {"id": "clef-1"} + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch("litellm.proxy.proxy_server.premium_user", True), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.get_db_model", + new=AsyncMock(return_value=self._deployment()), + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db", + new=AsyncMock(return_value=updated_row), + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", + new=AsyncMock( + return_value=ReconcileOutcome( + still_desired=frozenset({"clef-1"}), + live_after=frozenset({"clef-1"}), + ) + ), + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.create_object_audit_log", + new=AsyncMock(), + ), + ): + with pytest.raises(ProxyException) as exc_info: + await patch_model( + model_id="clef-1", + patch_data=self._patch(), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert exc_info.value.code == "500" + assert exc_info.value.param == "model" + served: Final = router.get_deployment(model_id="clef-1") + assert served is not None + assert served.litellm_params.model == "cloudflare/clef-flash" + assert served.litellm_params.api_key == "old-key" + + @pytest.mark.asyncio + async def test_patch_model_accepts_workers_ai_model_id_when_the_router_serves_it(self): + router: Final = _clef_router( + model="cloudflare/clef-flash", + api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run", + api_key="old-key", + ) + updated_row: Final = MagicMock() + updated_row.model_dump_json.return_value = "{}" + updated_row.model_info = {"id": "clef-1"} + + async def serve_written_params() -> ReconcileOutcome: + router.upsert_deployment( + deployment=Deployment( + model_name="cloudflare/clef-flash", + litellm_params=LiteLLM_Params( + model="@cf/cloudflare/clef-flash", + api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1", + api_key="new-key", + ), + model_info=ModelInfo(id="clef-1", db_model=True), + ) + ) + return ReconcileOutcome(still_desired=frozenset({"clef-1"}), live_after=frozenset(router.get_model_ids())) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch("litellm.proxy.proxy_server.premium_user", True), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.get_db_model", + new=AsyncMock(return_value=self._deployment()), + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", + new=AsyncMock(return_value=None), + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db", + new=AsyncMock(return_value=updated_row), + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", + new=AsyncMock(side_effect=serve_written_params), + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.create_object_audit_log", + new=AsyncMock(), + ), + ): + await patch_model( + model_id="clef-1", + patch_data=self._patch(), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + served: Final = router.get_deployment(model_id="clef-1") + assert served is not None + assert served.litellm_params.model == "@cf/cloudflare/clef-flash" + assert served.litellm_params.api_base == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1" + assert served.litellm_params.api_key == "new-key" + + def _build_db_model_with_pricing(): """Wildcard deployment with custom pricing in litellm_params; Deployment.__init__ mirrors SPECIAL_MODEL_INFO_PARAMS into model_info, so both blobs hold the rate.""" diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index f06f9e08f7e..ed78e34f268 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -14240,6 +14240,43 @@ class TestUpsertDeploymentRollback: assert restored.litellm_params.model == "gpt-4o" assert [model["model_name"] for model in router.model_list] == ["prod-model"] + def test_workers_ai_model_id_replaces_the_served_deployment(self): + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + router = litellm.Router( + model_list=[ + { + "model_name": "cloudflare/clef-flash", + "litellm_params": { + "model": "cloudflare/clef-flash", + "api_base": "https://api.cloudflare.com/client/v4/accounts/acct/ai/run", + "api_key": "old-key", + }, + "model_info": {"id": "clef-1", "db_model": True}, + } + ], + ignore_invalid_deployments=True, + ) + + result = router.upsert_deployment( + deployment=Deployment( + model_name="cloudflare/clef-flash", + litellm_params=LiteLLM_Params( + model="@cf/cloudflare/clef-flash", + api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1", + api_key="new-key", + ), + model_info=ModelInfo(id="clef-1", db_model=True), + ) + ) + + assert result is not None + served = router.get_deployment(model_id="clef-1") + assert served is not None + assert served.litellm_params.model == "@cf/cloudflare/clef-flash" + assert served.litellm_params.api_base == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1" + assert served.litellm_params.api_key == "new-key" + def test_failed_fresh_add_returns_none_without_restore(self): from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo From d9ecd77eafc53d70bdb2275788a1246b672478e4 Mon Sep 17 00:00:00 2001 From: pranavyk10 <144428009+pranavyk10@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:25:19 +0530 Subject: [PATCH 2/2] fix(proxy): suppress the workers ai provider rebind The new prefix branch assigns custom_llm_provider, which pushed LIT011 one past the merge-base ceiling. --- litellm/litellm_core_utils/get_llm_provider_logic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 9cd3e976470..221806304bb 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -540,7 +540,7 @@ def get_llm_provider( elif model.startswith(("@cf/", "@hf/")): # Workers AI ids are not `cloudflare/`. Stripping or rejecting them makes a # model update roll back to the previous deployment while PATCH still returns 200. - custom_llm_provider = "cloudflare" + custom_llm_provider = "cloudflare" # rebind-ok: mirrors sibling provider prefix branches # Last resort for an otherwise-unknown model: a declarative # fallback-generalization routing rule (e.g. routes future claude-* to anthropic).