This commit is contained in:
pranavyk10 2026-10-26 12:30:17 -05:00 • committed by GitHub
commit 0699aa8bf1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 244 additions and 0 deletions

View file

@ -537,6 +537,10 @@ def get_llm_provider(
custom_llm_provider = "amazon_nova"
elif model.startswith("sap/"):
custom_llm_provider = "sap"
elif model.startswith(("@cf/", "@hf/")):
# Workers AI ids are not `cloudflare/<id>`. Stripping or rejecting them makes a
# model update roll back to the previous deployment while PATCH still returns 200.
custom_llm_provider = "cloudflare" # rebind-ok: mirrors sibling provider prefix branches
# Last resort for an otherwise-unknown model: a declarative
# fallback-generalization routing rule (e.g. routes future claude-* to anthropic).

View file

@ -1467,6 +1467,7 @@ async def patch_model(
still_desired=reload_outcome.still_desired,
live_after=reload_outcome.live_after,
)
raise_if_written_params_not_served(model_id=model_id, patch_data=patch_data)
return _model_write_response(updated_model, member_write)
@ -3042,6 +3043,7 @@ async def update_model(
still_desired=reload_outcome.still_desired,
live_after=reload_outcome.live_after,
)
raise_if_written_params_not_served(model_id=_model_id, patch_data=model_params)
return None if model_response is None else _model_write_response(model_response, member_write)
except Exception as e:
@ -3466,6 +3468,41 @@ def reload_serving_verdict(
return (missing, collateral)
def raise_if_written_params_not_served(model_id: str, patch_data: updateDeployment) -> None:
"""Refuse a 200 when the reload restored the previous model, api_base, or api_key.
A rejected deployment is put back under the same id, so the id-liveness check still passes.
"""
from litellm.proxy.proxy_server import llm_router
params: Final = patch_data.litellm_params
if llm_router is None or params is None:
return
deployment: Final = llm_router.get_deployment(model_id=model_id)
if not isinstance(deployment, Deployment):
return
requested: Final = tuple(
(field, incoming)
for field in ("model", "api_base", "api_key")
if (incoming := getattr(params, field)) is not None
)
unserved: Final = tuple(
field for field, incoming in requested if getattr(deployment.litellm_params, field, None) != incoming
)
if not unserved:
return
raise ProxyException(
message=(
"Model update was saved to the database, but this pod is still serving the previous "
f"{', '.join(unserved)}. The reload kept the last working deployment. "
"Check server logs for 'Error upserting deployment'."
),
type=ProxyErrorTypes.internal_server_error,
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
param=unserved[0],
)
def raise_if_reload_degraded_serving(
before: frozenset[str],
written_models: Sequence[tuple[str, object]],

View file

@ -57,6 +57,14 @@ def test_get_llm_provider_still_rejects_unregistered_prefix(registered_custom_pr
get_llm_provider(model="not-registered-llm/my-model")
@pytest.mark.parametrize("model", ["@cf/cloudflare/clef-flash", "@hf/thebloke/model"])
def test_get_llm_provider_keeps_workers_ai_model_id(model: str) -> None:
resolved_model, provider, _, _ = get_llm_provider(model=model)
assert provider == "cloudflare"
assert resolved_model == model
@pytest.mark.parametrize(
("candidate", "expected"),
[(CUSTOM_PROVIDER, True), ("not-registered-llm", False), (None, False), ("", False)],

View file

@ -3672,6 +3672,164 @@ class TestUpdateDBModelKeepsLegacyDropParams:
assert decrypt_value_helper(value=stored["drop_params"], key="drop_params") == "true"
def _clef_router(*, model: str, api_base: str, api_key: str) -> Router:
return Router(
model_list=[
{
"model_name": "cloudflare/clef-flash",
"litellm_params": {"model": model, "api_base": api_base, "api_key": api_key},
"model_info": {"id": "clef-1", "db_model": True},
}
],
ignore_invalid_deployments=True,
)
class TestPatchModelServesWrittenParams:
"""PATCH must not report success while the router keeps the previous model, api_base, or api_key."""
@staticmethod
def _deployment() -> Deployment:
return Deployment(
model_name="cloudflare/clef-flash",
litellm_params=LiteLLM_Params(
model="cloudflare/clef-flash",
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run",
api_key="old-key",
),
model_info=ModelInfo(id="clef-1", db_model=True),
)
@staticmethod
def _patch() -> updateDeployment:
return updateDeployment(
litellm_params=updateLiteLLMParams(
model="@cf/cloudflare/clef-flash",
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1",
api_key="new-key",
)
)
@pytest.mark.asyncio
async def test_patch_model_refuses_success_when_reload_keeps_previous_params(self):
router: Final = _clef_router(
model="cloudflare/clef-flash",
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run",
api_key="old-key",
)
updated_row: Final = MagicMock()
updated_row.model_dump_json.return_value = "{}"
updated_row.model_info = {"id": "clef-1"}
with (
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
patch("litellm.proxy.proxy_server.llm_router", router),
patch("litellm.proxy.proxy_server.store_model_in_db", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints.get_db_model",
new=AsyncMock(return_value=self._deployment()),
),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
new=AsyncMock(return_value=None),
),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db",
new=AsyncMock(return_value=updated_row),
),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
new=AsyncMock(
return_value=ReconcileOutcome(
still_desired=frozenset({"clef-1"}),
live_after=frozenset({"clef-1"}),
)
),
),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints.create_object_audit_log",
new=AsyncMock(),
),
):
with pytest.raises(ProxyException) as exc_info:
await patch_model(
model_id="clef-1",
patch_data=self._patch(),
user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert exc_info.value.code == "500"
assert exc_info.value.param == "model"
served: Final = router.get_deployment(model_id="clef-1")
assert served is not None
assert served.litellm_params.model == "cloudflare/clef-flash"
assert served.litellm_params.api_key == "old-key"
@pytest.mark.asyncio
async def test_patch_model_accepts_workers_ai_model_id_when_the_router_serves_it(self):
router: Final = _clef_router(
model="cloudflare/clef-flash",
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run",
api_key="old-key",
)
updated_row: Final = MagicMock()
updated_row.model_dump_json.return_value = "{}"
updated_row.model_info = {"id": "clef-1"}
async def serve_written_params() -> ReconcileOutcome:
router.upsert_deployment(
deployment=Deployment(
model_name="cloudflare/clef-flash",
litellm_params=LiteLLM_Params(
model="@cf/cloudflare/clef-flash",
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1",
api_key="new-key",
),
model_info=ModelInfo(id="clef-1", db_model=True),
)
)
return ReconcileOutcome(still_desired=frozenset({"clef-1"}), live_after=frozenset(router.get_model_ids()))
with (
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
patch("litellm.proxy.proxy_server.llm_router", router),
patch("litellm.proxy.proxy_server.store_model_in_db", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints.get_db_model",
new=AsyncMock(return_value=self._deployment()),
),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
new=AsyncMock(return_value=None),
),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db",
new=AsyncMock(return_value=updated_row),
),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
new=AsyncMock(side_effect=serve_written_params),
),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints.create_object_audit_log",
new=AsyncMock(),
),
):
await patch_model(
model_id="clef-1",
patch_data=self._patch(),
user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
)
served: Final = router.get_deployment(model_id="clef-1")
assert served is not None
assert served.litellm_params.model == "@cf/cloudflare/clef-flash"
assert served.litellm_params.api_base == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1"
assert served.litellm_params.api_key == "new-key"
def _build_db_model_with_pricing():
"""Wildcard deployment with custom pricing in litellm_params; Deployment.__init__
mirrors SPECIAL_MODEL_INFO_PARAMS into model_info, so both blobs hold the rate."""

View file

@ -14371,6 +14371,43 @@ class TestUpsertDeploymentRollback:
assert restored.litellm_params.model == "gpt-4o"
assert [model["model_name"] for model in router.model_list] == ["prod-model"]
def test_workers_ai_model_id_replaces_the_served_deployment(self):
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
router = litellm.Router(
model_list=[
{
"model_name": "cloudflare/clef-flash",
"litellm_params": {
"model": "cloudflare/clef-flash",
"api_base": "https://api.cloudflare.com/client/v4/accounts/acct/ai/run",
"api_key": "old-key",
},
"model_info": {"id": "clef-1", "db_model": True},
}
],
ignore_invalid_deployments=True,
)
result = router.upsert_deployment(
deployment=Deployment(
model_name="cloudflare/clef-flash",
litellm_params=LiteLLM_Params(
model="@cf/cloudflare/clef-flash",
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1",
api_key="new-key",
),
model_info=ModelInfo(id="clef-1", db_model=True),
)
)
assert result is not None
served = router.get_deployment(model_id="clef-1")
assert served is not None
assert served.litellm_params.model == "@cf/cloudflare/clef-flash"
assert served.litellm_params.api_base == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1"
assert served.litellm_params.api_key == "new-key"
def test_failed_fresh_add_returns_none_without_restore(self):
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo