mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge d9ecd77eaf into e0982d6248
This commit is contained in:
commit
0699aa8bf1
5 changed files with 244 additions and 0 deletions
|
|
@ -537,6 +537,10 @@ def get_llm_provider(
|
|||
custom_llm_provider = "amazon_nova"
|
||||
elif model.startswith("sap/"):
|
||||
custom_llm_provider = "sap"
|
||||
elif model.startswith(("@cf/", "@hf/")):
|
||||
# Workers AI ids are not `cloudflare/<id>`. Stripping or rejecting them makes a
|
||||
# model update roll back to the previous deployment while PATCH still returns 200.
|
||||
custom_llm_provider = "cloudflare" # rebind-ok: mirrors sibling provider prefix branches
|
||||
|
||||
# Last resort for an otherwise-unknown model: a declarative
|
||||
# fallback-generalization routing rule (e.g. routes future claude-* to anthropic).
|
||||
|
|
|
|||
|
|
@ -1467,6 +1467,7 @@ async def patch_model(
|
|||
still_desired=reload_outcome.still_desired,
|
||||
live_after=reload_outcome.live_after,
|
||||
)
|
||||
raise_if_written_params_not_served(model_id=model_id, patch_data=patch_data)
|
||||
|
||||
return _model_write_response(updated_model, member_write)
|
||||
|
||||
|
|
@ -3042,6 +3043,7 @@ async def update_model(
|
|||
still_desired=reload_outcome.still_desired,
|
||||
live_after=reload_outcome.live_after,
|
||||
)
|
||||
raise_if_written_params_not_served(model_id=_model_id, patch_data=model_params)
|
||||
|
||||
return None if model_response is None else _model_write_response(model_response, member_write)
|
||||
except Exception as e:
|
||||
|
|
@ -3466,6 +3468,41 @@ def reload_serving_verdict(
|
|||
return (missing, collateral)
|
||||
|
||||
|
||||
def raise_if_written_params_not_served(model_id: str, patch_data: updateDeployment) -> None:
|
||||
"""Refuse a 200 when the reload restored the previous model, api_base, or api_key.
|
||||
|
||||
A rejected deployment is put back under the same id, so the id-liveness check still passes.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
params: Final = patch_data.litellm_params
|
||||
if llm_router is None or params is None:
|
||||
return
|
||||
deployment: Final = llm_router.get_deployment(model_id=model_id)
|
||||
if not isinstance(deployment, Deployment):
|
||||
return
|
||||
requested: Final = tuple(
|
||||
(field, incoming)
|
||||
for field in ("model", "api_base", "api_key")
|
||||
if (incoming := getattr(params, field)) is not None
|
||||
)
|
||||
unserved: Final = tuple(
|
||||
field for field, incoming in requested if getattr(deployment.litellm_params, field, None) != incoming
|
||||
)
|
||||
if not unserved:
|
||||
return
|
||||
raise ProxyException(
|
||||
message=(
|
||||
"Model update was saved to the database, but this pod is still serving the previous "
|
||||
f"{', '.join(unserved)}. The reload kept the last working deployment. "
|
||||
"Check server logs for 'Error upserting deployment'."
|
||||
),
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
param=unserved[0],
|
||||
)
|
||||
|
||||
|
||||
def raise_if_reload_degraded_serving(
|
||||
before: frozenset[str],
|
||||
written_models: Sequence[tuple[str, object]],
|
||||
|
|
|
|||
|
|
@ -57,6 +57,14 @@ def test_get_llm_provider_still_rejects_unregistered_prefix(registered_custom_pr
|
|||
get_llm_provider(model="not-registered-llm/my-model")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["@cf/cloudflare/clef-flash", "@hf/thebloke/model"])
|
||||
def test_get_llm_provider_keeps_workers_ai_model_id(model: str) -> None:
|
||||
resolved_model, provider, _, _ = get_llm_provider(model=model)
|
||||
|
||||
assert provider == "cloudflare"
|
||||
assert resolved_model == model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("candidate", "expected"),
|
||||
[(CUSTOM_PROVIDER, True), ("not-registered-llm", False), (None, False), ("", False)],
|
||||
|
|
|
|||
|
|
@ -3672,6 +3672,164 @@ class TestUpdateDBModelKeepsLegacyDropParams:
|
|||
assert decrypt_value_helper(value=stored["drop_params"], key="drop_params") == "true"
|
||||
|
||||
|
||||
def _clef_router(*, model: str, api_base: str, api_key: str) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "cloudflare/clef-flash",
|
||||
"litellm_params": {"model": model, "api_base": api_base, "api_key": api_key},
|
||||
"model_info": {"id": "clef-1", "db_model": True},
|
||||
}
|
||||
],
|
||||
ignore_invalid_deployments=True,
|
||||
)
|
||||
|
||||
|
||||
class TestPatchModelServesWrittenParams:
|
||||
"""PATCH must not report success while the router keeps the previous model, api_base, or api_key."""
|
||||
|
||||
@staticmethod
|
||||
def _deployment() -> Deployment:
|
||||
return Deployment(
|
||||
model_name="cloudflare/clef-flash",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="cloudflare/clef-flash",
|
||||
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run",
|
||||
api_key="old-key",
|
||||
),
|
||||
model_info=ModelInfo(id="clef-1", db_model=True),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _patch() -> updateDeployment:
|
||||
return updateDeployment(
|
||||
litellm_params=updateLiteLLMParams(
|
||||
model="@cf/cloudflare/clef-flash",
|
||||
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1",
|
||||
api_key="new-key",
|
||||
)
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_model_refuses_success_when_reload_keeps_previous_params(self):
|
||||
router: Final = _clef_router(
|
||||
model="cloudflare/clef-flash",
|
||||
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run",
|
||||
api_key="old-key",
|
||||
)
|
||||
updated_row: Final = MagicMock()
|
||||
updated_row.model_dump_json.return_value = "{}"
|
||||
updated_row.model_info = {"id": "clef-1"}
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.llm_router", router),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True),
|
||||
patch("litellm.proxy.proxy_server.premium_user", True),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.get_db_model",
|
||||
new=AsyncMock(return_value=self._deployment()),
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
|
||||
new=AsyncMock(return_value=None),
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db",
|
||||
new=AsyncMock(return_value=updated_row),
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
|
||||
new=AsyncMock(
|
||||
return_value=ReconcileOutcome(
|
||||
still_desired=frozenset({"clef-1"}),
|
||||
live_after=frozenset({"clef-1"}),
|
||||
)
|
||||
),
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.create_object_audit_log",
|
||||
new=AsyncMock(),
|
||||
),
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await patch_model(
|
||||
model_id="clef-1",
|
||||
patch_data=self._patch(),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "500"
|
||||
assert exc_info.value.param == "model"
|
||||
served: Final = router.get_deployment(model_id="clef-1")
|
||||
assert served is not None
|
||||
assert served.litellm_params.model == "cloudflare/clef-flash"
|
||||
assert served.litellm_params.api_key == "old-key"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_model_accepts_workers_ai_model_id_when_the_router_serves_it(self):
|
||||
router: Final = _clef_router(
|
||||
model="cloudflare/clef-flash",
|
||||
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/run",
|
||||
api_key="old-key",
|
||||
)
|
||||
updated_row: Final = MagicMock()
|
||||
updated_row.model_dump_json.return_value = "{}"
|
||||
updated_row.model_info = {"id": "clef-1"}
|
||||
|
||||
async def serve_written_params() -> ReconcileOutcome:
|
||||
router.upsert_deployment(
|
||||
deployment=Deployment(
|
||||
model_name="cloudflare/clef-flash",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="@cf/cloudflare/clef-flash",
|
||||
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1",
|
||||
api_key="new-key",
|
||||
),
|
||||
model_info=ModelInfo(id="clef-1", db_model=True),
|
||||
)
|
||||
)
|
||||
return ReconcileOutcome(still_desired=frozenset({"clef-1"}), live_after=frozenset(router.get_model_ids()))
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.llm_router", router),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True),
|
||||
patch("litellm.proxy.proxy_server.premium_user", True),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.get_db_model",
|
||||
new=AsyncMock(return_value=self._deployment()),
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
|
||||
new=AsyncMock(return_value=None),
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints._update_team_model_in_db",
|
||||
new=AsyncMock(return_value=updated_row),
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
|
||||
new=AsyncMock(side_effect=serve_written_params),
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.create_object_audit_log",
|
||||
new=AsyncMock(),
|
||||
),
|
||||
):
|
||||
await patch_model(
|
||||
model_id="clef-1",
|
||||
patch_data=self._patch(),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
served: Final = router.get_deployment(model_id="clef-1")
|
||||
assert served is not None
|
||||
assert served.litellm_params.model == "@cf/cloudflare/clef-flash"
|
||||
assert served.litellm_params.api_base == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1"
|
||||
assert served.litellm_params.api_key == "new-key"
|
||||
|
||||
|
||||
def _build_db_model_with_pricing():
|
||||
"""Wildcard deployment with custom pricing in litellm_params; Deployment.__init__
|
||||
mirrors SPECIAL_MODEL_INFO_PARAMS into model_info, so both blobs hold the rate."""
|
||||
|
|
|
|||
|
|
@ -14371,6 +14371,43 @@ class TestUpsertDeploymentRollback:
|
|||
assert restored.litellm_params.model == "gpt-4o"
|
||||
assert [model["model_name"] for model in router.model_list] == ["prod-model"]
|
||||
|
||||
def test_workers_ai_model_id_replaces_the_served_deployment(self):
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "cloudflare/clef-flash",
|
||||
"litellm_params": {
|
||||
"model": "cloudflare/clef-flash",
|
||||
"api_base": "https://api.cloudflare.com/client/v4/accounts/acct/ai/run",
|
||||
"api_key": "old-key",
|
||||
},
|
||||
"model_info": {"id": "clef-1", "db_model": True},
|
||||
}
|
||||
],
|
||||
ignore_invalid_deployments=True,
|
||||
)
|
||||
|
||||
result = router.upsert_deployment(
|
||||
deployment=Deployment(
|
||||
model_name="cloudflare/clef-flash",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="@cf/cloudflare/clef-flash",
|
||||
api_base="https://api.cloudflare.com/client/v4/accounts/acct/ai/v1",
|
||||
api_key="new-key",
|
||||
),
|
||||
model_info=ModelInfo(id="clef-1", db_model=True),
|
||||
)
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
served = router.get_deployment(model_id="clef-1")
|
||||
assert served is not None
|
||||
assert served.litellm_params.model == "@cf/cloudflare/clef-flash"
|
||||
assert served.litellm_params.api_base == "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1"
|
||||
assert served.litellm_params.api_key == "new-key"
|
||||
|
||||
def test_failed_fresh_add_returns_none_without_restore(self):
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue