From 5e1d9705dbe09e059f418d222b790278fcb16291 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:45:00 +0000 Subject: [PATCH] fix(proxy): skip team model aliases that point at deleted deployments A team's model_aliases can map a public name like gpt-4 to the internal routing key (model_name_{team_id}_{uuid}) of a team deployment that has since been deleted, e.g. after replacing per-team duplicates with one gateway-level model. The pre-call rewrite then sent every request to a name the router cannot serve, failing with "no healthy deployments for model_name_..." even though the requested name still resolves at the gateway level. The rewrite is now skipped when the alias target has no live deployment in the router delete_model also skipped the team alias scan for internal-shaped names on the assumption they can never be alias values, which is exactly the shape legacy team model aliases have, so deleting a legacy team model left the stale alias behind. The scan now always runs, and a public name that still resolves to a live router deployment (e.g. a shared gateway-level model group) stays in team.models so the delete does not revoke the team's access to it --- litellm/proxy/litellm_pre_call_utils.py | 99 ++++++++++++------- .../model_management_endpoints.py | 45 ++++++--- tests/proxy_unit_tests/test_proxy_utils.py | 6 +- .../test_model_management_endpoints.py | 95 +++++++++++++++++- .../proxy/test_litellm_pre_call_utils.py | 61 ++++++++++++ 5 files changed, 250 insertions(+), 56 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d94fed0ee5b..c40718e41fd 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1829,6 +1829,15 @@ async def add_litellm_data_to_request( return data +def _warn_stale_team_alias_once(warning_key: str, message: str, *args: str) -> None: + if warning_key in _STALE_TEAM_ALIAS_WARNING_KEYS: + return + _STALE_TEAM_ALIAS_WARNING_KEYS[warning_key] = None + while len(_STALE_TEAM_ALIAS_WARNING_KEYS) > _MAX_STALE_ALIAS_WARNING_KEYS: + _STALE_TEAM_ALIAS_WARNING_KEYS.popitem(last=False) + verbose_proxy_logger.warning(message, *args) + + def _update_model_if_team_alias_exists( data: dict, user_api_key_dict: UserAPIKeyAuth, @@ -1848,49 +1857,63 @@ def _update_model_if_team_alias_exists( Note: model_aliases for team models are deprecated. This function only applies to legacy non-team-scoped aliases. Team-scoped deployments use team_public_model_name and are resolved via map_team_model in route_llm_request. + + An alias that targets a team-scoped internal name (``model_name_{team_id}_{uuid}``) + with no live deployment behind it is never applied: the deployment was deleted, so + the rewrite could only fail with an error naming a model the caller never sent. + Keeping the requested model name lets it resolve against the deployments that still + exist (e.g. a gateway-level model group shared with the team). """ _model = data.get("model") - if _model and user_api_key_dict.team_model_aliases and _model in user_api_key_dict.team_model_aliases: - from litellm.proxy.proxy_server import llm_router + if not _model or not user_api_key_dict.team_model_aliases or _model not in user_api_key_dict.team_model_aliases: + return - # Skip alias rewrite if this model resolves to team-specific deployments - # (team models use team_public_model_name, not model_aliases) - aliased_target = user_api_key_dict.team_model_aliases[_model] + from litellm.proxy.proxy_server import llm_router - # Optional bypass for stale aliases from pre-PR deployments: - # only enabled via feature flag to preserve backwards compatibility. - # Cached at module level to avoid hot-path secret lookups on every request. - global _ENABLE_TEAM_STALE_ALIAS_BYPASS - if _ENABLE_TEAM_STALE_ALIAS_BYPASS is None: - _ENABLE_TEAM_STALE_ALIAS_BYPASS = get_secret_bool("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", False) - enable_stale_alias_bypass = _ENABLE_TEAM_STALE_ALIAS_BYPASS - # Check if the alias points to a team-scoped UUID name - # (format: "model_name_{team_id}_{uuid}") - is_stale_team_alias = aliased_target.startswith(f"model_name_{user_api_key_dict.team_id}_") - if is_stale_team_alias and llm_router: - # This is a stale alias from pre-PR deployments. - # Check if current team deployments exist for the public name. - key = (user_api_key_dict.team_id, _model) - if key in llm_router.team_model_to_deployment_indices: - if enable_stale_alias_bypass: - # Team deployments exist; skip stale alias - return - warning_key = f"{user_api_key_dict.team_id}:{_model}:{aliased_target}" - if warning_key not in _STALE_TEAM_ALIAS_WARNING_KEYS: - _STALE_TEAM_ALIAS_WARNING_KEYS[warning_key] = None - while len(_STALE_TEAM_ALIAS_WARNING_KEYS) > _MAX_STALE_ALIAS_WARNING_KEYS: - _STALE_TEAM_ALIAS_WARNING_KEYS.popitem(last=False) - verbose_proxy_logger.warning( - "Stale team model alias detected for model='%s', team_id='%s'. " - "New sibling deployments may be unreachable. " - "Set LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true to enable " - "team-scoped sibling routing.", - _sanitize_for_log(_model), - user_api_key_dict.team_id, - ) + # Skip alias rewrite if this model resolves to team-specific deployments + # (team models use team_public_model_name, not model_aliases) + aliased_target = user_api_key_dict.team_model_aliases[_model] - data["model"] = aliased_target - return + # Optional bypass for stale aliases from pre-PR deployments: + # only enabled via feature flag to preserve backwards compatibility. + # Cached at module level to avoid hot-path secret lookups on every request. + global _ENABLE_TEAM_STALE_ALIAS_BYPASS + if _ENABLE_TEAM_STALE_ALIAS_BYPASS is None: + _ENABLE_TEAM_STALE_ALIAS_BYPASS = get_secret_bool("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", False) + enable_stale_alias_bypass = _ENABLE_TEAM_STALE_ALIAS_BYPASS + # Check if the alias points to a team-scoped UUID name + # (format: "model_name_{team_id}_{uuid}") + is_stale_team_alias = aliased_target.startswith(f"model_name_{user_api_key_dict.team_id}_") + if is_stale_team_alias and llm_router: + if aliased_target not in llm_router.model_name_to_deployment_indices: + _warn_stale_team_alias_once( + f"deleted:{user_api_key_dict.team_id}:{_model}:{aliased_target}", + "Team model alias for model='%s', team_id='%s' targets '%s', which has no live " + "deployment. Routing with the requested model name instead; remove the stale " + "entry from the team's model_aliases to silence this warning.", + _sanitize_for_log(_model), + _sanitize_for_log(user_api_key_dict.team_id), + _sanitize_for_log(aliased_target), + ) + return + # This is a stale alias from pre-PR deployments. + # Check if current team deployments exist for the public name. + key = (user_api_key_dict.team_id, _model) + if key in llm_router.team_model_to_deployment_indices: + if enable_stale_alias_bypass: + # Team deployments exist; skip stale alias + return + _warn_stale_team_alias_once( + f"{user_api_key_dict.team_id}:{_model}:{aliased_target}", + "Stale team model alias detected for model='%s', team_id='%s'. " + "New sibling deployments may be unreachable. " + "Set LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true to enable " + "team-scoped sibling routing.", + _sanitize_for_log(_model), + _sanitize_for_log(user_api_key_dict.team_id), + ) + + data["model"] = aliased_target def _update_model_if_key_alias_exists( diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 91f0b9b2790..ddb7f08ad15 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -52,6 +52,7 @@ from litellm.proxy.utils import PrismaClient from litellm.repositories.model_repository import ModelRepository from litellm.repositories.table_repositories import ModelTableRepository from litellm.repositories.team_repository import TeamRepository +from litellm.router import Router from litellm.types.proxy.management_endpoints.model_management_endpoints import ( UpdateUsefulLinksRequest, ) @@ -788,6 +789,7 @@ async def _remove_unbacked_team_models( prisma_client: PrismaClient, user_api_key_cache: Any, proxy_logging_obj: Any, + llm_router: Router | None = None, ) -> None: """ Strip a deleted team model's public name(s) from team.models and refresh the cache. @@ -795,26 +797,40 @@ async def _remove_unbacked_team_models( Must be called after the deployment row is deleted: a public name is removed only when no remaining team deployment still backs it, so a load-balanced replica isn't revoked while siblings serve it, and concurrent deletes can't leave a ghost. + + Legacy team models (created before team_public_model_name existed) store a + ``{public_name: "model_name_{team_id}_{uuid}"}`` entry in the team's model_aliases, + so the alias scan runs for every team model; skipping it for internal-shaped names + left stale aliases that rewrote requests to deployments that no longer exist. + + A public name that still resolves to a live router deployment (e.g. a gateway-level + model group shared with the team) is kept in team.models, so deleting a per-team + duplicate does not revoke the team's access to the shared deployment. """ team_id = model_params.model_info.team_id if team_id is None: return - # BYOK models carry an internal `model_name_{team_id}_{uuid}` name that can never - # be a team alias value, so skip the full litellm_modeltable scan for them. - removed_model_aliases: List[Tuple[str, str]] = [] - if not model_params.model_name.startswith(f"model_name_{team_id}_"): - removed_model_aliases = await delete_team_model_alias( - public_model_name=model_params.model_name, - prisma_client=prisma_client, - ) - names_to_remove = {alias for alias_team_id, alias in removed_model_aliases if alias_team_id == team_id} - if model_params.model_info.team_public_model_name is not None: - names_to_remove.add(model_params.model_info.team_public_model_name) - - if names_to_remove: - names_to_remove -= await _get_team_public_model_names(team_id=team_id, prisma_client=prisma_client) + removed_model_aliases = await delete_team_model_alias( + public_model_name=model_params.model_name, + prisma_client=prisma_client, + ) + removed_alias_names = {alias for alias_team_id, alias in removed_model_aliases if alias_team_id == team_id} + candidate_names = ( + removed_alias_names | {model_params.model_info.team_public_model_name} + if model_params.model_info.team_public_model_name is not None + else removed_alias_names + ) + if not candidate_names: + return + team_backed_names = await _get_team_public_model_names(team_id=team_id, prisma_client=prisma_client) + router_served_names = ( + frozenset(name for name in candidate_names if name in llm_router.model_name_to_deployment_indices) + if llm_router is not None + else frozenset() + ) + names_to_remove = candidate_names - team_backed_names - router_served_names if not names_to_remove: return @@ -1120,6 +1136,7 @@ async def delete_model( prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, ) ## CREATE AUDIT LOG ## diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index d2218b08386..ad852c16905 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -2184,7 +2184,8 @@ def test_team_alias_stale_bypass_disabled_by_default(monkeypatch): pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None class _MockRouter: - team_model_to_deployment_indices = {("team-1", "gpt-4o"): [0]} + model_name_to_deployment_indices = {"model_name_team-1_legacy-uuid": [0]} + team_model_to_deployment_indices = {("team-1", "gpt-4o"): [1]} test_data = {"model": "gpt-4o"} user_api_key_dict = UserAPIKeyAuth( @@ -2209,7 +2210,8 @@ def test_team_alias_stale_bypass_enabled_by_flag(monkeypatch): pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None class _MockRouter: - team_model_to_deployment_indices = {("team-1", "gpt-4o"): [0]} + model_name_to_deployment_indices = {"model_name_team-1_legacy-uuid": [0]} + team_model_to_deployment_indices = {("team-1", "gpt-4o"): [1]} test_data = {"model": "gpt-4o"} user_api_key_dict = UserAPIKeyAuth( diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index bd5eda0197b..54c17a845d2 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -2031,8 +2031,7 @@ class TestDeleteTeamBYOKModelGhost: mock_refresh.assert_awaited_once() assert mock_refresh.await_args.kwargs["team_row"] is updated_team_row - # BYOK internal name can't be an alias value -> the alias-table scan is skipped. - mock_prisma.db.litellm_modeltable.find_many.assert_not_awaited() + mock_prisma.db.litellm_modeltable.find_many.assert_awaited() @pytest.mark.asyncio async def test_delete_non_internal_team_model_still_scans_aliases(self): @@ -2186,6 +2185,98 @@ class TestDeleteTeamBYOKModelGhost: mock_prisma.db.litellm_teamtable.update.assert_not_awaited() mock_refresh.assert_not_awaited() + @pytest.mark.asyncio + async def test_delete_legacy_team_model_scrubs_stale_alias_and_keeps_gateway_access( + self, + ): + """Regression: legacy team models store {public_name: internal model_name} in + the team's model_aliases. delete_model skipped the alias scan for + internal-shaped names, so the stale alias kept rewriting requests for the + public name to a deployment that no longer existed ("no healthy deployments + for model_name_{team_id}_..."). Deleting the deployment must scrub the + alias, and the public name must stay in team.models while a gateway-level + deployment still serves it.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + ModelInfoDelete, + delete_model as delete_model_endpoint, + ) + + team_id = "team-legacy-alias" + model_id = "legacy-alias-model-1" + public_name = "gpt-4" + internal_name = f"model_name_{team_id}_abc-uuid" + + db_row = LiteLLM_ProxyModelTable( + model_id=model_id, + model_name=internal_name, + litellm_params={"model": "openai/gpt-4.1-nano"}, + model_info={"id": model_id, "team_id": team_id}, + created_by="admin", + updated_by="admin", + ) + team_row = LiteLLM_TeamTable( + team_id=team_id, + team_alias="legacy-alias-team", + members_with_roles=[Member(user_id="admin", role="admin")], + models=[public_name], + ) + alias_row = MagicMock( + id="alias-row-1", model_aliases={public_name: internal_name} + ) + alias_row.team = MagicMock() + alias_row.team.team_id = team_id + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_proxymodeltable = AsyncMock() + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock( + return_value=db_row + ) + mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row) + mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_teamtable = AsyncMock() + mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row) + mock_prisma.db.litellm_teamtable.update = AsyncMock(return_value=team_row) + mock_prisma.db.litellm_modeltable = AsyncMock() + mock_prisma.db.litellm_modeltable.find_many = AsyncMock( + return_value=[alias_row] + ) + mock_prisma.db.litellm_modeltable.update = AsyncMock() + + mock_router = MagicMock() + mock_router.model_name_to_deployment_indices = {public_name: [0]} + + admin_user = UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + + _PS = "litellm.proxy.proxy_server" + _MOD = "litellm.proxy.management_endpoints.model_management_endpoints" + with ( + patch(f"{_PS}.prisma_client", mock_prisma), + patch(f"{_PS}.store_model_in_db", True), + patch(f"{_PS}.premium_user", True), + patch(f"{_PS}.llm_router", mock_router), + patch(f"{_PS}.proxy_logging_obj", MagicMock()), + patch(f"{_PS}.user_api_key_cache", MagicMock()), + patch(f"{_MOD}._refresh_cached_team", new=AsyncMock()) as mock_refresh, + ): + result = await delete_model_endpoint( + model_info=ModelInfoDelete(id=model_id), + user_api_key_dict=admin_user, + ) + + assert "deleted successfully" in result["message"] + + mock_prisma.db.litellm_modeltable.update.assert_awaited_once() + alias_update_kwargs = mock_prisma.db.litellm_modeltable.update.await_args.kwargs + assert alias_update_kwargs["where"] == {"id": "alias-row-1"} + assert json.loads(alias_update_kwargs["data"]["model_aliases"]) == {} + + # A gateway-level deployment still serves the public name -> team access stays. + mock_prisma.db.litellm_teamtable.update.assert_not_awaited() + mock_refresh.assert_not_awaited() + class TestDeleteModelTeamAuth: """Team auth on the /model/delete path. diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 143884c8a0d..36fde1b10d8 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -5457,3 +5457,64 @@ def test_get_sanitized_user_information_from_key_drops_callback_config(): # UserAPIKeyAuth is the live auth object; the per-key callbacks are resolved # from it during pre-call, so it must not be mutated by building the log view assert "logging" in (user_api_key_dict.metadata or {}) + + +def test_team_alias_targeting_deleted_team_deployment_keeps_requested_model(monkeypatch): + """ + Regression: a team's model_aliases can point at the internal routing key + (model_name_{team_id}_{uuid}) of a team deployment that was since deleted, + e.g. after an admin replaces per-team duplicates with one gateway-level + model. Rewriting to the dead internal name made every request fail with + "no healthy deployments for model_name_..." even though the requested + public name resolves at the gateway level. The rewrite must be skipped + when the alias target has no live deployment. + """ + import litellm.proxy.litellm_pre_call_utils as pre_call_utils + from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists + + monkeypatch.delenv("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", raising=False) + pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None + + class _MockRouter: + model_name_to_deployment_indices = {"gpt-4": [0]} + team_model_to_deployment_indices = {} + + test_data = {"model": "gpt-4"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + team_id="team-1", + team_model_aliases={"gpt-4": "model_name_team-1_dead-uuid"}, + ) + + with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()): + _update_model_if_team_alias_exists( + data=test_data, user_api_key_dict=user_api_key_dict + ) + + assert test_data.get("model") == "gpt-4" + + +def test_team_alias_targeting_live_team_deployment_still_rewrites(monkeypatch): + import litellm.proxy.litellm_pre_call_utils as pre_call_utils + from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists + + monkeypatch.delenv("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", raising=False) + pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None + + class _MockRouter: + model_name_to_deployment_indices = {"model_name_team-1_live-uuid": [0]} + team_model_to_deployment_indices = {} + + test_data = {"model": "gpt-4"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + team_id="team-1", + team_model_aliases={"gpt-4": "model_name_team-1_live-uuid"}, + ) + + with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()): + _update_model_if_team_alias_exists( + data=test_data, user_api_key_dict=user_api_key_dict + ) + + assert test_data.get("model") == "model_name_team-1_live-uuid"