scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

fix(proxy): skip team model aliases that point at deleted deployments

Browse source 
A team's model_aliases can map a public name like gpt-4 to the internal
routing key (model_name_{team_id}_{uuid}) of a team deployment that has
since been deleted, e.g. after replacing per-team duplicates with one
gateway-level model. The pre-call rewrite then sent every request to a
name the router cannot serve, failing with "no healthy deployments for
model_name_..." even though the requested name still resolves at the
gateway level. The rewrite is now skipped when the alias target has no
live deployment in the router

delete_model also skipped the team alias scan for internal-shaped names
on the assumption they can never be alias values, which is exactly the
shape legacy team model aliases have, so deleting a legacy team model
left the stale alias behind. The scan now always runs, and a public
name that still resolves to a live router deployment (e.g. a shared
gateway-level model group) stays in team.models so the delete does not
revoke the team's access to it
This commit is contained in:
mateo-berri 2026-07-28 19:45:00 +00:00
parent f4a68a75ff
commit 5e1d9705db
No known key found for this signature in database
5 changed files with 250 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
litellm/proxy/litellm_pre_call_utils.py
View file

@ -1829,6 +1829,15 @@ async def add_litellm_data_to_request(
return data
def _warn_stale_team_alias_once(warning_key: str, message: str, *args: str) -> None:
if warning_key in _STALE_TEAM_ALIAS_WARNING_KEYS:
return
_STALE_TEAM_ALIAS_WARNING_KEYS[warning_key] = None
while len(_STALE_TEAM_ALIAS_WARNING_KEYS) > _MAX_STALE_ALIAS_WARNING_KEYS:
_STALE_TEAM_ALIAS_WARNING_KEYS.popitem(last=False)
verbose_proxy_logger.warning(message, *args)
def _update_model_if_team_alias_exists(
data: dict,
user_api_key_dict: UserAPIKeyAuth,
@ -1848,49 +1857,63 @@ def _update_model_if_team_alias_exists(
Note: model_aliases for team models are deprecated. This function only applies
to legacy non-team-scoped aliases. Team-scoped deployments use team_public_model_name
and are resolved via map_team_model in route_llm_request.
An alias that targets a team-scoped internal name (``model_name_{team_id}_{uuid}``)
with no live deployment behind it is never applied: the deployment was deleted, so
the rewrite could only fail with an error naming a model the caller never sent.
Keeping the requested model name lets it resolve against the deployments that still
exist (e.g. a gateway-level model group shared with the team).
"""
_model = data.get("model")
if _model and user_api_key_dict.team_model_aliases and _model in user_api_key_dict.team_model_aliases:
from litellm.proxy.proxy_server import llm_router
if not _model or not user_api_key_dict.team_model_aliases or _model not in user_api_key_dict.team_model_aliases:
return
# Skip alias rewrite if this model resolves to team-specific deployments
# (team models use team_public_model_name, not model_aliases)
aliased_target = user_api_key_dict.team_model_aliases[_model]
from litellm.proxy.proxy_server import llm_router
# Optional bypass for stale aliases from pre-PR deployments:
# only enabled via feature flag to preserve backwards compatibility.
# Cached at module level to avoid hot-path secret lookups on every request.
global _ENABLE_TEAM_STALE_ALIAS_BYPASS
if _ENABLE_TEAM_STALE_ALIAS_BYPASS is None:
_ENABLE_TEAM_STALE_ALIAS_BYPASS = get_secret_bool("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", False)
enable_stale_alias_bypass = _ENABLE_TEAM_STALE_ALIAS_BYPASS
# Check if the alias points to a team-scoped UUID name
# (format: "model_name_{team_id}_{uuid}")
is_stale_team_alias = aliased_target.startswith(f"model_name_{user_api_key_dict.team_id}_")
if is_stale_team_alias and llm_router:
# This is a stale alias from pre-PR deployments.
# Check if current team deployments exist for the public name.
key = (user_api_key_dict.team_id, _model)
if key in llm_router.team_model_to_deployment_indices:
if enable_stale_alias_bypass:
# Team deployments exist; skip stale alias
return
warning_key = f"{user_api_key_dict.team_id}:{_model}:{aliased_target}"
if warning_key not in _STALE_TEAM_ALIAS_WARNING_KEYS:
_STALE_TEAM_ALIAS_WARNING_KEYS[warning_key] = None
while len(_STALE_TEAM_ALIAS_WARNING_KEYS) > _MAX_STALE_ALIAS_WARNING_KEYS:
_STALE_TEAM_ALIAS_WARNING_KEYS.popitem(last=False)
verbose_proxy_logger.warning(
"Stale team model alias detected for model='%s', team_id='%s'. "
"New sibling deployments may be unreachable. "
"Set LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true to enable "
"team-scoped sibling routing.",
_sanitize_for_log(_model),
user_api_key_dict.team_id,
)
# Skip alias rewrite if this model resolves to team-specific deployments
# (team models use team_public_model_name, not model_aliases)
aliased_target = user_api_key_dict.team_model_aliases[_model]
data["model"] = aliased_target
return
# Optional bypass for stale aliases from pre-PR deployments:
# only enabled via feature flag to preserve backwards compatibility.
# Cached at module level to avoid hot-path secret lookups on every request.
global _ENABLE_TEAM_STALE_ALIAS_BYPASS
if _ENABLE_TEAM_STALE_ALIAS_BYPASS is None:
_ENABLE_TEAM_STALE_ALIAS_BYPASS = get_secret_bool("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", False)
enable_stale_alias_bypass = _ENABLE_TEAM_STALE_ALIAS_BYPASS
# Check if the alias points to a team-scoped UUID name
# (format: "model_name_{team_id}_{uuid}")
is_stale_team_alias = aliased_target.startswith(f"model_name_{user_api_key_dict.team_id}_")
if is_stale_team_alias and llm_router:
if aliased_target not in llm_router.model_name_to_deployment_indices:
_warn_stale_team_alias_once(
f"deleted:{user_api_key_dict.team_id}:{_model}:{aliased_target}",
"Team model alias for model='%s', team_id='%s' targets '%s', which has no live "
"deployment. Routing with the requested model name instead; remove the stale "
"entry from the team's model_aliases to silence this warning.",
_sanitize_for_log(_model),
_sanitize_for_log(user_api_key_dict.team_id),
_sanitize_for_log(aliased_target),
)
return
# This is a stale alias from pre-PR deployments.
# Check if current team deployments exist for the public name.
key = (user_api_key_dict.team_id, _model)
if key in llm_router.team_model_to_deployment_indices:
if enable_stale_alias_bypass:
# Team deployments exist; skip stale alias
return
_warn_stale_team_alias_once(
f"{user_api_key_dict.team_id}:{_model}:{aliased_target}",
"Stale team model alias detected for model='%s', team_id='%s'. "
"New sibling deployments may be unreachable. "
"Set LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true to enable "
"team-scoped sibling routing.",
_sanitize_for_log(_model),
_sanitize_for_log(user_api_key_dict.team_id),
)
data["model"] = aliased_target
def _update_model_if_key_alias_exists(

45
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -52,6 +52,7 @@ from litellm.proxy.utils import PrismaClient
from litellm.repositories.model_repository import ModelRepository
from litellm.repositories.table_repositories import ModelTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.router import Router
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
UpdateUsefulLinksRequest,
)
@ -788,6 +789,7 @@ async def _remove_unbacked_team_models(
prisma_client: PrismaClient,
user_api_key_cache: Any,
proxy_logging_obj: Any,
llm_router: Router | None = None,
) -> None:
"""
Strip a deleted team model's public name(s) from team.models and refresh the cache.
@ -795,26 +797,40 @@ async def _remove_unbacked_team_models(
Must be called after the deployment row is deleted: a public name is removed only
when no remaining team deployment still backs it, so a load-balanced replica isn't
revoked while siblings serve it, and concurrent deletes can't leave a ghost.
Legacy team models (created before team_public_model_name existed) store a
``{public_name: "model_name_{team_id}_{uuid}"}`` entry in the team's model_aliases,
so the alias scan runs for every team model; skipping it for internal-shaped names
left stale aliases that rewrote requests to deployments that no longer exist.
A public name that still resolves to a live router deployment (e.g. a gateway-level
model group shared with the team) is kept in team.models, so deleting a per-team
duplicate does not revoke the team's access to the shared deployment.
"""
team_id = model_params.model_info.team_id
if team_id is None:
return
# BYOK models carry an internal `model_name_{team_id}_{uuid}` name that can never
# be a team alias value, so skip the full litellm_modeltable scan for them.
removed_model_aliases: List[Tuple[str, str]] = []
if not model_params.model_name.startswith(f"model_name_{team_id}_"):
removed_model_aliases = await delete_team_model_alias(
public_model_name=model_params.model_name,
prisma_client=prisma_client,
)
names_to_remove = {alias for alias_team_id, alias in removed_model_aliases if alias_team_id == team_id}
if model_params.model_info.team_public_model_name is not None:
names_to_remove.add(model_params.model_info.team_public_model_name)
if names_to_remove:
names_to_remove -= await _get_team_public_model_names(team_id=team_id, prisma_client=prisma_client)
removed_model_aliases = await delete_team_model_alias(
public_model_name=model_params.model_name,
prisma_client=prisma_client,
)
removed_alias_names = {alias for alias_team_id, alias in removed_model_aliases if alias_team_id == team_id}
candidate_names = (
removed_alias_names | {model_params.model_info.team_public_model_name}
if model_params.model_info.team_public_model_name is not None
else removed_alias_names
)
if not candidate_names:
return
team_backed_names = await _get_team_public_model_names(team_id=team_id, prisma_client=prisma_client)
router_served_names = (
frozenset(name for name in candidate_names if name in llm_router.model_name_to_deployment_indices)
if llm_router is not None
else frozenset()
)
names_to_remove = candidate_names - team_backed_names - router_served_names
if not names_to_remove:
return
@ -1120,6 +1136,7 @@ async def delete_model(
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
llm_router=llm_router,
)
## CREATE AUDIT LOG ##

6
tests/proxy_unit_tests/test_proxy_utils.py
View file

@ -2184,7 +2184,8 @@ def test_team_alias_stale_bypass_disabled_by_default(monkeypatch):
pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None
class _MockRouter:
team_model_to_deployment_indices = {("team-1", "gpt-4o"): [0]}
model_name_to_deployment_indices = {"model_name_team-1_legacy-uuid": [0]}
team_model_to_deployment_indices = {("team-1", "gpt-4o"): [1]}
test_data = {"model": "gpt-4o"}
user_api_key_dict = UserAPIKeyAuth(
@ -2209,7 +2210,8 @@ def test_team_alias_stale_bypass_enabled_by_flag(monkeypatch):
pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None
class _MockRouter:
team_model_to_deployment_indices = {("team-1", "gpt-4o"): [0]}
model_name_to_deployment_indices = {"model_name_team-1_legacy-uuid": [0]}
team_model_to_deployment_indices = {("team-1", "gpt-4o"): [1]}
test_data = {"model": "gpt-4o"}
user_api_key_dict = UserAPIKeyAuth(

95
tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
View file

@ -2031,8 +2031,7 @@ class TestDeleteTeamBYOKModelGhost:
mock_refresh.assert_awaited_once()
assert mock_refresh.await_args.kwargs["team_row"] is updated_team_row
# BYOK internal name can't be an alias value -> the alias-table scan is skipped.
mock_prisma.db.litellm_modeltable.find_many.assert_not_awaited()
mock_prisma.db.litellm_modeltable.find_many.assert_awaited()
@pytest.mark.asyncio
async def test_delete_non_internal_team_model_still_scans_aliases(self):
@ -2186,6 +2185,98 @@ class TestDeleteTeamBYOKModelGhost:
mock_prisma.db.litellm_teamtable.update.assert_not_awaited()
mock_refresh.assert_not_awaited()
@pytest.mark.asyncio
async def test_delete_legacy_team_model_scrubs_stale_alias_and_keeps_gateway_access(
self,
):
"""Regression: legacy team models store {public_name: internal model_name} in
the team's model_aliases. delete_model skipped the alias scan for
internal-shaped names, so the stale alias kept rewriting requests for the
public name to a deployment that no longer existed ("no healthy deployments
for model_name_{team_id}_..."). Deleting the deployment must scrub the
alias, and the public name must stay in team.models while a gateway-level
deployment still serves it."""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
delete_model as delete_model_endpoint,
)
team_id = "team-legacy-alias"
model_id = "legacy-alias-model-1"
public_name = "gpt-4"
internal_name = f"model_name_{team_id}_abc-uuid"
db_row = LiteLLM_ProxyModelTable(
model_id=model_id,
model_name=internal_name,
litellm_params={"model": "openai/gpt-4.1-nano"},
model_info={"id": model_id, "team_id": team_id},
created_by="admin",
updated_by="admin",
)
team_row = LiteLLM_TeamTable(
team_id=team_id,
team_alias="legacy-alias-team",
members_with_roles=[Member(user_id="admin", role="admin")],
models=[public_name],
)
alias_row = MagicMock(
id="alias-row-1", model_aliases={public_name: internal_name}
)
alias_row.team = MagicMock()
alias_row.team.team_id = team_id
mock_prisma = MagicMock()
mock_prisma.db = MagicMock()
mock_prisma.db.litellm_proxymodeltable = AsyncMock()
mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(
return_value=db_row
)
mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row)
mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[])
mock_prisma.db.litellm_teamtable = AsyncMock()
mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row)
mock_prisma.db.litellm_teamtable.update = AsyncMock(return_value=team_row)
mock_prisma.db.litellm_modeltable = AsyncMock()
mock_prisma.db.litellm_modeltable.find_many = AsyncMock(
return_value=[alias_row]
)
mock_prisma.db.litellm_modeltable.update = AsyncMock()
mock_router = MagicMock()
mock_router.model_name_to_deployment_indices = {public_name: [0]}
admin_user = UserAPIKeyAuth(
user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
)
_PS = "litellm.proxy.proxy_server"
_MOD = "litellm.proxy.management_endpoints.model_management_endpoints"
with (
patch(f"{_PS}.prisma_client", mock_prisma),
patch(f"{_PS}.store_model_in_db", True),
patch(f"{_PS}.premium_user", True),
patch(f"{_PS}.llm_router", mock_router),
patch(f"{_PS}.proxy_logging_obj", MagicMock()),
patch(f"{_PS}.user_api_key_cache", MagicMock()),
patch(f"{_MOD}._refresh_cached_team", new=AsyncMock()) as mock_refresh,
):
result = await delete_model_endpoint(
model_info=ModelInfoDelete(id=model_id),
user_api_key_dict=admin_user,
)
assert "deleted successfully" in result["message"]
mock_prisma.db.litellm_modeltable.update.assert_awaited_once()
alias_update_kwargs = mock_prisma.db.litellm_modeltable.update.await_args.kwargs
assert alias_update_kwargs["where"] == {"id": "alias-row-1"}
assert json.loads(alias_update_kwargs["data"]["model_aliases"]) == {}
# A gateway-level deployment still serves the public name -> team access stays.
mock_prisma.db.litellm_teamtable.update.assert_not_awaited()
mock_refresh.assert_not_awaited()
class TestDeleteModelTeamAuth:
"""Team auth on the /model/delete path.

61
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -5457,3 +5457,64 @@ def test_get_sanitized_user_information_from_key_drops_callback_config():
# UserAPIKeyAuth is the live auth object; the per-key callbacks are resolved
# from it during pre-call, so it must not be mutated by building the log view
assert "logging" in (user_api_key_dict.metadata or {})
def test_team_alias_targeting_deleted_team_deployment_keeps_requested_model(monkeypatch):
"""
Regression: a team's model_aliases can point at the internal routing key
(model_name_{team_id}_{uuid}) of a team deployment that was since deleted,
e.g. after an admin replaces per-team duplicates with one gateway-level
model. Rewriting to the dead internal name made every request fail with
"no healthy deployments for model_name_..." even though the requested
public name resolves at the gateway level. The rewrite must be skipped
when the alias target has no live deployment.
"""
import litellm.proxy.litellm_pre_call_utils as pre_call_utils
from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists
monkeypatch.delenv("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", raising=False)
pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None
class _MockRouter:
model_name_to_deployment_indices = {"gpt-4": [0]}
team_model_to_deployment_indices = {}
test_data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test_key",
team_id="team-1",
team_model_aliases={"gpt-4": "model_name_team-1_dead-uuid"},
)
with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()):
_update_model_if_team_alias_exists(
data=test_data, user_api_key_dict=user_api_key_dict
)
assert test_data.get("model") == "gpt-4"
def test_team_alias_targeting_live_team_deployment_still_rewrites(monkeypatch):
import litellm.proxy.litellm_pre_call_utils as pre_call_utils
from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists
monkeypatch.delenv("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", raising=False)
pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None
class _MockRouter:
model_name_to_deployment_indices = {"model_name_team-1_live-uuid": [0]}
team_model_to_deployment_indices = {}
test_data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test_key",
team_id="team-1",
team_model_aliases={"gpt-4": "model_name_team-1_live-uuid"},
)
with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()):
_update_model_if_team_alias_exists(
data=test_data, user_api_key_dict=user_api_key_dict
)
assert test_data.get("model") == "model_name_team-1_live-uuid"