scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

Merge pull request #34993 from BerriAI/claude/auto-til-blocked-cwalrj

Browse source 
fix(proxy): skip team model aliases that point at deleted deployments
This commit is contained in:
Mateo Wang 2026-07-28 20:09:20 -07:00 • committed by GitHub
commit 2bb297efa0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 360 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
litellm/proxy/litellm_pre_call_utils.py
View file

@ -1829,6 +1829,15 @@ async def add_litellm_data_to_request(
return data
def _warn_stale_team_alias_once(warning_key: str, message: str, *args: str) -> None:
if warning_key in _STALE_TEAM_ALIAS_WARNING_KEYS:
return
_STALE_TEAM_ALIAS_WARNING_KEYS[warning_key] = None
while len(_STALE_TEAM_ALIAS_WARNING_KEYS) > _MAX_STALE_ALIAS_WARNING_KEYS:
_STALE_TEAM_ALIAS_WARNING_KEYS.popitem(last=False)
verbose_proxy_logger.warning(message, *args)
def _update_model_if_team_alias_exists(
data: dict,
user_api_key_dict: UserAPIKeyAuth,
@ -1848,49 +1857,63 @@ def _update_model_if_team_alias_exists(
Note: model_aliases for team models are deprecated. This function only applies
to legacy non-team-scoped aliases. Team-scoped deployments use team_public_model_name
and are resolved via map_team_model in route_llm_request.
An alias that targets a team-scoped internal name (``model_name_{team_id}_{uuid}``)
with no live deployment behind it is never applied: the deployment was deleted, so
the rewrite could only fail with an error naming a model the caller never sent.
Keeping the requested model name lets it resolve against the deployments that still
exist (e.g. a gateway-level model group shared with the team).
"""
_model = data.get("model")
if _model and user_api_key_dict.team_model_aliases and _model in user_api_key_dict.team_model_aliases:
from litellm.proxy.proxy_server import llm_router
if not _model or not user_api_key_dict.team_model_aliases or _model not in user_api_key_dict.team_model_aliases:
return
# Skip alias rewrite if this model resolves to team-specific deployments
# (team models use team_public_model_name, not model_aliases)
aliased_target = user_api_key_dict.team_model_aliases[_model]
from litellm.proxy.proxy_server import llm_router
# Optional bypass for stale aliases from pre-PR deployments:
# only enabled via feature flag to preserve backwards compatibility.
# Cached at module level to avoid hot-path secret lookups on every request.
global _ENABLE_TEAM_STALE_ALIAS_BYPASS
if _ENABLE_TEAM_STALE_ALIAS_BYPASS is None:
_ENABLE_TEAM_STALE_ALIAS_BYPASS = get_secret_bool("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", False)
enable_stale_alias_bypass = _ENABLE_TEAM_STALE_ALIAS_BYPASS
# Check if the alias points to a team-scoped UUID name
# (format: "model_name_{team_id}_{uuid}")
is_stale_team_alias = aliased_target.startswith(f"model_name_{user_api_key_dict.team_id}_")
if is_stale_team_alias and llm_router:
# This is a stale alias from pre-PR deployments.
# Check if current team deployments exist for the public name.
key = (user_api_key_dict.team_id, _model)
if key in llm_router.team_model_to_deployment_indices:
if enable_stale_alias_bypass:
# Team deployments exist; skip stale alias
return
warning_key = f"{user_api_key_dict.team_id}:{_model}:{aliased_target}"
if warning_key not in _STALE_TEAM_ALIAS_WARNING_KEYS:
_STALE_TEAM_ALIAS_WARNING_KEYS[warning_key] = None
while len(_STALE_TEAM_ALIAS_WARNING_KEYS) > _MAX_STALE_ALIAS_WARNING_KEYS:
_STALE_TEAM_ALIAS_WARNING_KEYS.popitem(last=False)
verbose_proxy_logger.warning(
"Stale team model alias detected for model='%s', team_id='%s'. "
"New sibling deployments may be unreachable. "
"Set LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true to enable "
"team-scoped sibling routing.",
_sanitize_for_log(_model),
user_api_key_dict.team_id,
)
# Skip alias rewrite if this model resolves to team-specific deployments
# (team models use team_public_model_name, not model_aliases)
aliased_target = user_api_key_dict.team_model_aliases[_model]
data["model"] = aliased_target
return
# Optional bypass for stale aliases from pre-PR deployments:
# only enabled via feature flag to preserve backwards compatibility.
# Cached at module level to avoid hot-path secret lookups on every request.
global _ENABLE_TEAM_STALE_ALIAS_BYPASS
if _ENABLE_TEAM_STALE_ALIAS_BYPASS is None:
_ENABLE_TEAM_STALE_ALIAS_BYPASS = get_secret_bool("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", False)
enable_stale_alias_bypass = _ENABLE_TEAM_STALE_ALIAS_BYPASS
# Check if the alias points to a team-scoped UUID name
# (format: "model_name_{team_id}_{uuid}")
is_stale_team_alias = aliased_target.startswith(f"model_name_{user_api_key_dict.team_id}_")
if is_stale_team_alias and llm_router:
if aliased_target not in llm_router.model_name_to_deployment_indices:
_warn_stale_team_alias_once(
f"deleted:{user_api_key_dict.team_id}:{_model}:{aliased_target}",
"Team model alias for model='%s', team_id='%s' targets '%s', which has no live "
"deployment. Routing with the requested model name instead; remove the stale "
"entry from the team's model_aliases to silence this warning.",
_sanitize_for_log(_model),
_sanitize_for_log(user_api_key_dict.team_id),
_sanitize_for_log(aliased_target),
)
return
# This is a stale alias from pre-PR deployments.
# Check if current team deployments exist for the public name.
key = (user_api_key_dict.team_id, _model)
if key in llm_router.team_model_to_deployment_indices:
if enable_stale_alias_bypass:
# Team deployments exist; skip stale alias
return
_warn_stale_team_alias_once(
f"{user_api_key_dict.team_id}:{_model}:{aliased_target}",
"Stale team model alias detected for model='%s', team_id='%s'. "
"New sibling deployments may be unreachable. "
"Set LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true to enable "
"team-scoped sibling routing.",
_sanitize_for_log(_model),
_sanitize_for_log(user_api_key_dict.team_id),
)
data["model"] = aliased_target
def _update_model_if_key_alias_exists(

49
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -52,6 +52,7 @@ from litellm.proxy.utils import PrismaClient
from litellm.repositories.model_repository import ModelRepository
from litellm.repositories.table_repositories import ModelTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.router import Router
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
UpdateUsefulLinksRequest,
)
@ -788,6 +789,7 @@ async def _remove_unbacked_team_models(
prisma_client: PrismaClient,
user_api_key_cache: Any,
proxy_logging_obj: Any,
llm_router: Router | None = None,
) -> None:
"""
Strip a deleted team model's public name(s) from team.models and refresh the cache.
@ -795,26 +797,50 @@ async def _remove_unbacked_team_models(
Must be called after the deployment row is deleted: a public name is removed only
when no remaining team deployment still backs it, so a load-balanced replica isn't
revoked while siblings serve it, and concurrent deletes can't leave a ghost.
Legacy team models (created before team_public_model_name existed) store a
``{public_name: "model_name_{team_id}_{uuid}"}`` entry in the team's model_aliases,
so the alias scan runs for every team model; skipping it for internal-shaped names
left stale aliases that rewrote requests to deployments that no longer exist.
Aliases are scrubbed only when the deleted deployment's name no longer resolves in
the router, so deleting one replica of a load-balanced group never breaks aliases
that still route to the surviving replicas (in any team).
A public name that still resolves to a live router deployment (e.g. a gateway-level
model group shared with the team) is kept in team.models, so deleting a per-team
duplicate does not revoke the team's access to the shared deployment.
"""
team_id = model_params.model_info.team_id
if team_id is None:
return
# BYOK models carry an internal `model_name_{team_id}_{uuid}` name that can never
# be a team alias value, so skip the full litellm_modeltable scan for them.
removed_model_aliases: List[Tuple[str, str]] = []
if not model_params.model_name.startswith(f"model_name_{team_id}_"):
removed_model_aliases = await delete_team_model_alias(
deleted_name_still_served = (
llm_router is not None and model_params.model_name in llm_router.model_name_to_deployment_indices
)
removed_model_aliases: List[Tuple[str, str]] = (
[]
if deleted_name_still_served
else await delete_team_model_alias(
public_model_name=model_params.model_name,
prisma_client=prisma_client,
)
names_to_remove = {alias for alias_team_id, alias in removed_model_aliases if alias_team_id == team_id}
if model_params.model_info.team_public_model_name is not None:
names_to_remove.add(model_params.model_info.team_public_model_name)
if names_to_remove:
names_to_remove -= await _get_team_public_model_names(team_id=team_id, prisma_client=prisma_client)
)
removed_alias_names = {alias for alias_team_id, alias in removed_model_aliases if alias_team_id == team_id}
candidate_names = (
removed_alias_names | {model_params.model_info.team_public_model_name}
if model_params.model_info.team_public_model_name is not None
else removed_alias_names
)
if not candidate_names:
return
team_backed_names = await _get_team_public_model_names(team_id=team_id, prisma_client=prisma_client)
router_served_names = (
frozenset(name for name in candidate_names if name in llm_router.model_name_to_deployment_indices)
if llm_router is not None
else frozenset()
)
names_to_remove = candidate_names - team_backed_names - router_served_names
if not names_to_remove:
return
@ -1120,6 +1146,7 @@ async def delete_model(
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
llm_router=llm_router,
)
## CREATE AUDIT LOG ##

6
tests/proxy_unit_tests/test_proxy_utils.py
View file

@ -2184,7 +2184,8 @@ def test_team_alias_stale_bypass_disabled_by_default(monkeypatch):
pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None
class _MockRouter:
team_model_to_deployment_indices = {("team-1", "gpt-4o"): [0]}
model_name_to_deployment_indices = {"model_name_team-1_legacy-uuid": [0]}
team_model_to_deployment_indices = {("team-1", "gpt-4o"): [1]}
test_data = {"model": "gpt-4o"}
user_api_key_dict = UserAPIKeyAuth(
@ -2209,7 +2210,8 @@ def test_team_alias_stale_bypass_enabled_by_flag(monkeypatch):
pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None
class _MockRouter:
team_model_to_deployment_indices = {("team-1", "gpt-4o"): [0]}
model_name_to_deployment_indices = {"model_name_team-1_legacy-uuid": [0]}
team_model_to_deployment_indices = {("team-1", "gpt-4o"): [1]}
test_data = {"model": "gpt-4o"}
user_api_key_dict = UserAPIKeyAuth(

168
tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
View file

@ -2031,8 +2031,7 @@ class TestDeleteTeamBYOKModelGhost:
mock_refresh.assert_awaited_once()
assert mock_refresh.await_args.kwargs["team_row"] is updated_team_row
# BYOK internal name can't be an alias value -> the alias-table scan is skipped.
mock_prisma.db.litellm_modeltable.find_many.assert_not_awaited()
mock_prisma.db.litellm_modeltable.find_many.assert_awaited()
@pytest.mark.asyncio
async def test_delete_non_internal_team_model_still_scans_aliases(self):
@ -2186,6 +2185,171 @@ class TestDeleteTeamBYOKModelGhost:
mock_prisma.db.litellm_teamtable.update.assert_not_awaited()
mock_refresh.assert_not_awaited()
@pytest.mark.asyncio
async def test_delete_legacy_team_model_scrubs_stale_alias_and_keeps_gateway_access(
self,
):
"""Regression: legacy team models store {public_name: internal model_name} in
the team's model_aliases. delete_model skipped the alias scan for
internal-shaped names, so the stale alias kept rewriting requests for the
public name to a deployment that no longer existed ("no healthy deployments
for model_name_{team_id}_..."). Deleting the deployment must scrub the
alias, and the public name must stay in team.models while a gateway-level
deployment still serves it."""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
delete_model as delete_model_endpoint,
)
team_id = "team-legacy-alias"
model_id = "legacy-alias-model-1"
public_name = "gpt-4"
internal_name = f"model_name_{team_id}_abc-uuid"
db_row = LiteLLM_ProxyModelTable(
model_id=model_id,
model_name=internal_name,
litellm_params={"model": "openai/gpt-4.1-nano"},
model_info={"id": model_id, "team_id": team_id},
created_by="admin",
updated_by="admin",
)
team_row = LiteLLM_TeamTable(
team_id=team_id,
team_alias="legacy-alias-team",
members_with_roles=[Member(user_id="admin", role="admin")],
models=[public_name],
)
alias_row = MagicMock(
id="alias-row-1", model_aliases={public_name: internal_name}
)
alias_row.team = MagicMock()
alias_row.team.team_id = team_id
mock_prisma = MagicMock()
mock_prisma.db = MagicMock()
mock_prisma.db.litellm_proxymodeltable = AsyncMock()
mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(
return_value=db_row
)
mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row)
mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[])
mock_prisma.db.litellm_teamtable = AsyncMock()
mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row)
mock_prisma.db.litellm_teamtable.update = AsyncMock(return_value=team_row)
mock_prisma.db.litellm_modeltable = AsyncMock()
mock_prisma.db.litellm_modeltable.find_many = AsyncMock(
return_value=[alias_row]
)
mock_prisma.db.litellm_modeltable.update = AsyncMock()
mock_router = MagicMock()
mock_router.model_name_to_deployment_indices = {public_name: [0]}
admin_user = UserAPIKeyAuth(
user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
)
_PS = "litellm.proxy.proxy_server"
_MOD = "litellm.proxy.management_endpoints.model_management_endpoints"
with (
patch(f"{_PS}.prisma_client", mock_prisma),
patch(f"{_PS}.store_model_in_db", True),
patch(f"{_PS}.premium_user", True),
patch(f"{_PS}.llm_router", mock_router),
patch(f"{_PS}.proxy_logging_obj", MagicMock()),
patch(f"{_PS}.user_api_key_cache", MagicMock()),
patch(f"{_MOD}._refresh_cached_team", new=AsyncMock()) as mock_refresh,
):
result = await delete_model_endpoint(
model_info=ModelInfoDelete(id=model_id),
user_api_key_dict=admin_user,
)
assert "deleted successfully" in result["message"]
mock_prisma.db.litellm_modeltable.update.assert_awaited_once()
alias_update_kwargs = mock_prisma.db.litellm_modeltable.update.await_args.kwargs
assert alias_update_kwargs["where"] == {"id": "alias-row-1"}
assert json.loads(alias_update_kwargs["data"]["model_aliases"]) == {}
# A gateway-level deployment still serves the public name -> team access stays.
mock_prisma.db.litellm_teamtable.update.assert_not_awaited()
mock_refresh.assert_not_awaited()
@pytest.mark.asyncio
async def test_delete_replica_keeps_alias_while_surviving_replica_serves_it(self):
"""Deleting one replica of a load-balanced legacy team model (several
deployment rows sharing one internal model_name) must not scrub the team
alias: the surviving replicas still serve the aliased name, so removing
the alias would break routing that works."""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
delete_model as delete_model_endpoint,
)
team_id = "team-lb-legacy"
model_id = "lb-replica-1"
internal_name = f"model_name_{team_id}_shared-uuid"
db_row = LiteLLM_ProxyModelTable(
model_id=model_id,
model_name=internal_name,
litellm_params={"model": "openai/gpt-4.1-nano"},
model_info={"id": model_id, "team_id": team_id},
created_by="admin",
updated_by="admin",
)
team_row = LiteLLM_TeamTable(
team_id=team_id,
team_alias="lb-legacy-team",
members_with_roles=[Member(user_id="admin", role="admin")],
models=["gpt-4"],
)
mock_prisma = MagicMock()
mock_prisma.db = MagicMock()
mock_prisma.db.litellm_proxymodeltable = AsyncMock()
mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(
return_value=db_row
)
mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row)
mock_prisma.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[])
mock_prisma.db.litellm_teamtable = AsyncMock()
mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row)
mock_prisma.db.litellm_teamtable.update = AsyncMock(return_value=team_row)
mock_prisma.db.litellm_modeltable = AsyncMock()
mock_prisma.db.litellm_modeltable.find_many = AsyncMock(return_value=[])
mock_prisma.db.litellm_modeltable.update = AsyncMock()
mock_router = MagicMock()
mock_router.model_name_to_deployment_indices = {internal_name: [0]}
admin_user = UserAPIKeyAuth(
user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
)
_PS = "litellm.proxy.proxy_server"
_MOD = "litellm.proxy.management_endpoints.model_management_endpoints"
with (
patch(f"{_PS}.prisma_client", mock_prisma),
patch(f"{_PS}.store_model_in_db", True),
patch(f"{_PS}.premium_user", True),
patch(f"{_PS}.llm_router", mock_router),
patch(f"{_PS}.proxy_logging_obj", MagicMock()),
patch(f"{_PS}.user_api_key_cache", MagicMock()),
patch(f"{_MOD}._refresh_cached_team", new=AsyncMock()),
):
result = await delete_model_endpoint(
model_info=ModelInfoDelete(id=model_id),
user_api_key_dict=admin_user,
)
assert "deleted successfully" in result["message"]
mock_prisma.db.litellm_modeltable.find_many.assert_not_awaited()
mock_prisma.db.litellm_modeltable.update.assert_not_awaited()
mock_prisma.db.litellm_teamtable.update.assert_not_awaited()
class TestDeleteModelTeamAuth:
"""Team auth on the /model/delete path.

91
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -5457,3 +5457,94 @@ def test_get_sanitized_user_information_from_key_drops_callback_config():
# UserAPIKeyAuth is the live auth object; the per-key callbacks are resolved
# from it during pre-call, so it must not be mutated by building the log view
assert "logging" in (user_api_key_dict.metadata or {})
def test_team_alias_targeting_deleted_team_deployment_keeps_requested_model(monkeypatch):
"""
Regression: a team's model_aliases can point at the internal routing key
(model_name_{team_id}_{uuid}) of a team deployment that was since deleted,
e.g. after an admin replaces per-team duplicates with one gateway-level
model. Rewriting to the dead internal name made every request fail with
"no healthy deployments for model_name_..." even though the requested
public name resolves at the gateway level. The rewrite must be skipped
when the alias target has no live deployment.
"""
import litellm.proxy.litellm_pre_call_utils as pre_call_utils
from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists
monkeypatch.delenv("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", raising=False)
pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None
class _MockRouter:
model_name_to_deployment_indices = {"gpt-4": [0]}
team_model_to_deployment_indices = {}
test_data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test_key",
team_id="team-1",
team_model_aliases={"gpt-4": "model_name_team-1_dead-uuid"},
)
with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()):
_update_model_if_team_alias_exists(
data=test_data, user_api_key_dict=user_api_key_dict
)
assert test_data.get("model") == "gpt-4"
def test_team_alias_targeting_live_team_deployment_still_rewrites(monkeypatch):
import litellm.proxy.litellm_pre_call_utils as pre_call_utils
from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists
monkeypatch.delenv("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", raising=False)
pre_call_utils._ENABLE_TEAM_STALE_ALIAS_BYPASS = None
class _MockRouter:
model_name_to_deployment_indices = {"model_name_team-1_live-uuid": [0]}
team_model_to_deployment_indices = {}
test_data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test_key",
team_id="team-1",
team_model_aliases={"gpt-4": "model_name_team-1_live-uuid"},
)
with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()):
_update_model_if_team_alias_exists(
data=test_data, user_api_key_dict=user_api_key_dict
)
assert test_data.get("model") == "model_name_team-1_live-uuid"
def test_warn_stale_team_alias_once_logs_once_per_key(monkeypatch):
from collections import OrderedDict
import litellm.proxy.litellm_pre_call_utils as pre_call_utils
monkeypatch.setattr(pre_call_utils, "_STALE_TEAM_ALIAS_WARNING_KEYS", OrderedDict())
with patch.object(pre_call_utils.verbose_proxy_logger, "warning") as mock_warning:
pre_call_utils._warn_stale_team_alias_once("team-1:gpt-4", "stale alias %s", "gpt-4")
pre_call_utils._warn_stale_team_alias_once("team-1:gpt-4", "stale alias %s", "gpt-4")
assert mock_warning.call_count == 1
def test_warn_stale_team_alias_once_evicts_oldest_key_beyond_cap(monkeypatch):
from collections import OrderedDict
import litellm.proxy.litellm_pre_call_utils as pre_call_utils
monkeypatch.setattr(pre_call_utils, "_STALE_TEAM_ALIAS_WARNING_KEYS", OrderedDict())
monkeypatch.setattr(pre_call_utils, "_MAX_STALE_ALIAS_WARNING_KEYS", 2)
with patch.object(pre_call_utils.verbose_proxy_logger, "warning"):
pre_call_utils._warn_stale_team_alias_once("key-1", "stale alias")
pre_call_utils._warn_stale_team_alias_once("key-2", "stale alias")
pre_call_utils._warn_stale_team_alias_once("key-3", "stale alias")
assert list(pre_call_utils._STALE_TEAM_ALIAS_WARNING_KEYS) == ["key-2", "key-3"]