From a2fd8636e0641d72a66ab1a2d9eaa95a78199959 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 15:31:43 +0530 Subject: [PATCH 01/22] fix(team-routing): use deterministic team model group names Use a deterministic internal model_name for team-scoped deployments so sibling deployments with the same public model share a routing group. This makes team alias writes idempotent and preserves multi-deployment failover/load balancing behavior. Made-with: Cursor --- .../model_management_endpoints.py | 26 ++- .../test_model_management_endpoints.py | 199 ++++++++++++++---- 2 files changed, 172 insertions(+), 53 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 44d41097833..39383d4ee20 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -13,13 +13,13 @@ model/{model_id}/update - PATCH endpoint for model update. import asyncio import datetime import json -from litellm._uuid import uuid from typing import Dict, List, Literal, Optional, Tuple, Union, cast from fastapi import APIRouter, Depends, HTTPException, Request, status from pydantic import BaseModel, ConfigDict, Field from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.proxy._types import ( CommonProxyErrors, @@ -322,9 +322,13 @@ async def _add_team_model_to_db( """ If 'team_id' is provided, - - generate a unique 'model_name' for the model (e.g. 'model_name_{team_id}_{uuid}) - - store the model in the db with the unique 'model_name' - - store a team model alias mapping {"model_name": "model_name_{team_id}_{uuid}"} + - generate a deterministic 'model_name' for the model (e.g. 'model_name_{team_id}_{public_name}') + - store the model in the db with this shared group name + - store a team model alias mapping {"public_name": "model_name_{team_id}_{public_name}"} + + Using a deterministic name (not UUID) ensures sibling deployments for the + same public model share a model_name, so the router treats them as a single + candidate pool for load balancing and failover. """ _team_id = model_params.model_info.team_id if _team_id is None: @@ -333,9 +337,9 @@ async def _add_team_model_to_db( if original_model_name: model_params.model_info.team_public_model_name = original_model_name - unique_model_name = f"model_name_{_team_id}_{uuid.uuid4()}" + group_model_name = f"model_name_{_team_id}_{original_model_name}" - model_params.model_name = unique_model_name + model_params.model_name = group_model_name ## CREATE MODEL IN DB ## model_response = await _add_model_to_db( @@ -348,7 +352,7 @@ async def _add_team_model_to_db( await update_team( data=UpdateTeamRequest( team_id=_team_id, - model_aliases={original_model_name: unique_model_name}, + model_aliases={original_model_name: group_model_name}, ), user_api_key_dict=user_api_key_dict, http_request=Request(scope={"type": "http"}), @@ -453,14 +457,14 @@ async def _setup_new_team_model_assignment( patch_data: updateDeployment, user_api_key_dict: UserAPIKeyAuth, ) -> None: - """Set up a new team model with unique name, alias, and team membership.""" - unique_model_name = f"model_name_{team_id}_{uuid.uuid4()}" - patch_data.model_name = unique_model_name + """Set up a new team model with deterministic name, alias, and team membership.""" + group_model_name = f"model_name_{team_id}_{public_model_name}" + patch_data.model_name = group_model_name await update_team( data=UpdateTeamRequest( team_id=team_id, - model_aliases={public_model_name: unique_model_name}, + model_aliases={public_model_name: group_model_name}, ), user_api_key_dict=user_api_key_dict, http_request=Request(scope={"type": "http"}), diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index f3c89003105..7f64a3de935 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -1,13 +1,14 @@ import json import os import sys -from litellm._uuid import uuid from typing import Dict, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient +from litellm._uuid import uuid + sys.path.insert( 0, os.path.abspath("../../../..") ) # Adds the parent directory to the system path @@ -399,7 +400,9 @@ class TestClearCache: """ Test that clear_cache clears DB models and preserves config models. """ - from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache + from litellm.proxy.management_endpoints.model_management_endpoints import ( + clear_cache, + ) # Create mock router with mixed DB and config models mock_router = MagicMock() @@ -407,18 +410,18 @@ class TestClearCache: { "model_name": "gpt-4", "model_info": {"id": "db-model-1", "db_model": True}, - "litellm_params": {"model": "gpt-4"} + "litellm_params": {"model": "gpt-4"}, }, { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-3.5-turbo", "model_info": {"id": "config-model-1", "db_model": False}, - "litellm_params": {"model": "gpt-3.5-turbo"} + "litellm_params": {"model": "gpt-3.5-turbo"}, }, { "model_name": "claude-3", "model_info": {"id": "db-model-2", "db_model": True}, - "litellm_params": {"model": "claude-3"} - } + "litellm_params": {"model": "claude-3"}, + }, ] mock_router.delete_deployment = MagicMock(return_value=True) mock_router.auto_routers = MagicMock() @@ -466,8 +469,8 @@ class TestUpdatePublicModelGroups: """ import litellm from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_public_model_groups, UpdatePublicModelGroupsRequest, + update_public_model_groups, ) old_db_models = ["db-model-1", "db-model-2"] @@ -525,7 +528,10 @@ class TestUpdatePublicModelGroups: ) old_links = {"Old Doc": "https://old.example.com"} - new_links = {"New Doc": "https://new.example.com", "API Ref": "https://api.example.com"} + new_links = { + "New Doc": "https://new.example.com", + "API Ref": "https://api.example.com", + } async def mock_get_config(*args, **kwargs): litellm.public_model_groups_links = old_links @@ -558,6 +564,100 @@ class TestUpdatePublicModelGroups: litellm.public_model_groups_links = original_value +class TestTeamModelAliasSiblingOverwrite: + """ + Verify that two sibling team deployments for the same public model name + produce the same deterministic internal model_name, so the alias write + is idempotent and the router groups both deployments together. + """ + + @pytest.mark.asyncio + async def test_sibling_team_models_share_deterministic_name(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _add_team_model_to_db, + ) + from litellm.types.router import ModelInfo + + team_id = "team_alias_overwrite" + public_name = "gpt-4.1-mini" + + captured_alias_calls = [] + + async def mock_update_team(data, user_api_key_dict, http_request): + if data.model_aliases: + captured_alias_calls.append(dict(data.model_aliases)) + + async def mock_add_model_to_db(model_params, user_api_key_dict, prisma_client): + return MagicMock(model_id=str(uuid.uuid4())) + + async def mock_team_model_add(data, http_request, user_api_key_dict): + pass + + user = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + prisma_client = MockPrismaClient(team_exists=True) + + deployment_1 = Deployment( + model_name=public_name, + litellm_params=LiteLLM_Params( + model="azure/gpt-4o-mini", + api_key="key-1", + api_base="https://eastus.example.openai.azure.com", + ), + model_info=ModelInfo(team_id=team_id), + ) + deployment_2 = Deployment( + model_name=public_name, + litellm_params=LiteLLM_Params( + model="azure/gpt-4o-mini", + api_key="key-2", + api_base="https://westus.example.openai.azure.com", + ), + model_info=ModelInfo(team_id=team_id), + ) + + with patch( + "litellm.proxy.management_endpoints.model_management_endpoints.update_team", + side_effect=mock_update_team, + ), patch( + "litellm.proxy.management_endpoints.model_management_endpoints._add_model_to_db", + side_effect=mock_add_model_to_db, + ), patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_add", + side_effect=mock_team_model_add, + ): + await _add_team_model_to_db( + model_params=deployment_1, + user_api_key_dict=user, + prisma_client=prisma_client, + ) + await _add_team_model_to_db( + model_params=deployment_2, + user_api_key_dict=user, + prisma_client=prisma_client, + ) + + assert len(captured_alias_calls) == 2 + + internal_name_1 = captured_alias_calls[0][public_name] + internal_name_2 = captured_alias_calls[1][public_name] + + expected_group_name = f"model_name_{team_id}_{public_name}" + + # Both sibling deployments get the same deterministic group name + assert internal_name_1 == expected_group_name + assert internal_name_2 == expected_group_name + assert internal_name_1 == internal_name_2, ( + "Sibling deployments must share the same model_name so the " + "router treats them as a single candidate pool" + ) + + # The second alias write is idempotent β€” same key, same value + final_aliases = {} + for alias_call in captured_alias_calls: + final_aliases.update(alias_call) + assert final_aliases == {public_name: expected_group_name} + + class TestTeamModelUpdate: """Test team model update handles team_id consistently with model creation""" @@ -657,27 +757,37 @@ class TestModelInfoEndpoint: user_id="test_user", api_key="test_key", models=["gpt-4", "claude-3"], - team_models=["gpt-3.5-turbo"] + team_models=["gpt-3.5-turbo"], ) - with patch("litellm.proxy.proxy_server.llm_router") as mock_router, \ - patch("litellm.proxy.proxy_server.get_key_models") as mock_get_key_models, \ - patch("litellm.proxy.proxy_server.get_team_models") as mock_get_team_models, \ - patch("litellm.proxy.proxy_server.get_complete_model_list") as mock_get_complete_models, \ - patch("litellm.get_llm_provider") as mock_get_provider: - + with patch("litellm.proxy.proxy_server.llm_router") as mock_router, patch( + "litellm.proxy.proxy_server.get_key_models" + ) as mock_get_key_models, patch( + "litellm.proxy.proxy_server.get_team_models" + ) as mock_get_team_models, patch( + "litellm.proxy.proxy_server.get_complete_model_list" + ) as mock_get_complete_models, patch( + "litellm.get_llm_provider" + ) as mock_get_provider: # Setup mocks - mock_router.get_model_names.return_value = ["gpt-4", "claude-3", "gpt-3.5-turbo"] + mock_router.get_model_names.return_value = [ + "gpt-4", + "claude-3", + "gpt-3.5-turbo", + ] mock_router.get_model_access_groups.return_value = {} mock_get_key_models.return_value = ["gpt-4", "claude-3"] mock_get_team_models.return_value = ["gpt-3.5-turbo"] - mock_get_complete_models.return_value = ["gpt-4", "claude-3", "gpt-3.5-turbo"] + mock_get_complete_models.return_value = [ + "gpt-4", + "claude-3", + "gpt-3.5-turbo", + ] mock_get_provider.return_value = (None, "openai", None, None) # Test accessible model result = await model_info( - model_id="gpt-4", - user_api_key_dict=user_api_key_dict + model_id="gpt-4", user_api_key_dict=user_api_key_dict ) assert result["id"] == "gpt-4" @@ -688,22 +798,25 @@ class TestModelInfoEndpoint: @pytest.mark.asyncio async def test_model_info_inaccessible_model_returns_404(self): """Test model_info returns 404 for inaccessible models""" - from litellm.proxy.proxy_server import model_info from fastapi import HTTPException + from litellm.proxy.proxy_server import model_info + # Mock user with limited access user_api_key_dict = UserAPIKeyAuth( user_id="test_user", api_key="test_key", models=["gpt-4"], # Only has access to gpt-4 - team_models=[] + team_models=[], ) - with patch("litellm.proxy.proxy_server.llm_router") as mock_router, \ - patch("litellm.proxy.proxy_server.get_key_models") as mock_get_key_models, \ - patch("litellm.proxy.proxy_server.get_team_models") as mock_get_team_models, \ - patch("litellm.proxy.proxy_server.get_complete_model_list") as mock_get_complete_models: - + with patch("litellm.proxy.proxy_server.llm_router") as mock_router, patch( + "litellm.proxy.proxy_server.get_key_models" + ) as mock_get_key_models, patch( + "litellm.proxy.proxy_server.get_team_models" + ) as mock_get_team_models, patch( + "litellm.proxy.proxy_server.get_complete_model_list" + ) as mock_get_complete_models: # Setup mocks - user only has access to gpt-4 mock_router.get_model_names.return_value = ["gpt-4", "claude-3"] mock_router.get_model_access_groups.return_value = {} @@ -715,32 +828,35 @@ class TestModelInfoEndpoint: with pytest.raises(HTTPException) as exc_info: await model_info( model_id="claude-3", # Not in user's accessible models - user_api_key_dict=user_api_key_dict + user_api_key_dict=user_api_key_dict, ) - + assert exc_info.value.status_code == 404 assert "does not exist or is not accessible" in exc_info.value.detail - @pytest.mark.asyncio + @pytest.mark.asyncio async def test_model_info_team_model_access(self): """Test model_info works with team model access""" from litellm.proxy.proxy_server import model_info - + # Mock user with team access user_api_key_dict = UserAPIKeyAuth( user_id="test_user", - api_key="test_key", + api_key="test_key", team_id="test_team", models=[], # No direct key models - team_models=["team-model-1"] + team_models=["team-model-1"], ) - with patch("litellm.proxy.proxy_server.llm_router") as mock_router, \ - patch("litellm.proxy.proxy_server.get_key_models") as mock_get_key_models, \ - patch("litellm.proxy.proxy_server.get_team_models") as mock_get_team_models, \ - patch("litellm.proxy.proxy_server.get_complete_model_list") as mock_get_complete_models, \ - patch("litellm.get_llm_provider") as mock_get_provider: - + with patch("litellm.proxy.proxy_server.llm_router") as mock_router, patch( + "litellm.proxy.proxy_server.get_key_models" + ) as mock_get_key_models, patch( + "litellm.proxy.proxy_server.get_team_models" + ) as mock_get_team_models, patch( + "litellm.proxy.proxy_server.get_complete_model_list" + ) as mock_get_complete_models, patch( + "litellm.get_llm_provider" + ) as mock_get_provider: # Setup mocks mock_router.get_model_names.return_value = ["team-model-1"] mock_router.get_model_access_groups.return_value = {} @@ -751,10 +867,9 @@ class TestModelInfoEndpoint: # Test team model access result = await model_info( - model_id="team-model-1", - user_api_key_dict=user_api_key_dict + model_id="team-model-1", user_api_key_dict=user_api_key_dict ) assert result["id"] == "team-model-1" - assert result["object"] == "model" + assert result["object"] == "model" assert result["owned_by"] == "custom" From 35b8b8af07db3586de21e974ad147a9cbfe827df Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 16:12:27 +0530 Subject: [PATCH 02/22] fix(team-routing): keep team model routing on public names Remove team model_alias rewrites and resolve team deployments by team_public_model_name with team_id so sibling deployments stay in the routing candidate pool, with explicit logs showing candidate selection before load balancing. Made-with: Cursor --- .../model_management_endpoints.py | 50 ++---- litellm/router.py | 71 +++++++- .../test_model_management_endpoints.py | 160 ++++++++++-------- 3 files changed, 171 insertions(+), 110 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 39383d4ee20..694fa2b1d55 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -322,13 +322,9 @@ async def _add_team_model_to_db( """ If 'team_id' is provided, - - generate a deterministic 'model_name' for the model (e.g. 'model_name_{team_id}_{public_name}') - - store the model in the db with this shared group name - - store a team model alias mapping {"public_name": "model_name_{team_id}_{public_name}"} - - Using a deterministic name (not UUID) ensures sibling deployments for the - same public model share a model_name, so the router treats them as a single - candidate pool for load balancing and failover. + - generate a unique 'model_name' for the model (e.g. 'model_name_{team_id}_{uuid}) + - store the model in the db with the unique 'model_name' + - add the public model name to the team's allowed models list """ _team_id = model_params.model_info.team_id if _team_id is None: @@ -337,9 +333,9 @@ async def _add_team_model_to_db( if original_model_name: model_params.model_info.team_public_model_name = original_model_name - group_model_name = f"model_name_{_team_id}_{original_model_name}" + unique_model_name = f"model_name_{_team_id}_{uuid.uuid4()}" - model_params.model_name = group_model_name + model_params.model_name = unique_model_name ## CREATE MODEL IN DB ## model_response = await _add_model_to_db( @@ -348,17 +344,6 @@ async def _add_team_model_to_db( prisma_client=prisma_client, ) - ## CREATE MODEL ALIAS IN DB ## - await update_team( - data=UpdateTeamRequest( - team_id=_team_id, - model_aliases={original_model_name: group_model_name}, - ), - user_api_key_dict=user_api_key_dict, - http_request=Request(scope={"type": "http"}), - ) - - # add model to team object await team_model_add( data=TeamModelAddRequest( team_id=_team_id, @@ -457,18 +442,9 @@ async def _setup_new_team_model_assignment( patch_data: updateDeployment, user_api_key_dict: UserAPIKeyAuth, ) -> None: - """Set up a new team model with deterministic name, alias, and team membership.""" - group_model_name = f"model_name_{team_id}_{public_model_name}" - patch_data.model_name = group_model_name - - await update_team( - data=UpdateTeamRequest( - team_id=team_id, - model_aliases={public_model_name: group_model_name}, - ), - user_api_key_dict=user_api_key_dict, - http_request=Request(scope={"type": "http"}), - ) + """Set up a new team model with unique name and team membership.""" + unique_model_name = f"model_name_{team_id}_{uuid.uuid4()}" + patch_data.model_name = unique_model_name await team_model_add( data=TeamModelAddRequest( @@ -492,18 +468,16 @@ async def _update_existing_team_model_assignment( db_model.model_info.team_public_model_name if db_model.model_info else None ) - # Update alias only if public name changed if old_public_name and public_model_name != old_public_name: - await update_team( - data=UpdateTeamRequest( + await team_model_add( + data=TeamModelAddRequest( team_id=team_id, - model_aliases={public_model_name: db_model.model_name}, + models=[public_model_name], ), - user_api_key_dict=user_api_key_dict, http_request=Request(scope={"type": "http"}), + user_api_key_dict=user_api_key_dict, ) - # Keep existing unique model_name patch_data.model_name = None diff --git a/litellm/router.py b/litellm/router.py index 0fd4ba80c74..98b3334268e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8070,20 +8070,23 @@ class Router: def map_team_model(self, team_model_name: str, team_id: str) -> Optional[str]: """ - Map a team model name to a team-specific model name. + Check if team_model_name resolves to team-specific deployments. + + Returns the public model name (unchanged) so the router can find all + sibling deployments via team_id filtering, instead of collapsing to a + single internal model_name. Returns: - - deployment id: str - the deployment id of the team-specific model - - None: if no team-specific model name is found + - str: the team_model_name if team deployments exist for this team + - None: if no team-specific model is found """ models = self.get_model_list(model_name=team_model_name, team_id=team_id) if not models: return None for model in models: if model.get("model_info", {}).get("team_id") == team_id: - return model.get("model_name") + return team_model_name - ## wildcard models return None def should_include_deployment( @@ -8778,6 +8781,38 @@ class Router: model = _model_from_alias if model not in self.model_names: + # Check for team-specific deployments by team_public_model_name + if request_team_id is not None: + team_deployments = self._get_all_deployments( + model_name=model, team_id=request_team_id + ) + if team_deployments: + candidate_details = [] + for deployment in team_deployments: + deployment_info = deployment.get("model_info", {}) or {} + deployment_params = deployment.get("litellm_params", {}) or {} + candidate_details.append( + { + "model_name": deployment.get("model_name"), + "model_id": deployment_info.get("id"), + "team_public_model_name": deployment_info.get( + "team_public_model_name" + ), + "api_base": deployment_params.get("api_base"), + } + ) + verbose_router_logger.info( + "πŸ”₯ routing_candidates_before_lb " + f"model={model} count={len(team_deployments)} " + f"candidates={candidate_details}" + ) + if len(team_deployments) > 1: + verbose_router_logger.info( + "πŸ”₯ load_balancer_candidate_pool " + f"model={model} candidate_count={len(team_deployments)}" + ) + return model, team_deployments + # check if provider/ specific wildcard routing use pattern matching pattern_deployments = self.pattern_router.get_deployments_by_pattern( model=model, @@ -8816,6 +8851,32 @@ class Router: # check if the user sent in a deployment name instead healthy_deployments = self._get_deployment_by_litellm_model(model=model) + if isinstance(healthy_deployments, list) and len(healthy_deployments) > 0: + candidate_details = [] + for deployment in healthy_deployments: + deployment_info = deployment.get("model_info", {}) or {} + deployment_params = deployment.get("litellm_params", {}) or {} + candidate_details.append( + { + "model_name": deployment.get("model_name"), + "model_id": deployment_info.get("id"), + "team_public_model_name": deployment_info.get( + "team_public_model_name" + ), + "api_base": deployment_params.get("api_base"), + } + ) + verbose_router_logger.info( + "πŸ”₯ routing_candidates_before_lb " + f"model={model} count={len(healthy_deployments)} " + f"candidates={candidate_details}" + ) + if len(healthy_deployments) > 1: + verbose_router_logger.info( + "πŸ”₯ load_balancer_candidate_pool " + f"model={model} candidate_count={len(healthy_deployments)}" + ) + if verbose_router_logger.isEnabledFor(logging.DEBUG): verbose_router_logger.debug( f"initial list of deployments: {healthy_deployments}" diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 7f64a3de935..5ef7face1f2 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -564,98 +564,124 @@ class TestUpdatePublicModelGroups: litellm.public_model_groups_links = original_value -class TestTeamModelAliasSiblingOverwrite: +class TestTeamModelSiblingRouting: """ - Verify that two sibling team deployments for the same public model name - produce the same deterministic internal model_name, so the alias write - is idempotent and the router groups both deployments together. + Verify that sibling team deployments (same public model name, different + api_base) are all reachable through routing β€” no alias overwrite, no + collapse to a single deployment. """ @pytest.mark.asyncio - async def test_sibling_team_models_share_deterministic_name(self): + async def test_no_model_aliases_written_for_team_models(self): + """ + _add_team_model_to_db must NOT write model_aliases (which caused + the second sibling to overwrite the first). It should only call + team_model_add to register the public name on the team's models list. + """ from litellm.proxy.management_endpoints.model_management_endpoints import ( _add_team_model_to_db, ) from litellm.types.router import ModelInfo - team_id = "team_alias_overwrite" + team_id = "team_no_alias" public_name = "gpt-4.1-mini" - captured_alias_calls = [] - - async def mock_update_team(data, user_api_key_dict, http_request): - if data.model_aliases: - captured_alias_calls.append(dict(data.model_aliases)) + mock_update_team = AsyncMock() async def mock_add_model_to_db(model_params, user_api_key_dict, prisma_client): return MagicMock(model_id=str(uuid.uuid4())) - async def mock_team_model_add(data, http_request, user_api_key_dict): - pass + mock_team_model_add = AsyncMock() user = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) prisma_client = MockPrismaClient(team_exists=True) - deployment_1 = Deployment( - model_name=public_name, - litellm_params=LiteLLM_Params( - model="azure/gpt-4o-mini", - api_key="key-1", - api_base="https://eastus.example.openai.azure.com", - ), - model_info=ModelInfo(team_id=team_id), - ) - deployment_2 = Deployment( - model_name=public_name, - litellm_params=LiteLLM_Params( - model="azure/gpt-4o-mini", - api_key="key-2", - api_base="https://westus.example.openai.azure.com", - ), - model_info=ModelInfo(team_id=team_id), - ) - - with patch( - "litellm.proxy.management_endpoints.model_management_endpoints.update_team", - side_effect=mock_update_team, - ), patch( - "litellm.proxy.management_endpoints.model_management_endpoints._add_model_to_db", - side_effect=mock_add_model_to_db, - ), patch( - "litellm.proxy.management_endpoints.model_management_endpoints.team_model_add", - side_effect=mock_team_model_add, - ): - await _add_team_model_to_db( - model_params=deployment_1, - user_api_key_dict=user, - prisma_client=prisma_client, - ) - await _add_team_model_to_db( - model_params=deployment_2, - user_api_key_dict=user, - prisma_client=prisma_client, + for api_base in ["https://eastus.example.com", "https://westus.example.com"]: + dep = Deployment( + model_name=public_name, + litellm_params=LiteLLM_Params( + model="azure/gpt-4o-mini", + api_key="key", + api_base=api_base, + ), + model_info=ModelInfo(team_id=team_id), ) + with patch( + "litellm.proxy.management_endpoints.model_management_endpoints.update_team", + mock_update_team, + ), patch( + "litellm.proxy.management_endpoints.model_management_endpoints._add_model_to_db", + side_effect=mock_add_model_to_db, + ), patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_add", + mock_team_model_add, + ): + await _add_team_model_to_db( + model_params=dep, + user_api_key_dict=user, + prisma_client=prisma_client, + ) - assert len(captured_alias_calls) == 2 + mock_update_team.assert_not_called() + assert mock_team_model_add.call_count == 2 - internal_name_1 = captured_alias_calls[0][public_name] - internal_name_2 = captured_alias_calls[1][public_name] + @pytest.mark.asyncio + async def test_router_finds_all_sibling_team_deployments(self): + """ + When two team deployments share team_public_model_name="gpt-4.1-mini", + the router's _common_checks_available_deployment must return BOTH as + healthy_deployments (not collapse to one). + """ + import litellm - expected_group_name = f"model_name_{team_id}_{public_name}" + team_id = "teamA" + public_name = "gpt-4.1-mini" - # Both sibling deployments get the same deterministic group name - assert internal_name_1 == expected_group_name - assert internal_name_2 == expected_group_name - assert internal_name_1 == internal_name_2, ( - "Sibling deployments must share the same model_name so the " - "router treats them as a single candidate pool" + router = litellm.Router( + model_list=[ + { + "model_name": f"model_name_{team_id}_uuid1", + "litellm_params": { + "model": "azure/gpt-4o-mini", + "api_key": "key-1", + "api_base": "https://eastus.openai.azure.com", + }, + "model_info": { + "team_id": team_id, + "team_public_model_name": public_name, + }, + }, + { + "model_name": f"model_name_{team_id}_uuid2", + "litellm_params": { + "model": "azure/gpt-4o-mini", + "api_key": "key-2", + "api_base": "https://westus.openai.azure.com", + }, + "model_info": { + "team_id": team_id, + "team_public_model_name": public_name, + }, + }, + ], ) - # The second alias write is idempotent β€” same key, same value - final_aliases = {} - for alias_call in captured_alias_calls: - final_aliases.update(alias_call) - assert final_aliases == {public_name: expected_group_name} + # map_team_model should return the public name (not an internal UUID) + result = router.map_team_model(public_name, team_id) + assert result == public_name + + # _common_checks_available_deployment should return both deployments + model, healthy = router._common_checks_available_deployment( + model=public_name, + request_kwargs={"metadata": {"user_api_key_team_id": team_id}}, + ) + assert isinstance(healthy, list) + assert len(healthy) == 2 + api_bases = {d["litellm_params"]["api_base"] for d in healthy} + assert api_bases == { + "https://eastus.openai.azure.com", + "https://westus.openai.azure.com", + } class TestTeamModelUpdate: @@ -704,7 +730,7 @@ class TestTeamModelUpdate: assert result.get("model_name", "").startswith("model_name_test_team_123_") assert "team_public_model_name" in str(result.get("model_info", "")) - mock_update_team.assert_called_once() + mock_update_team.assert_not_called() mock_team_model_add.assert_called_once() @pytest.mark.asyncio From 8f68def60efc5fa6a84ca0947dac9c75c662feab Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 16:14:30 +0530 Subject: [PATCH 03/22] chore(team-routing): remove temporary candidate pool logs Remove temporary fire-emoji router logs used for local verification while keeping team sibling deployment routing behavior unchanged. Made-with: Cursor --- litellm/router.py | 50 ----------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 98b3334268e..78b772850b1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8787,30 +8787,6 @@ class Router: model_name=model, team_id=request_team_id ) if team_deployments: - candidate_details = [] - for deployment in team_deployments: - deployment_info = deployment.get("model_info", {}) or {} - deployment_params = deployment.get("litellm_params", {}) or {} - candidate_details.append( - { - "model_name": deployment.get("model_name"), - "model_id": deployment_info.get("id"), - "team_public_model_name": deployment_info.get( - "team_public_model_name" - ), - "api_base": deployment_params.get("api_base"), - } - ) - verbose_router_logger.info( - "πŸ”₯ routing_candidates_before_lb " - f"model={model} count={len(team_deployments)} " - f"candidates={candidate_details}" - ) - if len(team_deployments) > 1: - verbose_router_logger.info( - "πŸ”₯ load_balancer_candidate_pool " - f"model={model} candidate_count={len(team_deployments)}" - ) return model, team_deployments # check if provider/ specific wildcard routing use pattern matching @@ -8851,32 +8827,6 @@ class Router: # check if the user sent in a deployment name instead healthy_deployments = self._get_deployment_by_litellm_model(model=model) - if isinstance(healthy_deployments, list) and len(healthy_deployments) > 0: - candidate_details = [] - for deployment in healthy_deployments: - deployment_info = deployment.get("model_info", {}) or {} - deployment_params = deployment.get("litellm_params", {}) or {} - candidate_details.append( - { - "model_name": deployment.get("model_name"), - "model_id": deployment_info.get("id"), - "team_public_model_name": deployment_info.get( - "team_public_model_name" - ), - "api_base": deployment_params.get("api_base"), - } - ) - verbose_router_logger.info( - "πŸ”₯ routing_candidates_before_lb " - f"model={model} count={len(healthy_deployments)} " - f"candidates={candidate_details}" - ) - if len(healthy_deployments) > 1: - verbose_router_logger.info( - "πŸ”₯ load_balancer_candidate_pool " - f"model={model} candidate_count={len(healthy_deployments)}" - ) - if verbose_router_logger.isEnabledFor(logging.DEBUG): verbose_router_logger.debug( f"initial list of deployments: {healthy_deployments}" From 5c9dd540b143892c6345a7ce187e11502f843cd2 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 16:25:26 +0530 Subject: [PATCH 04/22] fix(router): address Greptile review comments - Add None guard for original_model_name in _add_team_model_to_db - Remove stale old public name when renaming team model - Add comment clarifying team deployment early-return priority Made-with: Cursor --- .../model_management_endpoints.py | 27 +++++++++++++------ litellm/router.py | 4 ++- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 694fa2b1d55..c091f6b5812 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -32,6 +32,7 @@ from litellm.proxy._types import ( ProxyErrorTypes, ProxyException, TeamModelAddRequest, + TeamModelDeleteRequest, UpdateTeamRequest, UserAPIKeyAuth, ) @@ -40,6 +41,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helpe from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin from litellm.proxy.management_endpoints.team_endpoints import ( team_model_add, + team_model_delete, update_team, ) from litellm.proxy.management_helpers.audit_logs import create_object_audit_log @@ -344,14 +346,15 @@ async def _add_team_model_to_db( prisma_client=prisma_client, ) - await team_model_add( - data=TeamModelAddRequest( - team_id=_team_id, - models=[original_model_name], - ), - http_request=Request(scope={"type": "http"}), - user_api_key_dict=user_api_key_dict, - ) + if original_model_name: + await team_model_add( + data=TeamModelAddRequest( + team_id=_team_id, + models=[original_model_name], + ), + http_request=Request(scope={"type": "http"}), + user_api_key_dict=user_api_key_dict, + ) return model_response @@ -469,6 +472,14 @@ async def _update_existing_team_model_assignment( ) if old_public_name and public_model_name != old_public_name: + await team_model_delete( + data=TeamModelDeleteRequest( + team_id=team_id, + models=[old_public_name], + ), + http_request=Request(scope={"type": "http"}), + user_api_key_dict=user_api_key_dict, + ) await team_model_add( data=TeamModelAddRequest( team_id=team_id, diff --git a/litellm/router.py b/litellm/router.py index 78b772850b1..825cca22362 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8781,7 +8781,9 @@ class Router: model = _model_from_alias if model not in self.model_names: - # Check for team-specific deployments by team_public_model_name + # Check for team-specific deployments by team_public_model_name. + # This intentionally takes priority over team pattern routers below, + # so that named team deployments shadow wildcard/pattern routes. if request_team_id is not None: team_deployments = self._get_all_deployments( model_name=model, team_id=request_team_id From d2f1359e7dbb60456e974bcdf69e2d49e3fcab65 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 16:33:26 +0530 Subject: [PATCH 05/22] fix(router): address remaining Greptile P0/P1 issues - Update map_team_model test to expect public name return - Only remove old public name if no sibling deployments use it Made-with: Cursor --- .../model_management_endpoints.py | 34 ++++++++++++++----- .../test_get_model_list_alias_optimization.py | 2 +- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index c091f6b5812..eb85112a66c 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -472,14 +472,32 @@ async def _update_existing_team_model_assignment( ) if old_public_name and public_model_name != old_public_name: - await team_model_delete( - data=TeamModelDeleteRequest( - team_id=team_id, - models=[old_public_name], - ), - http_request=Request(scope={"type": "http"}), - user_api_key_dict=user_api_key_dict, - ) + from litellm.proxy.proxy_server import llm_router + + other_deployments_with_old_name = [] + if llm_router: + all_deployments = llm_router.get_model_list( + model_name=old_public_name, team_id=team_id + ) + if all_deployments: + other_deployments_with_old_name = [ + d + for d in all_deployments + if d.get("model_name") != db_model.model_name + and d.get("model_info", {}).get("team_public_model_name") + == old_public_name + ] + + if not other_deployments_with_old_name: + await team_model_delete( + data=TeamModelDeleteRequest( + team_id=team_id, + models=[old_public_name], + ), + http_request=Request(scope={"type": "http"}), + user_api_key_dict=user_api_key_dict, + ) + await team_model_add( data=TeamModelAddRequest( team_id=team_id, diff --git a/tests/router_unit_tests/test_get_model_list_alias_optimization.py b/tests/router_unit_tests/test_get_model_list_alias_optimization.py index 31d992b6646..62baf0a3d22 100644 --- a/tests/router_unit_tests/test_get_model_list_alias_optimization.py +++ b/tests/router_unit_tests/test_get_model_list_alias_optimization.py @@ -46,5 +46,5 @@ def test_map_team_model_should_not_iterate_aliases_for_non_alias_team_model_name assert ( router.map_team_model(team_model_name="team-model", team_id="team-1") - == "gpt-3.5-turbo" + == "team-model" ) From 16fcb71557df2f7b534cf6f7171da20e1b08c560 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 16:44:30 +0530 Subject: [PATCH 06/22] fix(router): address Greptile P1/P2 performance issues - Guard against llm_router=None to prevent silent deletion - Add O(1) team_model index to avoid O(n) scan on every team request Made-with: Cursor --- .../model_management_endpoints.py | 26 ++++---- litellm/router.py | 61 +++++++++++++++++++ 2 files changed, 76 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index eb85112a66c..bfd67ea4e59 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -474,11 +474,15 @@ async def _update_existing_team_model_assignment( if old_public_name and public_model_name != old_public_name: from litellm.proxy.proxy_server import llm_router - other_deployments_with_old_name = [] - if llm_router: + if llm_router is None: + verbose_proxy_logger.warning( + "llm_router not initialized; skipping old public name cleanup to preserve sibling deployments" + ) + else: all_deployments = llm_router.get_model_list( model_name=old_public_name, team_id=team_id ) + other_deployments_with_old_name = [] if all_deployments: other_deployments_with_old_name = [ d @@ -488,15 +492,15 @@ async def _update_existing_team_model_assignment( == old_public_name ] - if not other_deployments_with_old_name: - await team_model_delete( - data=TeamModelDeleteRequest( - team_id=team_id, - models=[old_public_name], - ), - http_request=Request(scope={"type": "http"}), - user_api_key_dict=user_api_key_dict, - ) + if not other_deployments_with_old_name: + await team_model_delete( + data=TeamModelDeleteRequest( + team_id=team_id, + models=[old_public_name], + ), + http_request=Request(scope={"type": "http"}), + user_api_key_dict=user_api_key_dict, + ) await team_model_add( data=TeamModelAddRequest( diff --git a/litellm/router.py b/litellm/router.py index 825cca22362..7d7bb3d0924 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -466,6 +466,8 @@ class Router: # Initialize model name to deployment indices mapping for O(1) lookups # Maps model_name -> list of indices in model_list self.model_name_to_deployment_indices: Dict[str, List[int]] = {} + # Maps (team_id, team_public_model_name) -> list of indices in model_list + self.team_model_to_deployment_indices: Dict[Tuple[str, str], List[int]] = {} if model_list is not None: # set_model_list will build indices automatically @@ -6757,6 +6759,7 @@ class Router: self.model_list = [] self.model_id_to_deployment_index_map = {} # Reset the index self.model_name_to_deployment_indices = {} # Reset the model_name index + self.team_model_to_deployment_indices = {} # Reset the team_model index self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() # we add api_base/api_key each model so load balancing between azure/gpt on api_base1 and api_base2 works @@ -7072,6 +7075,26 @@ class Router: else: del self.model_name_to_deployment_indices[model_name] + # Update team_model_to_deployment_indices + for key, indices in list(self.team_model_to_deployment_indices.items()): + # Remove the deleted index + if removal_idx in indices: + indices.remove(removal_idx) + + # Decrement all indices greater than removal_idx + updated_indices = [] + for idx in indices: + if idx > removal_idx: + updated_indices.append(idx - 1) + else: + updated_indices.append(idx) + + # Update or remove the entry + if len(updated_indices) > 0: + self.team_model_to_deployment_indices[key] = updated_indices + else: + del self.team_model_to_deployment_indices[key] + def _add_model_to_list_and_index_map( self, model: dict, model_id: Optional[str] = None ) -> None: @@ -7100,6 +7123,17 @@ class Router: self.model_name_to_deployment_indices[model_name] = [] self.model_name_to_deployment_indices[model_name].append(idx) + # Update team_model index for O(1) team-scoped lookup + team_id = model.get("model_info", {}).get("team_id") + team_public_model_name = model.get("model_info", {}).get( + "team_public_model_name" + ) + if team_id and team_public_model_name: + key = (team_id, team_public_model_name) + if key not in self.team_model_to_deployment_indices: + self.team_model_to_deployment_indices[key] = [] + self.team_model_to_deployment_indices[key].append(idx) + def upsert_deployment(self, deployment: Deployment) -> Optional[Deployment]: """ Add or update deployment @@ -7930,6 +7964,7 @@ class Router: instead of O(n) linear scan through the entire model_list. """ self.model_name_to_deployment_indices.clear() + self.team_model_to_deployment_indices.clear() for idx, model in enumerate(model_list): model_name = model.get("model_name") @@ -7938,6 +7973,16 @@ class Router: self.model_name_to_deployment_indices[model_name] = [] self.model_name_to_deployment_indices[model_name].append(idx) + team_id = model.get("model_info", {}).get("team_id") + team_public_model_name = model.get("model_info", {}).get( + "team_public_model_name" + ) + if team_id and team_public_model_name: + key = (team_id, team_public_model_name) + if key not in self.team_model_to_deployment_indices: + self.team_model_to_deployment_indices[key] = [] + self.team_model_to_deployment_indices[key].append(idx) + def _build_model_id_to_deployment_index_map(self, model_list: list): """ Build model index from model list to enable O(1) lookups immediately. @@ -8122,6 +8167,22 @@ class Router: """ returned_models: List[DeploymentTypedDict] = [] + # O(1) lookup in team_model index when team_id is provided + if team_id is not None: + key = (team_id, model_name) + if key in self.team_model_to_deployment_indices: + indices = self.team_model_to_deployment_indices[key] + # O(k) where k = team deployments for this model_name (typically 1-10) + for idx in indices: + model = self.model_list[idx] + if model_alias is not None: + alias_model = model.copy() + alias_model["model_name"] = model_alias + returned_models.append(alias_model) + else: + returned_models.append(model) + return returned_models + # O(1) lookup in model_name index if model_name in self.model_name_to_deployment_indices: indices = self.model_name_to_deployment_indices[model_name] From 4b0c7b8a9f74b2ed726f6cba9925b12e4a3db260 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 17:00:30 +0530 Subject: [PATCH 07/22] fix(router): prevent cross-team deployment leakage in fallback path Guard should_include_deployment fallback to only return deployments matching the requested team_id, preventing public-name collisions from leaking deployments across teams Made-with: Cursor --- litellm/router.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 7d7bb3d0924..a610ca8ec64 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8147,7 +8147,8 @@ class Router: ): return True elif model_name is not None and model["model_name"] == model_name: - return True + if team_id is None or model["model_info"].get("team_id") == team_id: + return True return False def _get_all_deployments( From 80b5ccd358ea71d26301e6297a92f6d56d54a251 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 17:10:11 +0530 Subject: [PATCH 08/22] fix(management): query DB directly for sibling deployments on rename - Add clarifying comments to test assertions - Query prisma DB instead of in-memory router to avoid stale state - Prevents incorrect deletion of old public name when siblings exist Made-with: Cursor --- .../model_management_endpoints.py | 32 ++++++++++--------- .../test_model_management_endpoints.py | 9 ++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index bfd67ea4e59..c38a00e18a1 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -420,6 +420,7 @@ async def _update_team_model_in_db( db_model=db_model, patch_data=patch_data, user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, ) return update_db_model(db_model=db_model, updated_patch=patch_data) @@ -465,6 +466,7 @@ async def _update_existing_team_model_assignment( db_model: Deployment, patch_data: updateDeployment, user_api_key_dict: UserAPIKeyAuth, + prisma_client: PrismaClient, ) -> None: """Update an existing team model if the public name changed.""" old_public_name = ( @@ -472,25 +474,25 @@ async def _update_existing_team_model_assignment( ) if old_public_name and public_model_name != old_public_name: - from litellm.proxy.proxy_server import llm_router - - if llm_router is None: + if prisma_client is None: verbose_proxy_logger.warning( - "llm_router not initialized; skipping old public name cleanup to preserve sibling deployments" + "prisma_client not initialized; skipping old public name cleanup to preserve sibling deployments" ) else: - all_deployments = llm_router.get_model_list( - model_name=old_public_name, team_id=team_id + response = await prisma_client.db.litellm_proxymodeltable.find_many( + where={ + "model_info": { + "path": ["team_id"], + "equals": team_id, + } + } ) - other_deployments_with_old_name = [] - if all_deployments: - other_deployments_with_old_name = [ - d - for d in all_deployments - if d.get("model_name") != db_model.model_name - and d.get("model_info", {}).get("team_public_model_name") - == old_public_name - ] + other_deployments_with_old_name = [ + d + for d in response + if d.model_name != db_model.model_name + and d.model_info.get("team_public_model_name") == old_public_name + ] if not other_deployments_with_old_name: await team_model_delete( diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 5ef7face1f2..fd4f3d56b10 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -46,10 +46,17 @@ class MockPrismaClient: ) return None + async def find_many(self, where): + return [] + @property def litellm_teamtable(self): return self + @property + def litellm_proxymodeltable(self): + return self + class MockLLMRouter: def __init__(self): @@ -730,7 +737,9 @@ class TestTeamModelUpdate: assert result.get("model_name", "").startswith("model_name_test_team_123_") assert "team_public_model_name" in str(result.get("model_info", "")) + # update_team must not be called (no model_aliases writes for team models) mock_update_team.assert_not_called() + # team_model_add must be called to add public name to team's models list mock_team_model_add.assert_called_once() @pytest.mark.asyncio From a1ef6f7c5720b1e3569bd370ba1216e8f0ca1dac Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 17:19:11 +0530 Subject: [PATCH 09/22] fix(router): guard None model_info and deduplicate team index logic - Guard against None model_info in sibling deployment check - Extract _update_team_model_index helper to eliminate duplication Made-with: Cursor --- .../model_management_endpoints.py | 3 +- litellm/router.py | 38 ++++++++++--------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index c38a00e18a1..4b06c4460e7 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -491,7 +491,8 @@ async def _update_existing_team_model_assignment( d for d in response if d.model_name != db_model.model_name - and d.model_info.get("team_public_model_name") == old_public_name + and (d.model_info or {}).get("team_public_model_name") + == old_public_name ] if not other_deployments_with_old_name: diff --git a/litellm/router.py b/litellm/router.py index a610ca8ec64..91381b3763f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7095,6 +7095,24 @@ class Router: else: del self.team_model_to_deployment_indices[key] + def _update_team_model_index(self, model: dict, idx: int) -> None: + """ + Helper to update team_model_to_deployment_indices for a single deployment. + + Parameters: + - model: dict - the deployment to index + - idx: int - the index in model_list + """ + team_id = model.get("model_info", {}).get("team_id") + team_public_model_name = model.get("model_info", {}).get( + "team_public_model_name" + ) + if team_id and team_public_model_name: + key = (team_id, team_public_model_name) + if key not in self.team_model_to_deployment_indices: + self.team_model_to_deployment_indices[key] = [] + self.team_model_to_deployment_indices[key].append(idx) + def _add_model_to_list_and_index_map( self, model: dict, model_id: Optional[str] = None ) -> None: @@ -7124,15 +7142,7 @@ class Router: self.model_name_to_deployment_indices[model_name].append(idx) # Update team_model index for O(1) team-scoped lookup - team_id = model.get("model_info", {}).get("team_id") - team_public_model_name = model.get("model_info", {}).get( - "team_public_model_name" - ) - if team_id and team_public_model_name: - key = (team_id, team_public_model_name) - if key not in self.team_model_to_deployment_indices: - self.team_model_to_deployment_indices[key] = [] - self.team_model_to_deployment_indices[key].append(idx) + self._update_team_model_index(model, idx) def upsert_deployment(self, deployment: Deployment) -> Optional[Deployment]: """ @@ -7973,15 +7983,7 @@ class Router: self.model_name_to_deployment_indices[model_name] = [] self.model_name_to_deployment_indices[model_name].append(idx) - team_id = model.get("model_info", {}).get("team_id") - team_public_model_name = model.get("model_info", {}).get( - "team_public_model_name" - ) - if team_id and team_public_model_name: - key = (team_id, team_public_model_name) - if key not in self.team_model_to_deployment_indices: - self.team_model_to_deployment_indices[key] = [] - self.team_model_to_deployment_indices[key].append(idx) + self._update_team_model_index(model, idx) def _build_model_id_to_deployment_index_map(self, model_list: list): """ From db40ab027d17f66bc2fc1aa14a4c293fa75cefe0 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 17:33:07 +0530 Subject: [PATCH 10/22] fix(routing): prevent stale model_aliases from interfering with team routing - Skip model_aliases rewrite if model resolves to team deployments - Add test coverage for sibling-preservation branch - Update MockPrismaClient to support sibling deployment scenarios Made-with: Cursor --- litellm/proxy/litellm_pre_call_utils.py | 15 ++++ .../test_model_management_endpoints.py | 70 ++++++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index daf2867699e..b0bd92806e2 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1295,6 +1295,10 @@ def _update_model_if_team_alias_exists( "gpt-4o": "gpt-4o-team-1" } - requested_model = "gpt-4o-team-1" + + Note: model_aliases for team models are deprecated. This function only applies + to legacy non-team-scoped aliases. Team-scoped deployments use team_public_model_name + and are resolved via map_team_model in route_llm_request. """ _model = data.get("model") if ( @@ -1302,6 +1306,17 @@ def _update_model_if_team_alias_exists( and user_api_key_dict.team_model_aliases and _model in user_api_key_dict.team_model_aliases ): + from litellm.proxy.proxy_server import llm_router + + # Skip alias rewrite if this model resolves to team-specific deployments + # (team models use team_public_model_name, not model_aliases) + if ( + llm_router + and user_api_key_dict.team_id + and llm_router.map_team_model(_model, user_api_key_dict.team_id) is not None + ): + return + data["model"] = user_api_key_dict.team_model_aliases[_model] return diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index fd4f3d56b10..dcfd5847bd5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -28,9 +28,15 @@ from litellm.types.router import Deployment, LiteLLM_Params, updateDeployment class MockPrismaClient: - def __init__(self, team_exists: bool = True, user_admin: bool = True): + def __init__( + self, + team_exists: bool = True, + user_admin: bool = True, + sibling_deployments: list = None, + ): self.team_exists = team_exists self.user_admin = user_admin + self.sibling_deployments = sibling_deployments or [] self.db = self async def find_unique(self, where): @@ -47,7 +53,7 @@ class MockPrismaClient: return None async def find_many(self, where): - return [] + return self.sibling_deployments @property def litellm_teamtable(self): @@ -742,6 +748,66 @@ class TestTeamModelUpdate: # team_model_add must be called to add public name to team's models list mock_team_model_add.assert_called_once() + @pytest.mark.asyncio + async def test_rename_preserves_old_name_when_siblings_exist(self): + """Test that renaming a deployment preserves old public name when sibling deployments still use it""" + from unittest.mock import MagicMock + + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _update_existing_team_model_assignment, + ) + from litellm.types.router import ModelInfo + + # Create a deployment being renamed + db_model = Deployment( + model_name="model_name_team_123_uuid1", + litellm_params=LiteLLM_Params(model="azure/gpt-4o-mini"), + model_info=ModelInfo( + team_id="team_123", team_public_model_name="old-public-name" + ), + ) + + # Create a sibling deployment that still uses the old public name + sibling_deployment = MagicMock() + sibling_deployment.model_name = "model_name_team_123_uuid2" + sibling_deployment.model_info = { + "team_id": "team_123", + "team_public_model_name": "old-public-name", + } + + prisma_client = MockPrismaClient( + team_exists=True, sibling_deployments=[sibling_deployment] + ) + + patch_data = updateDeployment( + model_name="new-public-name", + model_info=ModelInfo(team_id="team_123"), + ) + + user_api_key_dict = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + with patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_delete" + ) as mock_delete, patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_add" + ) as mock_add: + await _update_existing_team_model_assignment( + team_id="team_123", + public_model_name="new-public-name", + db_model=db_model, + patch_data=patch_data, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, # type: ignore + ) + + # team_model_delete should NOT be called because sibling exists + mock_delete.assert_not_called() + # team_model_add should be called to add new public name + mock_add.assert_called_once() + @pytest.mark.asyncio async def test_patch_model_with_team_id_validates_permissions(self): """Test PATCH with team_id runs same validation as POST for team permissions""" From 4005fba2e21a3ead1a1bc9b7ce344ba002b6100b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 17:56:03 +0530 Subject: [PATCH 11/22] perf(routing): optimize team model checks and improve test coverage - Use O(1) team index lookup instead of map_team_model in alias guard - Fix MockPrismaClient to validate where clause filters - Add comment explaining DB query trade-off for team deployments Made-with: Cursor --- litellm/proxy/litellm_pre_call_utils.py | 11 ++++----- .../model_management_endpoints.py | 4 ++++ .../test_model_management_endpoints.py | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index b0bd92806e2..5de27bb7145 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1310,12 +1310,11 @@ def _update_model_if_team_alias_exists( # Skip alias rewrite if this model resolves to team-specific deployments # (team models use team_public_model_name, not model_aliases) - if ( - llm_router - and user_api_key_dict.team_id - and llm_router.map_team_model(_model, user_api_key_dict.team_id) is not None - ): - return + # Use O(1) index lookup instead of map_team_model to avoid O(n) scan + if llm_router and user_api_key_dict.team_id: + key = (user_api_key_dict.team_id, _model) + if key in llm_router.team_model_to_deployment_indices: + return data["model"] = user_api_key_dict.team_model_aliases[_model] return diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 4b06c4460e7..7682b65657a 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -479,6 +479,10 @@ async def _update_existing_team_model_assignment( "prisma_client not initialized; skipping old public name cleanup to preserve sibling deployments" ) else: + # Query DB for all deployments in this team, then filter by public name. + # Note: Prisma's JSON filtering doesn't support compound AND conditions + # across multiple JSON paths, so we filter team_public_model_name in Python. + # For most teams (typically <100 deployments), this is acceptable. response = await prisma_client.db.litellm_proxymodeltable.find_many( where={ "model_info": { diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index dcfd5847bd5..09410c19d34 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -53,6 +53,29 @@ class MockPrismaClient: return None async def find_many(self, where): + # Filter sibling deployments by team_id if where clause specifies it + if not self.sibling_deployments: + return [] + + # Extract team_id from where clause if present + team_id_filter = None + if where and "model_info" in where: + model_info_filter = where["model_info"] + if isinstance(model_info_filter, dict) and "path" in model_info_filter: + if ( + model_info_filter["path"] == ["team_id"] + and "equals" in model_info_filter + ): + team_id_filter = model_info_filter["equals"] + + # Filter deployments by team_id if specified + if team_id_filter: + return [ + d + for d in self.sibling_deployments + if d.model_info.get("team_id") == team_id_filter + ] + return self.sibling_deployments @property From b5f4e3c7f7c47728d72f7c3b205637b116a062de Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 18:13:35 +0530 Subject: [PATCH 12/22] fix(routing): address state consistency and type safety issues - Check alias target pattern to detect stale team aliases - Fix PrismaClient type annotation to Optional - Eliminate in-place mutation in index update logic Made-with: Cursor --- litellm/proxy/litellm_pre_call_utils.py | 19 +++++++++----- .../model_management_endpoints.py | 2 +- litellm/router.py | 26 ++++++++++--------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 5de27bb7145..d8ed29b0475 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1310,13 +1310,20 @@ def _update_model_if_team_alias_exists( # Skip alias rewrite if this model resolves to team-specific deployments # (team models use team_public_model_name, not model_aliases) - # Use O(1) index lookup instead of map_team_model to avoid O(n) scan - if llm_router and user_api_key_dict.team_id: - key = (user_api_key_dict.team_id, _model) - if key in llm_router.team_model_to_deployment_indices: - return + aliased_target = user_api_key_dict.team_model_aliases[_model] - data["model"] = user_api_key_dict.team_model_aliases[_model] + # Check if the alias points to a stale team-scoped UUID name + # (format: "model_name_{team_id}_{uuid}") + if aliased_target.startswith(f"model_name_{user_api_key_dict.team_id}_"): + # This is a stale alias from pre-PR deployments. + # Check if current team deployments exist for the public name. + if llm_router: + key = (user_api_key_dict.team_id, _model) + if key in llm_router.team_model_to_deployment_indices: + # Team deployments exist; skip stale alias + return + + data["model"] = aliased_target return diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 7682b65657a..7d0181a3687 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -466,7 +466,7 @@ async def _update_existing_team_model_assignment( db_model: Deployment, patch_data: updateDeployment, user_api_key_dict: UserAPIKeyAuth, - prisma_client: PrismaClient, + prisma_client: Optional[PrismaClient], ) -> None: """Update an existing team model if the public name changed.""" old_public_name = ( diff --git a/litellm/router.py b/litellm/router.py index 91381b3763f..fdc23fb5dd4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7057,16 +7057,17 @@ class Router: # Update model_name_to_deployment_indices for model_name, indices in list(self.model_name_to_deployment_indices.items()): - # Remove the deleted index - if removal_idx in indices: - indices.remove(removal_idx) - - # Decrement all indices greater than removal_idx + # Build new list without mutating the original updated_indices = [] for idx in indices: - if idx > removal_idx: + if idx == removal_idx: + # Skip the removed index + continue + elif idx > removal_idx: + # Decrement indices after removal updated_indices.append(idx - 1) else: + # Keep indices before removal unchanged updated_indices.append(idx) # Update or remove the entry @@ -7077,16 +7078,17 @@ class Router: # Update team_model_to_deployment_indices for key, indices in list(self.team_model_to_deployment_indices.items()): - # Remove the deleted index - if removal_idx in indices: - indices.remove(removal_idx) - - # Decrement all indices greater than removal_idx + # Build new list without mutating the original updated_indices = [] for idx in indices: - if idx > removal_idx: + if idx == removal_idx: + # Skip the removed index + continue + elif idx > removal_idx: + # Decrement indices after removal updated_indices.append(idx - 1) else: + # Keep indices before removal unchanged updated_indices.append(idx) # Update or remove the entry From a2e89c1ed1df0f5e528e70bf70e90d5f23c6d4d8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 18:26:43 +0530 Subject: [PATCH 13/22] Fix greptile comments --- litellm/proxy/litellm_pre_call_utils.py | 12 +++- .../model_management_endpoints.py | 20 +++++- tests/proxy_unit_tests/test_proxy_utils.py | 43 ++++++++++++ .../test_model_management_endpoints.py | 70 ++++++++++++++++++- 4 files changed, 140 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d8ed29b0475..fe4df6dcb13 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -26,6 +26,7 @@ _SPECIAL_HEADERS_CACHE = frozenset( v.value.lower() for v in SpecialHeaders._member_map_.values() ) from litellm.router import Router +from litellm.secret_managers.main import get_secret_bool from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS from litellm.types.services import ServiceTypes from litellm.types.utils import ( @@ -1312,9 +1313,16 @@ def _update_model_if_team_alias_exists( # (team models use team_public_model_name, not model_aliases) aliased_target = user_api_key_dict.team_model_aliases[_model] - # Check if the alias points to a stale team-scoped UUID name + # Optional bypass for stale aliases from pre-PR deployments: + # only enabled via feature flag to preserve backwards compatibility. + enable_stale_alias_bypass = get_secret_bool( + "LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", False + ) + # Check if the alias points to a team-scoped UUID name # (format: "model_name_{team_id}_{uuid}") - if aliased_target.startswith(f"model_name_{user_api_key_dict.team_id}_"): + if enable_stale_alias_bypass and aliased_target.startswith( + f"model_name_{user_api_key_dict.team_id}_" + ): # This is a stale alias from pre-PR deployments. # Check if current team deployments exist for the public name. if llm_router: diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 7d0181a3687..40f4d722dc6 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -469,6 +469,23 @@ async def _update_existing_team_model_assignment( prisma_client: Optional[PrismaClient], ) -> None: """Update an existing team model if the public name changed.""" + + def _get_team_public_model_name( + model_info: Optional[Union[dict, str]] + ) -> Optional[str]: + if isinstance(model_info, dict): + value = model_info.get("team_public_model_name") + return value if isinstance(value, str) else None + if isinstance(model_info, str): + try: + parsed = json.loads(model_info) + except (TypeError, ValueError): + return None + if isinstance(parsed, dict): + value = parsed.get("team_public_model_name") + return value if isinstance(value, str) else None + return None + old_public_name = ( db_model.model_info.team_public_model_name if db_model.model_info else None ) @@ -495,8 +512,7 @@ async def _update_existing_team_model_assignment( d for d in response if d.model_name != db_model.model_name - and (d.model_info or {}).get("team_public_model_name") - == old_public_name + and _get_team_public_model_name(d.model_info) == old_public_name ] if not other_deployments_with_old_name: diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 7cff4807f5a..45646fd6a62 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -2044,6 +2044,49 @@ def test_update_model_if_team_alias_exists(data, user_api_key_dict, expected_mod assert test_data.get("model") == expected_model +def test_team_alias_stale_bypass_disabled_by_default(): + from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists + + class _MockRouter: + team_model_to_deployment_indices = {("team-1", "gpt-4o"): [0]} + + test_data = {"model": "gpt-4o"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + team_id="team-1", + team_model_aliases={"gpt-4o": "model_name_team-1_legacy-uuid"}, + ) + + with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()): + _update_model_if_team_alias_exists( + data=test_data, user_api_key_dict=user_api_key_dict + ) + + assert test_data.get("model") == "model_name_team-1_legacy-uuid" + + +def test_team_alias_stale_bypass_enabled_by_flag(monkeypatch): + from litellm.proxy.litellm_pre_call_utils import _update_model_if_team_alias_exists + + class _MockRouter: + team_model_to_deployment_indices = {("team-1", "gpt-4o"): [0]} + + test_data = {"model": "gpt-4o"} + user_api_key_dict = UserAPIKeyAuth( + api_key="test_key", + team_id="team-1", + team_model_aliases={"gpt-4o": "model_name_team-1_legacy-uuid"}, + ) + monkeypatch.setenv("LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS", "true") + + with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()): + _update_model_if_team_alias_exists( + data=test_data, user_api_key_dict=user_api_key_dict + ) + + assert test_data.get("model") == "gpt-4o" + + @pytest.fixture def mock_prisma_client(): client = MagicMock() diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 09410c19d34..83e6b0c93a5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -70,10 +70,23 @@ class MockPrismaClient: # Filter deployments by team_id if specified if team_id_filter: + + def _get_team_id(model_info): + if isinstance(model_info, dict): + return model_info.get("team_id") + if isinstance(model_info, str): + try: + parsed = json.loads(model_info) + except (TypeError, ValueError): + return None + if isinstance(parsed, dict): + return parsed.get("team_id") + return None + return [ d for d in self.sibling_deployments - if d.model_info.get("team_id") == team_id_filter + if _get_team_id(d.model_info) == team_id_filter ] return self.sibling_deployments @@ -831,6 +844,61 @@ class TestTeamModelUpdate: # team_model_add should be called to add new public name mock_add.assert_called_once() + @pytest.mark.asyncio + async def test_rename_handles_legacy_string_model_info(self): + """Test rename path handles legacy string-encoded model_info rows without crashing.""" + from unittest.mock import MagicMock + + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _update_existing_team_model_assignment, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_team_123_uuid1", + litellm_params=LiteLLM_Params(model="azure/gpt-4o-mini"), + model_info=ModelInfo( + team_id="team_123", team_public_model_name="old-public-name" + ), + ) + + sibling_deployment = MagicMock() + sibling_deployment.model_name = "model_name_team_123_uuid2" + sibling_deployment.model_info = ( + '{"team_id":"team_123","team_public_model_name":"old-public-name"}' + ) + + prisma_client = MockPrismaClient( + team_exists=True, sibling_deployments=[sibling_deployment] + ) + + patch_data = updateDeployment( + model_name="new-public-name", + model_info=ModelInfo(team_id="team_123"), + ) + + user_api_key_dict = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + with patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_delete" + ) as mock_delete, patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_add" + ) as mock_add: + await _update_existing_team_model_assignment( + team_id="team_123", + public_model_name="new-public-name", + db_model=db_model, + patch_data=patch_data, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, # type: ignore + ) + + mock_delete.assert_not_called() + mock_add.assert_called_once() + @pytest.mark.asyncio async def test_patch_model_with_team_id_validates_permissions(self): """Test PATCH with team_id runs same validation as POST for team permissions""" From b1c7d442a6ae478012411d0b8240f1d363c94695 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 19:16:38 +0530 Subject: [PATCH 14/22] Fix greptile comments --- .../model_management_endpoints.py | 27 +++++++++++-------- litellm/router.py | 4 +++ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 40f4d722dc6..1acedfb346a 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -468,7 +468,13 @@ async def _update_existing_team_model_assignment( user_api_key_dict: UserAPIKeyAuth, prisma_client: Optional[PrismaClient], ) -> None: - """Update an existing team model if the public name changed.""" + """Update an existing team model if the public name changed. + + Note on DB scan: Prisma's JSON filtering does not support compound AND conditions + across multiple JSON paths, so we fetch all deployments for the team and filter + team_public_model_name in Python. For teams with many deployments this scan grows + linearly; if team deployment counts become large this should be revisited. + """ def _get_team_public_model_name( model_info: Optional[Union[dict, str]] @@ -496,10 +502,6 @@ async def _update_existing_team_model_assignment( "prisma_client not initialized; skipping old public name cleanup to preserve sibling deployments" ) else: - # Query DB for all deployments in this team, then filter by public name. - # Note: Prisma's JSON filtering doesn't support compound AND conditions - # across multiple JSON paths, so we filter team_public_model_name in Python. - # For most teams (typically <100 deployments), this is acceptable. response = await prisma_client.db.litellm_proxymodeltable.find_many( where={ "model_info": { @@ -508,12 +510,15 @@ async def _update_existing_team_model_assignment( } } ) - other_deployments_with_old_name = [ - d - for d in response - if d.model_name != db_model.model_name - and _get_team_public_model_name(d.model_info) == old_public_name - ] + if not response: + other_deployments_with_old_name = [] + else: + other_deployments_with_old_name = [ + d + for d in response + if d.model_name != db_model.model_name + and _get_team_public_model_name(d.model_info) == old_public_name + ] if not other_deployments_with_old_name: await team_model_delete( diff --git a/litellm/router.py b/litellm/router.py index fdc23fb5dd4..e61e08ebc27 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8180,6 +8180,10 @@ class Router: # O(k) where k = team deployments for this model_name (typically 1-10) for idx in indices: model = self.model_list[idx] + if not self.should_include_deployment( + model_name=model_name, model=model, team_id=team_id + ): + continue if model_alias is not None: alias_model = model.copy() alias_model["model_name"] = model_alias From e015ffffc0a919644d496e44c8f3c4ccd743cfbb Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 19:37:05 +0530 Subject: [PATCH 15/22] Fix greptile comments --- litellm/router.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index e61e08ebc27..438f87eb297 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7105,8 +7105,8 @@ class Router: - model: dict - the deployment to index - idx: int - the index in model_list """ - team_id = model.get("model_info", {}).get("team_id") - team_public_model_name = model.get("model_info", {}).get( + team_id = (model.get("model_info") or {}).get("team_id") + team_public_model_name = (model.get("model_info") or {}).get( "team_public_model_name" ) if team_id and team_public_model_name: @@ -7164,7 +7164,10 @@ class Router: ) if _deployment_on_router is not None: # deployment with this model_id exists on the router - if deployment.litellm_params == _deployment_on_router.litellm_params: + if ( + deployment.litellm_params == _deployment_on_router.litellm_params + and deployment.model_info == _deployment_on_router.model_info + ): # No need to update return None @@ -8190,7 +8193,8 @@ class Router: returned_models.append(alias_model) else: returned_models.append(model) - return returned_models + if returned_models: + return returned_models # O(1) lookup in model_name index if model_name in self.model_name_to_deployment_indices: From 6fa99b06c72447d36606755484d722542bf1823b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 19:50:33 +0530 Subject: [PATCH 16/22] Fix greptile comments --- .../model_management_endpoints.py | 57 ++++++++++--------- litellm/router.py | 10 +++- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 1acedfb346a..d8a1075a168 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -499,36 +499,37 @@ async def _update_existing_team_model_assignment( if old_public_name and public_model_name != old_public_name: if prisma_client is None: verbose_proxy_logger.warning( - "prisma_client not initialized; skipping old public name cleanup to preserve sibling deployments" + "prisma_client not initialized; skipping public name update entirely to avoid orphaned entries" ) - else: - response = await prisma_client.db.litellm_proxymodeltable.find_many( - where={ - "model_info": { - "path": ["team_id"], - "equals": team_id, - } - } - ) - if not response: - other_deployments_with_old_name = [] - else: - other_deployments_with_old_name = [ - d - for d in response - if d.model_name != db_model.model_name - and _get_team_public_model_name(d.model_info) == old_public_name - ] + return - if not other_deployments_with_old_name: - await team_model_delete( - data=TeamModelDeleteRequest( - team_id=team_id, - models=[old_public_name], - ), - http_request=Request(scope={"type": "http"}), - user_api_key_dict=user_api_key_dict, - ) + response = await prisma_client.db.litellm_proxymodeltable.find_many( + where={ + "model_info": { + "path": ["team_id"], + "equals": team_id, + } + } + ) + if not response: + other_deployments_with_old_name = [] + else: + other_deployments_with_old_name = [ + d + for d in response + if d.model_name != db_model.model_name + and _get_team_public_model_name(d.model_info) == old_public_name + ] + + if not other_deployments_with_old_name: + await team_model_delete( + data=TeamModelDeleteRequest( + team_id=team_id, + models=[old_public_name], + ), + http_request=Request(scope={"type": "http"}), + user_api_key_dict=user_api_key_dict, + ) await team_model_add( data=TeamModelAddRequest( diff --git a/litellm/router.py b/litellm/router.py index 438f87eb297..7ee72745ccb 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8149,12 +8149,16 @@ class Router: """ if ( team_id is not None - and model["model_info"].get("team_id") == team_id - and model_name == model["model_info"].get("team_public_model_name") + and (model.get("model_info") or {}).get("team_id") == team_id + and model_name + == (model.get("model_info") or {}).get("team_public_model_name") ): return True elif model_name is not None and model["model_name"] == model_name: - if team_id is None or model["model_info"].get("team_id") == team_id: + if ( + team_id is None + or (model.get("model_info") or {}).get("team_id") == team_id + ): return True return False From dcdb07308c9cc1bd3e2afba93c226293bf8d5b2a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 20:12:06 +0530 Subject: [PATCH 17/22] Fix greptile comments --- .../model_management_endpoints.py | 19 +++++---- litellm/router.py | 4 +- .../test_model_management_endpoints.py | 41 +++++++++++++++++++ 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index d8a1075a168..721a8c2a1c4 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -521,6 +521,16 @@ async def _update_existing_team_model_assignment( and _get_team_public_model_name(d.model_info) == old_public_name ] + # Add new name first, then delete old name to prevent access loss on partial failure + await team_model_add( + data=TeamModelAddRequest( + team_id=team_id, + models=[public_model_name], + ), + http_request=Request(scope={"type": "http"}), + user_api_key_dict=user_api_key_dict, + ) + if not other_deployments_with_old_name: await team_model_delete( data=TeamModelDeleteRequest( @@ -531,15 +541,6 @@ async def _update_existing_team_model_assignment( user_api_key_dict=user_api_key_dict, ) - await team_model_add( - data=TeamModelAddRequest( - team_id=team_id, - models=[public_model_name], - ), - http_request=Request(scope={"type": "http"}), - user_api_key_dict=user_api_key_dict, - ) - patch_data.model_name = None diff --git a/litellm/router.py b/litellm/router.py index 7ee72745ccb..a257736a407 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8155,9 +8155,11 @@ class Router: ): return True elif model_name is not None and model["model_name"] == model_name: + model_team_id = (model.get("model_info") or {}).get("team_id") if ( team_id is None - or (model.get("model_info") or {}).get("team_id") == team_id + or model_team_id is None # global deployment - accessible to all teams + or model_team_id == team_id ): return True return False diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 83e6b0c93a5..2dd29fd5c9c 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -712,6 +712,15 @@ class TestTeamModelSiblingRouting: "team_public_model_name": public_name, }, }, + { + "model_name": "global-gpt-4o", + "litellm_params": { + "model": "azure/gpt-4o", + "api_key": "global-key", + "api_base": "https://global.openai.azure.com", + }, + "model_info": {}, # No team_id - global deployment + }, ], ) @@ -732,6 +741,38 @@ class TestTeamModelSiblingRouting: "https://westus.openai.azure.com", } + def test_global_deployments_accessible_to_teams(self): + """Test that global deployments (no team_id) are accessible to all teams""" + import litellm + + router = litellm.Router( + model_list=[ + { + "model_name": "global-gpt-4o", + "litellm_params": { + "model": "azure/gpt-4o", + "api_key": "global-key", + "api_base": "https://global.openai.azure.com", + }, + "model_info": {}, # No team_id - global deployment + }, + ], + ) + + # Global deployment should be accessible when team_id is provided + deployments = router._get_all_deployments( + model_name="global-gpt-4o", team_id="teamA" + ) + assert len(deployments) == 1 + assert deployments[0]["model_name"] == "global-gpt-4o" + + # should_include_deployment should return True for global deployments + assert router.should_include_deployment( + model_name="global-gpt-4o", + model={"model_name": "global-gpt-4o", "model_info": {}}, + team_id="teamA", + ) + class TestTeamModelUpdate: """Test team model update handles team_id consistently with model creation""" From b4463f0e543dfb2263f8cf38bd4895c4052f23b2 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 22:02:50 +0530 Subject: [PATCH 18/22] Fix code qa issues --- .../model_management_endpoints.py | 2 -- .../test_router_index_management.py | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 721a8c2a1c4..8f6b8a626e4 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -33,7 +33,6 @@ from litellm.proxy._types import ( ProxyException, TeamModelAddRequest, TeamModelDeleteRequest, - UpdateTeamRequest, UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -42,7 +41,6 @@ from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin from litellm.proxy.management_endpoints.team_endpoints import ( team_model_add, team_model_delete, - update_team, ) from litellm.proxy.management_helpers.audit_logs import create_object_audit_log from litellm.proxy.utils import PrismaClient diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py index 90d98b8ab0a..2694c62827c 100644 --- a/tests/router_unit_tests/test_router_index_management.py +++ b/tests/router_unit_tests/test_router_index_management.py @@ -118,6 +118,28 @@ class TestRouterIndexManagement: assert router.model_id_to_deployment_index_map["id-2"] == 1 assert router.model_id_to_deployment_index_map["id-3"] == 2 + def test_update_team_model_index(self, router): + """Test _update_team_model_index updates team_model_to_deployment_indices.""" + model = { + "model_name": "team-alias", + "model_info": { + "id": "dep-1", + "team_id": "team-abc", + "team_public_model_name": "gpt-4o", + }, + } + router._update_team_model_index(model, 0) + assert router.team_model_to_deployment_indices[("team-abc", "gpt-4o")] == [0] + router._update_team_model_index(model, 2) + assert router.team_model_to_deployment_indices[("team-abc", "gpt-4o")] == [0, 2] + + router._update_team_model_index( + {"model_name": "x", "model_info": {"id": "dep-2"}}, 5 + ) + assert router.team_model_to_deployment_indices == { + ("team-abc", "gpt-4o"): [0, 2], + } + def test_has_model_id(self, router): """Test has_model_id function for O(1) membership check""" # Setup: Add models to router From bee332f9f59e3f9c4563ba55920abdce497ecc57 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 22:26:08 +0530 Subject: [PATCH 19/22] Fix greptile reviews and mock test --- .../model_management_endpoints.py | 11 ++++ .../test_model_management_endpoints.py | 51 +++++++++++++++---- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 8f6b8a626e4..5952aede853 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -538,6 +538,17 @@ async def _update_existing_team_model_assignment( http_request=Request(scope={"type": "http"}), user_api_key_dict=user_api_key_dict, ) + elif not old_public_name and public_model_name: + # First-time assignment of public name on an existing team deployment: + # ensure the team's models list is updated so team routing can resolve it. + await team_model_add( + data=TeamModelAddRequest( + team_id=team_id, + models=[public_model_name], + ), + http_request=Request(scope={"type": "http"}), + user_api_key_dict=user_api_key_dict, + ) patch_data.model_name = None diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 2dd29fd5c9c..751d0a02ff0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -635,8 +635,6 @@ class TestTeamModelSiblingRouting: team_id = "team_no_alias" public_name = "gpt-4.1-mini" - mock_update_team = AsyncMock() - async def mock_add_model_to_db(model_params, user_api_key_dict, prisma_client): return MagicMock(model_id=str(uuid.uuid4())) @@ -656,9 +654,6 @@ class TestTeamModelSiblingRouting: model_info=ModelInfo(team_id=team_id), ) with patch( - "litellm.proxy.management_endpoints.model_management_endpoints.update_team", - mock_update_team, - ), patch( "litellm.proxy.management_endpoints.model_management_endpoints._add_model_to_db", side_effect=mock_add_model_to_db, ), patch( @@ -671,7 +666,6 @@ class TestTeamModelSiblingRouting: prisma_client=prisma_client, ) - mock_update_team.assert_not_called() assert mock_team_model_add.call_count == 2 @pytest.mark.asyncio @@ -807,8 +801,6 @@ class TestTeamModelUpdate: "litellm.proxy.proxy_server.premium_user", True, ), patch( - "litellm.proxy.management_endpoints.model_management_endpoints.update_team" - ) as mock_update_team, patch( "litellm.proxy.management_endpoints.model_management_endpoints.team_model_add" ) as mock_team_model_add: result = await _update_team_model_in_db( @@ -820,8 +812,6 @@ class TestTeamModelUpdate: assert result.get("model_name", "").startswith("model_name_test_team_123_") assert "team_public_model_name" in str(result.get("model_info", "")) - # update_team must not be called (no model_aliases writes for team models) - mock_update_team.assert_not_called() # team_model_add must be called to add public name to team's models list mock_team_model_add.assert_called_once() @@ -885,6 +875,47 @@ class TestTeamModelUpdate: # team_model_add should be called to add new public name mock_add.assert_called_once() + @pytest.mark.asyncio + async def test_first_time_public_name_assignment_adds_team_model(self): + """If existing team deployment had no public name, first assignment must call team_model_add.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _update_existing_team_model_assignment, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_team_123_uuid1", + litellm_params=LiteLLM_Params(model="azure/gpt-4o-mini"), + model_info=ModelInfo(team_id="team_123"), + ) + + patch_data = updateDeployment( + model_name="new-public-name", + model_info=ModelInfo(team_id="team_123"), + ) + + user_api_key_dict = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + with patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_delete" + ) as mock_delete, patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_add" + ) as mock_add: + await _update_existing_team_model_assignment( + team_id="team_123", + public_model_name="new-public-name", + db_model=db_model, + patch_data=patch_data, + user_api_key_dict=user_api_key_dict, + prisma_client=None, + ) + + mock_add.assert_called_once() + mock_delete.assert_not_called() + @pytest.mark.asyncio async def test_rename_handles_legacy_string_model_info(self): """Test rename path handles legacy string-encoded model_info rows without crashing.""" From 50c8f213bf2e746af5fdecd28e17cb695311d3bb Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 22:38:54 +0530 Subject: [PATCH 20/22] Fix greptile reviews and mock test --- docs/my-website/docs/proxy/config_settings.md | 1 + docs/my-website/docs/proxy/load_balancing.md | 15 ++++++++++++ litellm/proxy/litellm_pre_call_utils.py | 23 +++++++++++++++---- litellm/router.py | 6 +++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index a0e404e3a18..83730324357 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -801,6 +801,7 @@ router_settings: | LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL | LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL | LITELLM_ENABLE_PYROSCOPE | If true, enables Pyroscope CPU profiling. Profiles are sent to PYROSCOPE_SERVER_ADDRESS. Off by default. See [Pyroscope profiling](/proxy/pyroscope_profiling). +| LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS | When `true`, if a team's legacy `model_aliases` entry maps a public model name to an internal `model_name__` deployment, pre-call handling can skip that rewrite when team-scoped sibling deployments exist for the public nameβ€”so load balancing / `order` apply across siblings. Default is `false` for backwards compatibility. See [Team-scoped models and legacy aliases](./load_balancing#team-scoped-models-and-legacy-model_aliases). When stale aliases are detected and this flag is off, the proxy may log a one-time warning. | PYROSCOPE_APP_NAME | Application name reported to Pyroscope. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default. | PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used. diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 5bf39d179f6..66857defd43 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -336,6 +336,21 @@ The `order` parameter requires `enable_pre_call_checks: true` in `router_setting If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments. +### Team-scoped models and legacy `model_aliases` {#team-scoped-models-and-legacy-model_aliases} + +Team-scoped deployments are identified by `model_info.team_id` and `model_info.team_public_model_name`. Requests should use the **public** model name; the router resolves all sibling deployments (same public name, different `api_base` / `order`, etc.) for routing, failover, and deployment `order`. + +For router internals: when a `team_id` is in scope, optimized lookups key off `(team_id, team_public_model_name)`. If code passes an internal deployment id (e.g. `model_name__`) instead of the public name, routing still works via the usual deployment-name paths, but the team-specific fast path applies only to the public name. + +**Legacy teams:** Older proxy versions could persist `model_aliases` on the team row mapping a public name to a single internal deployment id (`model_name__`). On each request, pre-call logic may still rewrite `model` to that internal name **before** routing, which collapses to one deployment and can make newer sibling deployments unreachable. + +**Migration options:** + +1. **Recommended for upgrades:** Set environment variable `LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true` so that when sibling team deployments exist for the public name, the stale alias rewrite is skipped and team-scoped routing (including `order` and failover) applies. See the [Environment variables](./config_settings) table in the proxy settings doc. +2. **Data cleanup:** Remove obsolete `model_aliases` entries for team public names from the team record in the database so only `team_public_model_name` + team model list drive access. + +If a stale alias is detected and the bypass is **not** enabled, the proxy may emit a **one-time** warning in logs explaining that sibling deployments may be unreachable until the flag is set or aliases are cleaned up. + ### When You'll See Load Balancing in Action **Immediate Effects:** diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index fe4df6dcb13..98f8b1e8983 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -37,6 +37,7 @@ from litellm.types.utils import ( ) service_logger_obj = ServiceLogging() # used for tracking latency on OTEL +_STALE_TEAM_ALIAS_WARNING_KEYS: set[str] = set() if TYPE_CHECKING: @@ -1320,16 +1321,28 @@ def _update_model_if_team_alias_exists( ) # Check if the alias points to a team-scoped UUID name # (format: "model_name_{team_id}_{uuid}") - if enable_stale_alias_bypass and aliased_target.startswith( + is_stale_team_alias = aliased_target.startswith( f"model_name_{user_api_key_dict.team_id}_" - ): + ) + if is_stale_team_alias and llm_router: # This is a stale alias from pre-PR deployments. # Check if current team deployments exist for the public name. - if llm_router: - key = (user_api_key_dict.team_id, _model) - if key in llm_router.team_model_to_deployment_indices: + key = (user_api_key_dict.team_id, _model) + if key in llm_router.team_model_to_deployment_indices: + if enable_stale_alias_bypass: # Team deployments exist; skip stale alias return + warning_key = f"{user_api_key_dict.team_id}:{_model}:{aliased_target}" + if warning_key not in _STALE_TEAM_ALIAS_WARNING_KEYS: + _STALE_TEAM_ALIAS_WARNING_KEYS.add(warning_key) + verbose_proxy_logger.warning( + "Stale team model alias detected for model='%s', team_id='%s'. " + "New sibling deployments may be unreachable. " + "Set LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true to enable " + "team-scoped sibling routing.", + _model, + user_api_key_dict.team_id, + ) data["model"] = aliased_target return diff --git a/litellm/router.py b/litellm/router.py index a257736a407..71dfcecd56e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8178,6 +8178,12 @@ class Router: if team_id specified, only return team-specific models Optimized with O(1) index lookup instead of O(n) linear scan. + + Note: when team_id is provided, O(1) lookup in + `team_model_to_deployment_indices` only applies when `model_name` is the + team public model name. If a caller passes an internal deployment model + name (for example, `model_name__`), this method falls back + to the standard model-name index / scan path. """ returned_models: List[DeploymentTypedDict] = [] From 7bb7b2479e38b8af1d4a2fd61d5a62df2840fc46 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 22:49:57 +0530 Subject: [PATCH 21/22] Fix greptile reviews and mock test --- litellm/proxy/litellm_pre_call_utils.py | 12 +++++-- .../model_management_endpoints.py | 3 ++ .../test_model_management_endpoints.py | 35 +++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 98f8b1e8983..4623ca24f1b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1,6 +1,7 @@ import asyncio import copy import time +from collections import OrderedDict from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from fastapi import Request @@ -37,7 +38,9 @@ from litellm.types.utils import ( ) service_logger_obj = ServiceLogging() # used for tracking latency on OTEL -_STALE_TEAM_ALIAS_WARNING_KEYS: set[str] = set() +# Bounded dedup for stale-alias warnings (FIFO eviction when over cap). +_MAX_STALE_ALIAS_WARNING_KEYS = 10_000 +_STALE_TEAM_ALIAS_WARNING_KEYS: OrderedDict[str, None] = OrderedDict() if TYPE_CHECKING: @@ -1334,7 +1337,12 @@ def _update_model_if_team_alias_exists( return warning_key = f"{user_api_key_dict.team_id}:{_model}:{aliased_target}" if warning_key not in _STALE_TEAM_ALIAS_WARNING_KEYS: - _STALE_TEAM_ALIAS_WARNING_KEYS.add(warning_key) + _STALE_TEAM_ALIAS_WARNING_KEYS[warning_key] = None + while ( + len(_STALE_TEAM_ALIAS_WARNING_KEYS) + > _MAX_STALE_ALIAS_WARNING_KEYS + ): + _STALE_TEAM_ALIAS_WARNING_KEYS.popitem(last=False) verbose_proxy_logger.warning( "Stale team model alias detected for model='%s', team_id='%s'. " "New sibling deployments may be unreachable. " diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 5952aede853..95c44a431b5 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -495,6 +495,9 @@ async def _update_existing_team_model_assignment( ) if old_public_name and public_model_name != old_public_name: + # Clear user-supplied public name from patch before any early return so the + # caller does not overwrite the internal UUID-based model_name in the DB. + patch_data.model_name = None if prisma_client is None: verbose_proxy_logger.warning( "prisma_client not initialized; skipping public name update entirely to avoid orphaned entries" diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 751d0a02ff0..2a85e24d780 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -916,6 +916,41 @@ class TestTeamModelUpdate: mock_add.assert_called_once() mock_delete.assert_not_called() + @pytest.mark.asyncio + async def test_rename_with_prisma_none_clears_patch_model_name(self): + """Rename path must clear patch_data.model_name even when prisma is unavailable (P1).""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _update_existing_team_model_assignment, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_team_123_uuid1", + litellm_params=LiteLLM_Params(model="azure/gpt-4o-mini"), + model_info=ModelInfo( + team_id="team_123", team_public_model_name="old-public-name" + ), + ) + patch_data = updateDeployment( + model_name="new-public-name", + model_info=ModelInfo(team_id="team_123"), + ) + user_api_key_dict = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + await _update_existing_team_model_assignment( + team_id="team_123", + public_model_name="new-public-name", + db_model=db_model, + patch_data=patch_data, + user_api_key_dict=user_api_key_dict, + prisma_client=None, + ) + + assert patch_data.model_name is None + @pytest.mark.asyncio async def test_rename_handles_legacy_string_model_info(self): """Test rename path handles legacy string-encoded model_info rows without crashing.""" From 92c2beb07791ec7a7cb361f3bd7ba09f3ebdc4e3 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 9 Apr 2026 14:35:09 -0700 Subject: [PATCH 22/22] bump litellm-proxy-extras to 0.4.63 Restores MCP server columns dropped by the schema_sync migration shipped in 0.4.56 (source_url, approval_status, submitted_by, submitted_at, reviewed_at, review_notes) by pulling in the 20260319000000_restore_mcp_approval_fields migration. --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 07004f3ae16..2a2cdad4565 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ boto3 = { version = "^1.40.76", optional = true } redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "^0.4.56", optional = true} +litellm-proxy-extras = {version = "^0.4.63", optional = true} rich = {version = "^13.7.1", optional = true} litellm-enterprise = {version = "^0.1.33", optional = true} diskcache = {version = "^5.6.1", optional = true} diff --git a/requirements.txt b/requirements.txt index 2bdafda612a..edb40fcba30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ grpcio>=1.75.0; python_version >= "3.14" sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.56 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.63 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env