diff --git a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py index f19cb87ae18..37a893973e3 100644 --- a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py +++ b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py @@ -279,11 +279,10 @@ async def load_active_user_by_id( catches every DB failure and re-raises a bare ``ValueError`` (a deleted user and a real outage look identical, the original error surviving only as ``__context__``), so the outage check walks the cause chain, and a missing user falls through to ``no_active_key`` rather than an opaque gateway fault. - ``source="database"`` reads the row from the database, never the cache, and leaves the fresh row in the - cache for the requests the credential makes next: JWT auth caches the user it creates before it adds - that user to the JWT's team and adding a member never evicts the cached row, so a credential minted - off the cache would refuse the very first exchange as not a member. Every other caller keeps the cache - read, so introspection, which a resource server may call per request, stays off the database.""" + ``source="database"`` reads the row from the database, never the cache, so the credential mint refuses + a user that a writer deactivated or deleted without evicting the cached row, and it leaves the fresh + row in the cache for the requests the credential makes next. Every other caller keeps the cache read, + so introspection, which a resource server may call per request, stays off the database.""" from litellm.proxy._types import ( ProxyException, # noqa: PLC0415 # inline import avoids a module-load circular import ) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index e719d6d761a..84e3e9f87e6 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3154,6 +3154,7 @@ async def team_member_add( ``` """ + from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.proxy_server import ( litellm_proxy_admin_name, premium_user, @@ -3248,6 +3249,10 @@ async def team_member_add( litellm_proxy_admin_name=litellm_proxy_admin_name, ) + await evict_and_broadcast( + cache_keys=tuple(sorted(user.user_id for user in updated_users)), + user_api_key_cache=user_api_key_cache, + ) await _evict_created_membership_caches( user_ids=(tm.user_id for tm in updated_team_memberships), team_id=data.team_id, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_proxy_api_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_proxy_api_credentials.py index 04fbbe4a6ce..ed3e5f48516 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_proxy_api_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_proxy_api_credentials.py @@ -152,6 +152,28 @@ async def test_mint_reads_the_users_teams_from_the_database_not_a_stale_cached_r assert _decoded(minted).team_id == "team-a" +@pytest.mark.asyncio +async def test_mint_refuses_a_user_scim_deactivated_after_the_cache_last_saw_them_active(fetch_teams, monkeypatch): + """SCIM deactivation writes the user row without evicting the cached copy, so a mint off the cache would + keep issuing credentials for the management-object TTL. The mint reads the database row, so the + deactivated user is refused on the first refresh after the deactivation.""" + from litellm.proxy import proxy_server + + cache = UserApiKeyCache() + await cache.async_set_cache( + key="deactivated-user", value=_user(user_id="deactivated-user", teams=["team-a"]), model_type=LiteLLM_UserTable + ) + prisma = MagicMock() + prisma.db.litellm_usertable.find_unique = AsyncMock( + return_value=_user(user_id="deactivated-user", teams=["team-a"], metadata={"scim_active": False}) + ) + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + assert await mint_proxy_credential("deactivated-user", "team-a") == "no_active_key" + fetch_teams.assert_not_awaited() + + @pytest.mark.asyncio async def test_mint_refuses_a_team_the_user_is_not_on(load_user, fetch_teams): assert await mint_proxy_credential("u1", "team-c") == "not_a_member" diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index ebbedc6541e..748cdbbc175 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -13090,6 +13090,77 @@ async def test_team_member_add_audits_a_user_created_from_a_list_payload(monkeyp assert created_user_id not in mock_audit.call_args.kwargs["existing_user_ids"] +@pytest.mark.asyncio +async def test_team_member_add_evicts_the_new_members_cached_user_row_on_every_worker(monkeypatch): + """Auth admits a team-bound credential off the teams list of the cached user row. The add wrote the + new team to the database row only, so a worker still holding the old row refused the member's + credential with 403 until the management-object TTL expired. The add now evicts the row here and + broadcasts the eviction to the other workers, the way /team/member_delete already does.""" + from litellm.proxy._types import TeamMemberAddRequest + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.management_endpoints.team_endpoints import team_member_add + + team_id = "team-b" + user_id = "dev-1" + cache = UserApiKeyCache() + await cache.async_set_cache( + key=user_id, value=LiteLLM_UserTable(user_id=user_id, teams=["team-a"]), model_type=LiteLLM_UserTable + ) + broadcast = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", AsyncMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "default_user_id") + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + monkeypatch.setattr( + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.publish_auth_cache_invalidation", broadcast + ) + + updated_team = MagicMock() + updated_team.model_dump.return_value = { + "team_id": team_id, + "members_with_roles": [{"user_id": user_id, "role": "user"}], + } + + async def fake_add_team_members_to_team(**kwargs): + return updated_team, [LiteLLM_UserTable(user_id=user_id, teams=["team-a", team_id])], [] + + with ( + patch( # test-quality-ok: team_member_add has no injection seam for its prisma-backed helpers + "litellm.proxy.management_endpoints.team_endpoints.get_team_object", + new_callable=AsyncMock, + return_value=LiteLLM_TeamTable(team_id=team_id, members_with_roles=[]), + ), + patch( # test-quality-ok: team_member_add has no injection seam for its prisma-backed helpers + "litellm.proxy.management_endpoints.team_endpoints._validate_team_member_add_permissions", + new_callable=AsyncMock, + ), + patch( # test-quality-ok: team_member_add has no injection seam for its prisma-backed helpers + "litellm.proxy.management_endpoints.team_endpoints._validate_and_populate_member_user_info", + new_callable=AsyncMock, + ), + patch( # test-quality-ok: team_member_add has no injection seam for its prisma-backed helpers + "litellm.proxy.management_endpoints.team_endpoints._resolve_existing_member_user_ids", + new_callable=AsyncMock, + return_value=frozenset({user_id}), + ), + patch( # test-quality-ok: team_member_add has no injection seam for its prisma-backed helpers + "litellm.proxy.management_endpoints.team_endpoints._add_team_members_to_team", + side_effect=fake_add_team_members_to_team, + ), + patch( # test-quality-ok: team_member_add has no injection seam for its prisma-backed helpers + "litellm.proxy.management_endpoints.team_endpoints._create_team_member_add_audit_logs", + new_callable=AsyncMock, + ), + ): + await team_member_add( + data=TeamMemberAddRequest(team_id=team_id, member=Member(user_id=user_id, role="user")), + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1"), + ) + + assert await cache.async_get_cache(key=user_id, model_type=LiteLLM_UserTable) is None + broadcast.assert_awaited_once_with(cache_key=user_id) + + def test_validate_member_user_id_provisioning_caps_the_ids_it_echoes_back(): """A large member list must not echo every id back in the error body.""" from litellm.proxy.management_endpoints.team_endpoints import (