From 2bbc941cbdb1e16795932fd678d19ae8dfa2ae61 Mon Sep 17 00:00:00 2001 From: lewis Date: Sat, 10 Oct 2026 08:28:35 -0400 Subject: [PATCH 01/10] fix(proxy): persist customer block and unblock state --- .../enterprise_hooks/blocked_user_list.py | 7 +- .../proxy/common_utils/user_api_key_cache.py | 4 + .../customer_endpoints.py | 79 ++-- .../test_customer_endpoints.py | 388 +++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 5 files changed, 432 insertions(+), 50 deletions(-) diff --git a/enterprise/enterprise_hooks/blocked_user_list.py b/enterprise/enterprise_hooks/blocked_user_list.py index f6998d58cd5..25a027b899c 100644 --- a/enterprise/enterprise_hooks/blocked_user_list.py +++ b/enterprise/enterprise_hooks/blocked_user_list.py @@ -7,7 +7,7 @@ ## This accepts a list of user id's for whom calls will be rejected -from typing import Literal, Optional +from typing import Final, Literal, Optional from fastapi import HTTPException @@ -17,12 +17,13 @@ from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth -from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET +from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, end_user_block_cache_key from litellm.proxy.utils import PrismaClient class ENTERPRISE_BlockedUserList(CustomLogger): enforces_request_content: bool = True + blocked_user_list: list[str] | None # Class variables or attributes def __init__(self, prisma_client: Optional[PrismaClient]): self.prisma_client = prisma_client @@ -88,7 +89,7 @@ class ENTERPRISE_BlockedUserList(CustomLogger): }, ) - cache_key = f"litellm:end_user_id:{user}" + cache_key: Final = end_user_block_cache_key(f"{user}") end_user_cache_obj: Optional[LiteLLM_EndUserTable] = cache.get_cache( # type: ignore key=cache_key ) diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index 8c830816207..76ad9e9d17b 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -372,6 +372,10 @@ def end_user_cache_key(end_user_id: str) -> str: return f"end_user_id:{end_user_id}" +def end_user_block_cache_key(end_user_id: str) -> str: + return f"litellm:end_user_id:{end_user_id}" + + def end_user_restricted_registry_cache_key() -> str: """Cache key for the set of end-user ids whose row carries a restriction auth enforces.""" return "end_user_restricted_registry" diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index d604c58d1f5..a86c792ff6e 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.user_api_key_cache import ( + end_user_block_cache_key, end_user_cache_key, end_user_restricted_registry_cache_key, ) @@ -58,6 +59,7 @@ from litellm.types.proxy.management_endpoints.customer_endpoints import ( _RowT_co: Final = TypeVar("_RowT_co", covariant=True) _STR_OBJECT_DICT: Final = TypeAdapter(dict[str, object]) +_STRING_IDS: Final = TypeAdapter(tuple[str, ...]) _CLEARABLE_LIST_FIELDS: Final = frozenset({"models"}) @@ -105,6 +107,8 @@ if TYPE_CHECKING: data: Mapping[str, Mapping[str, object]], ) -> _RowT_co: ... + async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ... + async def delete_many(self, where: Mapping[str, object]) -> int: ... @@ -135,7 +139,11 @@ async def _evict_end_user_cache_keys(cache_keys: Sequence[str]) -> None: def _end_user_cache_keys(user_ids: Sequence[str]) -> tuple[str, ...]: """The per-id entries plus the registry, which any restriction change can move ids in or out of.""" - return (*(end_user_cache_key(user_id) for user_id in user_ids), end_user_restricted_registry_cache_key()) + return ( + *(end_user_cache_key(user_id) for user_id in user_ids), + end_user_restricted_registry_cache_key(), + *(end_user_block_cache_key(user_id) for user_id in user_ids), + ) def _to_customer_response(record: BaseModel) -> CustomerResponse: @@ -160,7 +168,7 @@ def _to_customer_response(record: BaseModel) -> CustomerResponse: dependencies=[Depends(user_api_key_auth)], response_model=BlockUsersResponse, ) -async def block_user(data: BlockUsers): +async def block_user(data: BlockUsers) -> BlockUsersResponse: """ [BETA] Reject calls with this end-user id @@ -180,25 +188,26 @@ async def block_user(data: BlockUsers): from litellm.proxy.proxy_server import prisma_client try: - records: Final = [] - if prisma_client is not None: - for id in data.user_ids: - record = await _typed_table(EndUserRepository(prisma_client)).upsert( - where={"user_id": id}, - data={ - "create": {"user_id": id, "blocked": True}, - "update": {"blocked": True}, - }, - ) - records.append(record) - await _evict_end_user_cache_keys(_end_user_cache_keys(data.user_ids)) - else: + if prisma_client is None: raise HTTPException( status_code=500, detail={"error": "Postgres DB Not connected"}, ) - - return {"blocked_users": records} + table: Final = _typed_table(EndUserRepository(prisma_client)) + records: Final = [ + _to_customer_response( + await table.upsert( + where={"user_id": user_id}, + data={ + "create": {"user_id": user_id, "blocked": True}, + "update": {"blocked": True}, + }, + ) + ) + for user_id in data.user_ids + ] + await _evict_end_user_cache_keys(_end_user_cache_keys(data.user_ids)) + return BlockUsersResponse.model_validate({"blocked_users": records}) except Exception as e: verbose_proxy_logger.error("An error occurred - %s", e) raise HTTPException(status_code=500, detail={"error": str(e)}) @@ -216,7 +225,7 @@ async def block_user(data: BlockUsers): dependencies=[Depends(user_api_key_auth)], response_model=UnblockUsersResponse, ) -async def unblock_user(data: BlockUsers): +async def unblock_user(data: BlockUsers) -> UnblockUsersResponse: """ [BETA] Unblock calls with this user id @@ -229,10 +238,10 @@ async def unblock_user(data: BlockUsers): }' ``` """ + from litellm.proxy.proxy_server import prisma_client + try: - from enterprise.enterprise_hooks.blocked_user_list import ( - ENTERPRISE_BlockedUserList, - ) + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList except ImportError: raise HTTPException( status_code=400, @@ -241,26 +250,34 @@ async def unblock_user(data: BlockUsers): + CommonProxyErrors.missing_enterprise_package_docker.value }, ) - - if ( - not any(isinstance(x, ENTERPRISE_BlockedUserList) for x in litellm.callbacks) - or litellm.blocked_user_list is None - ): + configured_hook: Final = next( + (callback for callback in litellm.callbacks if isinstance(callback, ENTERPRISE_BlockedUserList)), None + ) + if configured_hook is None: raise HTTPException( status_code=400, detail={"error": "Blocked user check was never set. This call has no effect."}, ) + requested_ids: Final = frozenset(data.user_ids) + configured_ids: Final = _STRING_IDS.validate_python( + configured_hook.blocked_user_list + if isinstance(litellm.blocked_user_list, str) + else litellm.blocked_user_list or () + ) + remaining_legacy_ids: Final = tuple(user_id for user_id in configured_ids if user_id not in requested_ids) + table: Final = _typed_table(EndUserRepository(prisma_client)) if prisma_client is not None else None + if table is not None: + await table.update_many(where={"user_id": {"in": data.user_ids}}, data={"blocked": False}) if isinstance(litellm.blocked_user_list, list): - for id in data.user_ids: - litellm.blocked_user_list.remove(id) - else: + litellm.blocked_user_list[:] = remaining_legacy_ids + await _evict_end_user_cache_keys(_end_user_cache_keys(data.user_ids)) + if isinstance(litellm.blocked_user_list, str) and requested_ids.intersection(configured_ids): raise HTTPException( status_code=500, detail={"error": "`blocked_user_list` must be set as a list. Filepaths can't be updated."}, ) - - return {"blocked_users": litellm.blocked_user_list} + return UnblockUsersResponse(blocked_users=list(remaining_legacy_ids)) def new_budget_request(data: NewCustomerRequest) -> BudgetNewRequest | None: diff --git a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py index c99db2d8054..ee8d404c8f5 100644 --- a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py @@ -1,4 +1,7 @@ from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -6,14 +9,22 @@ from fastapi import FastAPI, HTTPException, Request, status from fastapi.responses import JSONResponse from fastapi.routing import APIRoute from fastapi.testclient import TestClient +from prisma.models import LiteLLM_EndUserTable as PrismaEndUserRow from litellm.proxy._types import ( + BlockUsers, LiteLLM_EndUserTable, LitellmUserRoles, ProxyException, ) from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth -from litellm.proxy.management_endpoints.customer_endpoints import _should_update_field, router +from litellm.proxy.common_utils.user_api_key_cache import ( + UserApiKeyCache, + end_user_block_cache_key, + end_user_cache_key, + end_user_restricted_registry_cache_key, +) +from litellm.proxy.management_endpoints.customer_endpoints import _should_update_field, block_user, router, unblock_user from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, ) @@ -546,25 +557,27 @@ def test_update_customer_response_keeps_nested_budget_server_fields(mock_prisma_ assert "updated_by" not in budget -def test_block_customer_success_serializes_through_response_model(mock_prisma_client, mock_user_api_key_auth): +def test_block_customer_success_serializes_through_response_model( + mock_prisma_client: MagicMock, mock_user_api_key_auth: object +) -> None: """ /customer/block returns {"blocked_users": []}. With response_model=BlockUsersResponse, a shape mismatch would raise a 500 ResponseValidationError, so a clean 200 proves the model matches runtime output. """ - blocked_row = LiteLLM_EndUserTable(user_id="blocked-1", blocked=True) + blocked_row: Final = PrismaEndUserRow(user_id="blocked-1", blocked=True, spend=0, models=[]) mock_prisma_client.db.litellm_endusertable.upsert = AsyncMock(return_value=blocked_row) - response = client.post( + response: Final = client.post( "/customer/block", json={"user_ids": ["blocked-1"]}, headers={"Authorization": "Bearer test-key"}, ) assert response.status_code == 200 - body = response.json() - assert body["blocked_users"][0]["user_id"] == "blocked-1" - assert body["blocked_users"][0]["blocked"] is True + body: Final = BlockUsersResponse.model_validate(response.json()) + assert body.blocked_users[0].user_id == blocked_row.user_id + assert body.blocked_users[0].blocked is True def test_delete_customer_success_serializes_through_response_model(mock_prisma_client, mock_user_api_key_auth): @@ -915,9 +928,7 @@ def test_customer_new_forwards_models_to_db(mock_prisma_client, mock_user_api_ke @pytest.mark.parametrize("bad_duration", ["0s", "-5m"]) -def test_customer_new_rejects_a_duration_that_never_advances( - mock_prisma_client, mock_user_api_key_auth, bad_duration -): +def test_customer_new_rejects_a_duration_that_never_advances(mock_prisma_client, mock_user_api_key_auth, bad_duration): """A zero-length window resets to "now", leaving the customer's budget row permanently due for the reset job to re-read every tick.""" mock_prisma_client.db.litellm_endusertable.create = AsyncMock(return_value=_row(_FULL_DB_ROW)) @@ -1073,8 +1084,8 @@ def test_customer_new_invalidates_end_user_and_registry_caches(mock_prisma_clien ) assert response.status_code == 200, response.text - assert recording_cache.deleted == ["end_user_id:c1", "end_user_restricted_registry"] - assert _published_keys(mock_publish) == ["end_user_id:c1", "end_user_restricted_registry"] + assert recording_cache.deleted == ["end_user_id:c1", "end_user_restricted_registry", "litellm:end_user_id:c1"] + assert _published_keys(mock_publish) == ["end_user_id:c1", "end_user_restricted_registry", "litellm:end_user_id:c1"] def test_customer_update_invalidates_end_user_and_registry_caches(mock_prisma_client, mock_user_api_key_auth): @@ -1092,8 +1103,8 @@ def test_customer_update_invalidates_end_user_and_registry_caches(mock_prisma_cl ) assert response.status_code == 200, response.text - assert recording_cache.deleted == ["end_user_id:c1", "end_user_restricted_registry"] - assert _published_keys(mock_publish) == ["end_user_id:c1", "end_user_restricted_registry"] + assert recording_cache.deleted == ["end_user_id:c1", "end_user_restricted_registry", "litellm:end_user_id:c1"] + assert _published_keys(mock_publish) == ["end_user_id:c1", "end_user_restricted_registry", "litellm:end_user_id:c1"] def test_customer_block_invalidates_end_user_and_registry_caches(mock_prisma_client, mock_user_api_key_auth): @@ -1114,14 +1125,359 @@ def test_customer_block_invalidates_end_user_and_registry_caches(mock_prisma_cli "end_user_id:c1", "end_user_id:c2", "end_user_restricted_registry", + "litellm:end_user_id:c1", + "litellm:end_user_id:c2", ] assert _published_keys(mock_publish) == [ "end_user_id:c1", "end_user_id:c2", "end_user_restricted_registry", + "litellm:end_user_id:c1", + "litellm:end_user_id:c2", ] +def test_customer_block_legacy_route_matches_canonical_response( + mock_prisma_client: MagicMock, mock_user_api_key_auth: object +) -> None: + blocked_row: Final = PrismaEndUserRow( + user_id="legacy-customer", blocked=True, alias="Legacy customer", spend=0, models=[] + ) + upsert: Final = AsyncMock(return_value=blocked_row) + mock_prisma_client.db = SimpleNamespace(litellm_endusertable=SimpleNamespace(upsert=upsert)) + cache: Final = UserApiKeyCache() + with ( + patch("litellm.proxy.proxy_server.user_api_key_cache", cache), + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + ): + canonical: Final = client.post( + "/customer/block", json={"user_ids": [blocked_row.user_id]}, headers={"Authorization": "Bearer k"} + ) + legacy: Final = client.post( + "/end_user/block", json={"user_ids": [blocked_row.user_id]}, headers={"Authorization": "Bearer k"} + ) + assert canonical.status_code == legacy.status_code == 200 + assert legacy.json() == canonical.json() + body: Final = BlockUsersResponse.model_validate(legacy.json()) + assert len(body.blocked_users) == 1 + assert body.blocked_users[0].user_id == blocked_row.user_id + assert body.blocked_users[0].blocked is True + assert upsert.await_count == 2 + + +@pytest.mark.parametrize("legacy_ids", (None, (), ("legacy-customer", "still-blocked"))) +def test_customer_unblock_legacy_route_matches_canonical_response( + mock_prisma_client: MagicMock, mock_user_api_key_auth: object, legacy_ids: tuple[str, ...] | None +) -> None: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + + update_many: Final = AsyncMock(return_value=1) + mock_prisma_client.db = SimpleNamespace(litellm_endusertable=SimpleNamespace(update_many=update_many)) + configured_ids: Final = list(legacy_ids) if legacy_ids is not None else None + cache: Final = UserApiKeyCache() + with ( + patch("litellm.blocked_user_list", configured_ids), + patch("litellm.proxy.proxy_server.user_api_key_cache", cache), + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + ): + hook: Final = ENTERPRISE_BlockedUserList(prisma_client=mock_prisma_client) + with patch("litellm.callbacks", [hook]): + canonical: Final = client.post( + "/customer/unblock", json={"user_ids": ["legacy-customer"]}, headers={"Authorization": "Bearer k"} + ) + legacy: Final = client.post( + "/end_user/unblock", json={"user_ids": ["legacy-customer"]}, headers={"Authorization": "Bearer k"} + ) + assert canonical.status_code == legacy.status_code == 200 + assert legacy.json() == canonical.json() + body: Final = UnblockUsersResponse.model_validate(legacy.json()) + assert body.blocked_users == (["still-blocked"] if legacy_ids else []) + update_many.assert_awaited_with(where={"user_id": {"in": ["legacy-customer"]}}, data={"blocked": False}) + assert update_many.await_count == 2 + + +@pytest.mark.parametrize("legacy_ids", ((), ("c1", "legacy-only", "c2"))) +def test_customer_unblock_persists_database_and_evicts_both_auth_caches( + mock_prisma_client: MagicMock, + mock_user_api_key_auth: object, + legacy_ids: tuple[str, ...], +) -> None: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + + update_many: Final = AsyncMock(return_value=2) + find_many: Final = AsyncMock(side_effect=AssertionError("Unblock must not read unrelated database customers")) + mock_prisma_client.db = SimpleNamespace( + litellm_endusertable=SimpleNamespace(update_many=update_many, find_many=find_many) + ) + configured_ids: Final = list(legacy_ids) + with ( + patch("litellm.blocked_user_list", configured_ids), + _end_user_cache_doubles() as (recording_cache, mock_publish), + ): + legacy_hook: Final = ENTERPRISE_BlockedUserList(prisma_client=None) + with patch("litellm.callbacks", [legacy_hook]): + response: Final = client.post( + "/customer/unblock", + json={"user_ids": ["c1", "c2"]}, + headers={"Authorization": "Bearer k"}, + ) + + assert response.status_code == 200, response.text + update_many.assert_awaited_once_with(where={"user_id": {"in": ["c1", "c2"]}}, data={"blocked": False}) + body: Final = UnblockUsersResponse.model_validate(response.json()) + assert body.blocked_users == (["legacy-only"] if legacy_ids else []) + assert legacy_hook.blocked_user_list == (["legacy-only"] if legacy_ids else []) + expected_keys: Final = ( + "end_user_id:c1", + "end_user_id:c2", + "end_user_restricted_registry", + "litellm:end_user_id:c1", + "litellm:end_user_id:c2", + ) + assert tuple(recording_cache.deleted) == expected_keys + assert tuple(_published_keys(mock_publish)) == expected_keys + + +@pytest.mark.asyncio +async def test_customer_block_unblock_refreshes_enterprise_hook_cache(mock_prisma_client: MagicMock) -> None: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + + active_row: Final = PrismaEndUserRow(user_id="cached-customer", blocked=False, spend=0, models=[]) + blocked_row: Final = active_row.model_copy(update={"blocked": True}) + read_customer: Final = AsyncMock(side_effect=(active_row, blocked_row, active_row)) + mock_prisma_client.db = SimpleNamespace( + litellm_endusertable=SimpleNamespace( + find_unique=read_customer, + upsert=AsyncMock(return_value=blocked_row), + update_many=AsyncMock(return_value=1), + ) + ) + cache: Final = UserApiKeyCache() + customer_ids: Final = BlockUsers(user_ids=[active_row.user_id]) + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + with ( + patch("litellm.blocked_user_list", []), + patch("litellm.proxy.proxy_server.user_api_key_cache", cache), + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + ): + hook: Final = ENTERPRISE_BlockedUserList(prisma_client=mock_prisma_client) + await hook.async_pre_call_hook(auth, cache, {"user": active_row.user_id}, "completion") + cached_row: Final = cache.get_cache(end_user_block_cache_key(active_row.user_id), model_type=PrismaEndUserRow) + assert cached_row is not None and cached_row.blocked is False + + await block_user(customer_ids) + with pytest.raises(HTTPException, match="User blocked from making LLM API Calls"): + await hook.async_pre_call_hook(auth, cache, {"user": active_row.user_id}, "completion") + + with patch("litellm.callbacks", [hook]): + await unblock_user(customer_ids) + await hook.async_pre_call_hook(auth, cache, {"user": active_row.user_id}, "completion") + + assert read_customer.await_count == 3 + + +@pytest.mark.parametrize( + "operation,initially_blocked,finally_blocked", + ( + ("new", False, True), + ("update", False, True), + ("update", True, False), + ("delete", True, False), + ), +) +@pytest.mark.asyncio +async def test_customer_mutations_refresh_warm_enterprise_hook_cache( + mock_prisma_client: MagicMock, + mock_user_api_key_auth: object, + operation: str, + initially_blocked: bool, + finally_blocked: bool, +) -> None: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + + before_row: Final = PrismaEndUserRow(user_id="mutated-customer", blocked=initially_blocked, spend=0, models=[]) + after_row: Final = before_row.model_copy(update={"blocked": finally_blocked}) + read_customer: Final = AsyncMock( + side_effect=(None if operation == "new" else before_row, None if operation == "delete" else after_row) + ) + mock_prisma_client.db = SimpleNamespace( + litellm_endusertable=SimpleNamespace( + find_unique=read_customer, + find_first=AsyncMock(return_value=before_row), + find_many=AsyncMock(return_value=[before_row]), + create=AsyncMock(return_value=after_row), + update=AsyncMock(return_value=after_row), + delete_many=AsyncMock(return_value=1), + ) + ) + cache: Final = UserApiKeyCache() + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + with ( + patch("litellm.blocked_user_list", []), + patch("litellm.proxy.proxy_server.user_api_key_cache", cache), + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + ): + hook: Final = ENTERPRISE_BlockedUserList(prisma_client=mock_prisma_client) + if initially_blocked: + with pytest.raises(HTTPException, match="User blocked from making LLM API Calls"): + await hook.async_pre_call_hook(auth, cache, {"user": before_row.user_id}, "completion") + else: + await hook.async_pre_call_hook(auth, cache, {"user": before_row.user_id}, "completion") + cached_before: Final = cache.get_cache( + end_user_block_cache_key(before_row.user_id), model_type=PrismaEndUserRow + ) + assert cached_before is not None and cached_before.blocked is initially_blocked + + payload: Final = ( + {"user_ids": [before_row.user_id]} + if operation == "delete" + else {"user_id": before_row.user_id, "blocked": finally_blocked} + ) + response: Final = client.post(f"/customer/{operation}", json=payload, headers={"Authorization": "Bearer k"}) + assert response.status_code == 200, response.text + if finally_blocked: + with pytest.raises(HTTPException, match="User blocked from making LLM API Calls"): + await hook.async_pre_call_hook(auth, cache, {"user": before_row.user_id}, "completion") + else: + await hook.async_pre_call_hook(auth, cache, {"user": before_row.user_id}, "completion") + assert read_customer.await_count == 2 + + +@pytest.mark.parametrize("configured_hook,configured_ids", ((False, ()), (False, ("c1",)), (False, None), (True, None))) +def test_customer_unblock_requires_loaded_hook( + mock_prisma_client: MagicMock, + mock_user_api_key_auth: object, + configured_hook: bool, + configured_ids: tuple[str, ...] | None, +) -> None: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + + update_many: Final = AsyncMock(return_value=1) + mock_prisma_client.db = SimpleNamespace(litellm_endusertable=SimpleNamespace(update_many=update_many)) + blocked_row: Final = PrismaEndUserRow(user_id="c1", blocked=True, spend=0, models=[]) + cache: Final = UserApiKeyCache() + cache.set_cache(end_user_block_cache_key(blocked_row.user_id), blocked_row) + cache.set_cache(end_user_cache_key(blocked_row.user_id), blocked_row) + cache.set_cache(end_user_restricted_registry_cache_key(), [blocked_row.user_id]) + with ( + patch("litellm.blocked_user_list", list(configured_ids) if configured_ids is not None else None), + patch("litellm.proxy.proxy_server.user_api_key_cache", cache), + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + ): + hook: Final = ENTERPRISE_BlockedUserList(prisma_client=mock_prisma_client) + with patch("litellm.callbacks", [hook] if configured_hook else []): + response: Final = client.post( + "/customer/unblock", + json={"user_ids": [blocked_row.user_id]}, + headers={"Authorization": "Bearer k"}, + ) + + cached_row: Final = cache.get_cache(end_user_block_cache_key(blocked_row.user_id), model_type=PrismaEndUserRow) + if configured_hook: + assert response.status_code == 200, response.text + assert response.json() == {"blocked_users": []} + update_many.assert_awaited_once_with(where={"user_id": {"in": [blocked_row.user_id]}}, data={"blocked": False}) + assert cached_row is None + assert cache.get_cache(end_user_cache_key(blocked_row.user_id), model_type=PrismaEndUserRow) is None + assert cache.get_cache(end_user_restricted_registry_cache_key()) is None + else: + assert response.status_code == 400 + assert response.json() == {"detail": {"error": "Blocked user check was never set. This call has no effect."}} + update_many.assert_not_awaited() + assert cached_row is not None and cached_row.blocked is True + cached_customer: Final = cache.get_cache(end_user_cache_key(blocked_row.user_id), model_type=PrismaEndUserRow) + assert cached_customer is not None and cached_customer.blocked is True + assert cache.get_cache(end_user_restricted_registry_cache_key()) == [blocked_row.user_id] + + +@pytest.mark.asyncio +async def test_customer_unblock_database_only_hook_refreshes_cached_block(mock_prisma_client: MagicMock) -> None: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + + blocked_row: Final = PrismaEndUserRow(user_id="database-only-customer", blocked=True, spend=0, models=[]) + active_row: Final = blocked_row.model_copy(update={"blocked": False}) + read_customer: Final = AsyncMock(side_effect=(blocked_row, active_row)) + update_many: Final = AsyncMock(return_value=1) + mock_prisma_client.db = SimpleNamespace( + litellm_endusertable=SimpleNamespace(find_unique=read_customer, update_many=update_many) + ) + cache: Final = UserApiKeyCache() + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + cache.set_cache(end_user_cache_key(blocked_row.user_id), blocked_row) + cache.set_cache(end_user_restricted_registry_cache_key(), [blocked_row.user_id]) + with ( + patch("litellm.blocked_user_list", None), + patch("litellm.proxy.proxy_server.user_api_key_cache", cache), + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + ): + hook: Final = ENTERPRISE_BlockedUserList(prisma_client=mock_prisma_client) + with pytest.raises(HTTPException, match="User blocked from making LLM API Calls"): + await hook.async_pre_call_hook(auth, cache, {"user": blocked_row.user_id}, "completion") + cached_before: Final = cache.get_cache( + end_user_block_cache_key(blocked_row.user_id), model_type=PrismaEndUserRow + ) + assert cached_before is not None and cached_before.blocked is True + with patch("litellm.callbacks", [hook]): + response: Final = await unblock_user(BlockUsers(user_ids=[blocked_row.user_id])) + assert response.blocked_users == [] + update_many.assert_awaited_once_with(where={"user_id": {"in": [blocked_row.user_id]}}, data={"blocked": False}) + assert cache.get_cache(end_user_cache_key(blocked_row.user_id), model_type=PrismaEndUserRow) is None + assert cache.get_cache(end_user_restricted_registry_cache_key()) is None + assert cache.get_cache(end_user_block_cache_key(blocked_row.user_id), model_type=PrismaEndUserRow) is None + await hook.async_pre_call_hook(auth, cache, {"user": blocked_row.user_id}, "completion") + cached_after: Final = cache.get_cache( + end_user_block_cache_key(blocked_row.user_id), model_type=PrismaEndUserRow + ) + assert cached_after is not None and cached_after.blocked is False + assert read_customer.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("in_file", (False, True)) +async def test_customer_unblock_file_list_clears_database_and_keeps_file( + mock_prisma_client: MagicMock, tmp_path: Path, in_file: bool +) -> None: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + + user_id: Final = "file-customer" + file_contents: Final = f"{user_id}\nlegacy-only" if in_file else "legacy-only" + block_file: Final = tmp_path / "blocked-users.txt" + block_file.write_text(file_contents) + blocked_row: Final = PrismaEndUserRow(user_id=user_id, blocked=True, spend=0, models=[]) + active_row: Final = blocked_row.model_copy(update={"blocked": False}) + update_many: Final = AsyncMock(return_value=1) + mock_prisma_client.db = SimpleNamespace( + litellm_endusertable=SimpleNamespace(update_many=update_many, find_unique=AsyncMock(return_value=active_row)) + ) + cache: Final = UserApiKeyCache() + cache.set_cache(end_user_block_cache_key(user_id), blocked_row) + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + with ( + patch("litellm.blocked_user_list", str(block_file)), + patch("litellm.proxy.proxy_server.user_api_key_cache", cache), + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + ): + hook: Final = ENTERPRISE_BlockedUserList(prisma_client=mock_prisma_client) + with patch("litellm.callbacks", [hook]): + if in_file: + with pytest.raises(HTTPException, match="Filepaths can't be updated") as error: + await unblock_user(BlockUsers(user_ids=[user_id])) + assert error.value.status_code == 500 + else: + response: Final = await unblock_user(BlockUsers(user_ids=[user_id])) + assert response.blocked_users == ["legacy-only"] + + assert cache.get_cache(end_user_block_cache_key(user_id), model_type=PrismaEndUserRow) is None + if in_file: + with pytest.raises(HTTPException, match="User blocked from making LLM API Calls"): + await hook.async_pre_call_hook(auth, cache, {"user": user_id}, "completion") + else: + await hook.async_pre_call_hook(auth, cache, {"user": user_id}, "completion") + + update_many.assert_awaited_once_with(where={"user_id": {"in": [user_id]}}, data={"blocked": False}) + assert block_file.read_text() == file_contents + + def test_customer_delete_invalidates_end_user_and_registry_caches(mock_prisma_client, mock_user_api_key_auth): """Without this a deleted customer keeps its cached budget and block enforced until the TTL expires.""" mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock( @@ -1144,9 +1500,13 @@ def test_customer_delete_invalidates_end_user_and_registry_caches(mock_prisma_cl "end_user_id:c1", "end_user_id:c2", "end_user_restricted_registry", + "litellm:end_user_id:c1", + "litellm:end_user_id:c2", ] assert _published_keys(mock_publish) == [ "end_user_id:c1", "end_user_id:c2", "end_user_restricted_registry", + "litellm:end_user_id:c1", + "litellm:end_user_id:c2", ] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 53ecf9b2d75..f1c9bc6a8b3 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -56686,7 +56686,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["BlockUsersResponse"]; }; }; /** @description Validation Error */ @@ -57071,7 +57071,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["UnblockUsersResponse"]; }; }; /** @description Validation Error */ From 51dfceddf92200e75bfca48913b43fc22ad0e410 Mon Sep 17 00:00:00 2001 From: lewis Date: Wed, 14 Oct 2026 18:59:57 -0400 Subject: [PATCH 02/10] fix(proxy): validate customer block-state changes before saving --- litellm/proxy/_types.py | 2 +- .../customer_endpoints.py | 71 +++++++----- .../test_customer_endpoints.py | 103 ++++++++++++++++-- 3 files changed, 138 insertions(+), 38 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 477edf47c5f..fb35c24da38 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1259,7 +1259,7 @@ class ProviderInfo(LiteLLMPydanticObjectBase): class BlockUsers(LiteLLMPydanticObjectBase): - user_ids: list[str] # required + user_ids: list[str] = Field(min_length=1, max_length=1000) class ModelParams(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index a86c792ff6e..dacde45239f 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -24,6 +24,7 @@ if TYPE_CHECKING: from prisma.models import LiteLLM_BudgetTable as PrismaBudgetRow from prisma.models import LiteLLM_EndUserTable as PrismaEndUserRow + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList from litellm.proxy.utils import PrismaClient import litellm @@ -45,7 +46,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( # noqa: ) from litellm.proxy.utils import handle_exception_on_proxy from litellm.repositories.budget_repository import BudgetRepository -from litellm.repositories.chunked_in import find_many_in +from litellm.repositories.chunked_in import find_many_in, update_many_in from litellm.repositories.table_repositories import EndUserRepository from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, @@ -156,6 +157,29 @@ def _to_customer_response(record: BaseModel) -> CustomerResponse: return CustomerResponse.model_validate(record.model_dump()) +def _configured_blocked_user_hook() -> "ENTERPRISE_BlockedUserList": + try: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + except ImportError: + raise HTTPException( + status_code=400, + detail={ + "error": "Blocked user check was never set. This call has no effect." + + CommonProxyErrors.missing_enterprise_package_docker.value + }, + ) + configured_hook: Final = next( + (callback for callback in litellm.callbacks if isinstance(callback, ENTERPRISE_BlockedUserList)), None + ) + if configured_hook is None: + raise HTTPException( + status_code=400, + detail={"error": "Blocked user check was never set. This call has no effect."}, + ) + + return configured_hook + + @router.post( "/end_user/block", tags=["Customer Management"], @@ -187,6 +211,7 @@ async def block_user(data: BlockUsers) -> BlockUsersResponse: """ from litellm.proxy.proxy_server import prisma_client + _configured_blocked_user_hook() try: if prisma_client is None: raise HTTPException( @@ -240,43 +265,33 @@ async def unblock_user(data: BlockUsers) -> UnblockUsersResponse: """ from litellm.proxy.proxy_server import prisma_client - try: - from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList - except ImportError: + configured_hook: Final = _configured_blocked_user_hook() + + requested_ids: Final = frozenset(data.user_ids) + file_ids: Final = ( + _STRING_IDS.validate_python(configured_hook.blocked_user_list or ()) + if isinstance(litellm.blocked_user_list, str) + else () + ) + if requested_ids.intersection(file_ids): raise HTTPException( status_code=400, detail={ - "error": "Blocked user check was never set. This call has no effect." - + CommonProxyErrors.missing_enterprise_package_docker.value + "error": "These customers are blocked by a file. Update the file and restart the proxy to unblock them." }, ) - configured_hook: Final = next( - (callback for callback in litellm.callbacks if isinstance(callback, ENTERPRISE_BlockedUserList)), None - ) - if configured_hook is None: - raise HTTPException( - status_code=400, - detail={"error": "Blocked user check was never set. This call has no effect."}, - ) - - requested_ids: Final = frozenset(data.user_ids) - configured_ids: Final = _STRING_IDS.validate_python( - configured_hook.blocked_user_list - if isinstance(litellm.blocked_user_list, str) - else litellm.blocked_user_list or () - ) - remaining_legacy_ids: Final = tuple(user_id for user_id in configured_ids if user_id not in requested_ids) table: Final = _typed_table(EndUserRepository(prisma_client)) if prisma_client is not None else None if table is not None: - await table.update_many(where={"user_id": {"in": data.user_ids}}, data={"blocked": False}) + await update_many_in(table, "user_id", data.user_ids, data={"blocked": False}, atomicity="per_chunk_ok") + configured_ids: Final = ( + file_ids + if isinstance(litellm.blocked_user_list, str) + else _STRING_IDS.validate_python(litellm.blocked_user_list or ()) + ) + remaining_legacy_ids: Final = tuple(user_id for user_id in configured_ids if user_id not in requested_ids) if isinstance(litellm.blocked_user_list, list): litellm.blocked_user_list[:] = remaining_legacy_ids await _evict_end_user_cache_keys(_end_user_cache_keys(data.user_ids)) - if isinstance(litellm.blocked_user_list, str) and requested_ids.intersection(configured_ids): - raise HTTPException( - status_code=500, - detail={"error": "`blocked_user_list` must be set as a list. Filepaths can't be updated."}, - ) return UnblockUsersResponse(blocked_users=list(remaining_legacy_ids)) diff --git a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py index ee8d404c8f5..a4b33081c39 100644 --- a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py @@ -1,3 +1,5 @@ +import asyncio +from collections.abc import Iterator, Mapping from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace @@ -11,6 +13,7 @@ from fastapi.routing import APIRoute from fastapi.testclient import TestClient from prisma.models import LiteLLM_EndUserTable as PrismaEndUserRow +import litellm from litellm.proxy._types import ( BlockUsers, LiteLLM_EndUserTable, @@ -101,6 +104,16 @@ def mock_prisma_client(): yield mock +@pytest.fixture +def configured_block_hook(mock_prisma_client: MagicMock) -> Iterator[None]: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + + with patch("litellm.blocked_user_list", []): + hook: Final = ENTERPRISE_BlockedUserList(prisma_client=mock_prisma_client) + with patch("litellm.callbacks", [hook]): + yield + + @pytest.fixture def mock_user_api_key_auth(): original_overrides = app.dependency_overrides.copy() @@ -558,7 +571,7 @@ def test_update_customer_response_keeps_nested_budget_server_fields(mock_prisma_ def test_block_customer_success_serializes_through_response_model( - mock_prisma_client: MagicMock, mock_user_api_key_auth: object + mock_prisma_client: MagicMock, mock_user_api_key_auth: object, configured_block_hook: None ) -> None: """ /customer/block returns {"blocked_users": []}. With @@ -1107,7 +1120,9 @@ def test_customer_update_invalidates_end_user_and_registry_caches(mock_prisma_cl assert _published_keys(mock_publish) == ["end_user_id:c1", "end_user_restricted_registry", "litellm:end_user_id:c1"] -def test_customer_block_invalidates_end_user_and_registry_caches(mock_prisma_client, mock_user_api_key_auth): +def test_customer_block_invalidates_end_user_and_registry_caches( + mock_prisma_client, mock_user_api_key_auth, configured_block_hook: None +): """Blocking is the one mutation that must take effect instantly; a stale registry keeps serving it.""" mock_prisma_client.db.litellm_endusertable.upsert = AsyncMock( return_value=LiteLLM_EndUserTable(user_id="c1", blocked=True) @@ -1138,7 +1153,7 @@ def test_customer_block_invalidates_end_user_and_registry_caches(mock_prisma_cli def test_customer_block_legacy_route_matches_canonical_response( - mock_prisma_client: MagicMock, mock_user_api_key_auth: object + mock_prisma_client: MagicMock, mock_user_api_key_auth: object, configured_block_hook: None ) -> None: blocked_row: Final = PrismaEndUserRow( user_id="legacy-customer", blocked=True, alias="Legacy customer", spend=0, models=[] @@ -1265,7 +1280,8 @@ async def test_customer_block_unblock_refreshes_enterprise_hook_cache(mock_prism cached_row: Final = cache.get_cache(end_user_block_cache_key(active_row.user_id), model_type=PrismaEndUserRow) assert cached_row is not None and cached_row.blocked is False - await block_user(customer_ids) + with patch("litellm.callbacks", [hook]): + await block_user(customer_ids) with pytest.raises(HTTPException, match="User blocked from making LLM API Calls"): await hook.async_pre_call_hook(auth, cache, {"user": active_row.user_id}, "completion") @@ -1434,7 +1450,7 @@ async def test_customer_unblock_database_only_hook_refreshes_cached_block(mock_p @pytest.mark.asyncio @pytest.mark.parametrize("in_file", (False, True)) -async def test_customer_unblock_file_list_clears_database_and_keeps_file( +async def test_customer_unblock_file_list_validates_before_database_write( mock_prisma_client: MagicMock, tmp_path: Path, in_file: bool ) -> None: from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList @@ -1460,24 +1476,93 @@ async def test_customer_unblock_file_list_clears_database_and_keeps_file( hook: Final = ENTERPRISE_BlockedUserList(prisma_client=mock_prisma_client) with patch("litellm.callbacks", [hook]): if in_file: - with pytest.raises(HTTPException, match="Filepaths can't be updated") as error: + with pytest.raises(HTTPException, match="Update the file and restart the proxy") as error: await unblock_user(BlockUsers(user_ids=[user_id])) - assert error.value.status_code == 500 + assert error.value.status_code == 400 else: response: Final = await unblock_user(BlockUsers(user_ids=[user_id])) assert response.blocked_users == ["legacy-only"] - assert cache.get_cache(end_user_block_cache_key(user_id), model_type=PrismaEndUserRow) is None + cached_row: Final = cache.get_cache(end_user_block_cache_key(user_id), model_type=PrismaEndUserRow) if in_file: + assert cached_row is not None and cached_row.blocked is True with pytest.raises(HTTPException, match="User blocked from making LLM API Calls"): await hook.async_pre_call_hook(auth, cache, {"user": user_id}, "completion") else: + assert cached_row is None await hook.async_pre_call_hook(auth, cache, {"user": user_id}, "completion") - update_many.assert_awaited_once_with(where={"user_id": {"in": [user_id]}}, data={"blocked": False}) + if in_file: + update_many.assert_not_awaited() + else: + update_many.assert_awaited_once_with(where={"user_id": {"in": [user_id]}}, data={"blocked": False}) assert block_file.read_text() == file_contents +@pytest.mark.parametrize("route", ("/customer/block", "/end_user/block")) +def test_customer_block_requires_loaded_hook_before_writing( + mock_prisma_client: MagicMock, mock_user_api_key_auth: object, route: str +) -> None: + upsert: Final = AsyncMock() + mock_prisma_client.db = SimpleNamespace(litellm_endusertable=SimpleNamespace(upsert=upsert)) + with patch("litellm.callbacks", []), _end_user_cache_doubles() as (cache, publish): + response: Final = client.post(route, json={"user_ids": ["c1"]}, headers={"Authorization": "Bearer k"}) + assert response.status_code == 400 + assert response.json() == {"detail": {"error": "Blocked user check was never set. This call has no effect."}} + upsert.assert_not_awaited() + assert cache.deleted == [] + publish.assert_not_awaited() + + +@pytest.mark.parametrize("route", ("/customer/block", "/end_user/block", "/customer/unblock", "/end_user/unblock")) +@pytest.mark.parametrize("size", (0, 1001)) +def test_customer_block_requests_reject_empty_or_oversized_batches( + mock_prisma_client: MagicMock, mock_user_api_key_auth: object, route: str, size: int +) -> None: + upsert: Final = AsyncMock() + update_many: Final = AsyncMock() + mock_prisma_client.db = SimpleNamespace( + litellm_endusertable=SimpleNamespace(upsert=upsert, update_many=update_many) + ) + response: Final = client.post( + route, json={"user_ids": [f"c{index}" for index in range(size)]}, headers={"Authorization": "Bearer k"} + ) + assert response.status_code == 422 + upsert.assert_not_awaited() + update_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_concurrent_customer_unblocks_keep_both_ids_unblocked(mock_prisma_client: MagicMock) -> None: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + + first_write_started: Final = asyncio.Event() + release_first_write: Final = asyncio.Event() + + async def write_block_state(where: Mapping[str, object], data: Mapping[str, object]) -> int: + assert data == {"blocked": False} + if where == {"user_id": {"in": ["c1"]}}: + first_write_started.set() + await release_first_write.wait() + return 1 + + mock_prisma_client.db = SimpleNamespace( + litellm_endusertable=SimpleNamespace(update_many=AsyncMock(side_effect=write_block_state)) + ) + with patch("litellm.blocked_user_list", ["c1", "c2", "still-blocked"]), _end_user_cache_doubles(): + hook: Final = ENTERPRISE_BlockedUserList(prisma_client=mock_prisma_client) + with patch("litellm.callbacks", [hook]): + first_request: Final = asyncio.create_task(unblock_user(BlockUsers(user_ids=["c1"]))) + await first_write_started.wait() + second_response: Final = await unblock_user(BlockUsers(user_ids=["c2"])) + assert second_response.blocked_users == ["c1", "still-blocked"] + release_first_write.set() + first_response: Final = await first_request + assert first_response.blocked_users == ["still-blocked"] + assert hook.blocked_user_list == ["still-blocked"] + assert litellm.blocked_user_list == ["still-blocked"] + + def test_customer_delete_invalidates_end_user_and_registry_caches(mock_prisma_client, mock_user_api_key_auth): """Without this a deleted customer keeps its cached budget and block enforced until the TTL expires.""" mock_prisma_client.db.litellm_endusertable.find_many = AsyncMock( From ebb18bb246f2348df0c72f450d139d012305c19d Mon Sep 17 00:00:00 2001 From: lewis Date: Thu, 15 Oct 2026 12:05:31 -0400 Subject: [PATCH 03/10] test(proxy): cover customer block without enterprise package or database --- .../test_customer_endpoints.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py index a4b33081c39..5d864bb9f13 100644 --- a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py @@ -1,4 +1,5 @@ import asyncio +import sys from collections.abc import Iterator, Mapping from contextlib import contextmanager from pathlib import Path @@ -16,6 +17,7 @@ from prisma.models import LiteLLM_EndUserTable as PrismaEndUserRow import litellm from litellm.proxy._types import ( BlockUsers, + CommonProxyErrors, LiteLLM_EndUserTable, LitellmUserRoles, ProxyException, @@ -1514,6 +1516,43 @@ def test_customer_block_requires_loaded_hook_before_writing( publish.assert_not_awaited() +@pytest.mark.parametrize("route", ("/customer/block", "/customer/unblock")) +def test_customer_block_without_enterprise_package_rejects_before_writing( + mock_prisma_client: MagicMock, mock_user_api_key_auth: object, route: str +) -> None: + upsert: Final = AsyncMock() + update_many: Final = AsyncMock() + mock_prisma_client.db = SimpleNamespace( + litellm_endusertable=SimpleNamespace(upsert=upsert, update_many=update_many) + ) + with ( + patch.dict(sys.modules, {"enterprise.enterprise_hooks.blocked_user_list": None}), + _end_user_cache_doubles() as (cache, publish), + ): + response: Final = client.post(route, json={"user_ids": ["c1"]}, headers={"Authorization": "Bearer k"}) + assert response.status_code == 400 + assert response.json() == { + "detail": { + "error": "Blocked user check was never set. This call has no effect." + + CommonProxyErrors.missing_enterprise_package_docker.value + } + } + upsert.assert_not_awaited() + update_many.assert_not_awaited() + assert cache.deleted == [] + publish.assert_not_awaited() + + +@pytest.mark.usefixtures("configured_block_hook") +def test_customer_block_without_database_fails_without_evicting_caches(mock_user_api_key_auth: object) -> None: + with patch("litellm.proxy.proxy_server.prisma_client", None), _end_user_cache_doubles() as (cache, publish): + response: Final = client.post("/customer/block", json={"user_ids": ["c1"]}, headers={"Authorization": "Bearer k"}) + assert response.status_code == 500 + assert "Postgres DB Not connected" in response.text + assert cache.deleted == [] + publish.assert_not_awaited() + + @pytest.mark.parametrize("route", ("/customer/block", "/end_user/block", "/customer/unblock", "/end_user/unblock")) @pytest.mark.parametrize("size", (0, 1001)) def test_customer_block_requests_reject_empty_or_oversized_batches( From 5e8ac300b28df2cc326a1f1516123b12cc7a2cb8 Mon Sep 17 00:00:00 2001 From: lewis Date: Thu, 15 Oct 2026 22:07:30 -0400 Subject: [PATCH 04/10] fix(proxy): enforce customer blocks when the hook loads before the database --- .../enterprise_hooks/blocked_user_list.py | 12 ++++++++-- .../test_customer_endpoints.py | 22 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/enterprise/enterprise_hooks/blocked_user_list.py b/enterprise/enterprise_hooks/blocked_user_list.py index 25a027b899c..339223d9b39 100644 --- a/enterprise/enterprise_hooks/blocked_user_list.py +++ b/enterprise/enterprise_hooks/blocked_user_list.py @@ -59,6 +59,13 @@ class ENTERPRISE_BlockedUserList(CustomLogger): if litellm.set_verbose is True: print(print_statement) # noqa + def _database(self) -> Optional[PrismaClient]: + if self.prisma_client is not None: + return self.prisma_client + from litellm.proxy.proxy_server import prisma_client + + return prisma_client + @with_service_target(AUTH_OBJECTS_TARGET) async def async_pre_call_hook( self, @@ -93,10 +100,11 @@ class ENTERPRISE_BlockedUserList(CustomLogger): end_user_cache_obj: Optional[LiteLLM_EndUserTable] = cache.get_cache( # type: ignore key=cache_key ) - if end_user_cache_obj is None and self.prisma_client is not None: + database: Final = self._database() + if end_user_cache_obj is None and database is not None: # check db end_user_obj = ( - await self.prisma_client.db.litellm_endusertable.find_unique( + await database.db.litellm_endusertable.find_unique( where={"user_id": user} ) ) diff --git a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py index 5d864bb9f13..7d7b2c2fd29 100644 --- a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py @@ -1408,6 +1408,28 @@ def test_customer_unblock_requires_loaded_hook( assert cache.get_cache(end_user_restricted_registry_cache_key()) == [blocked_row.user_id] +@pytest.mark.asyncio +async def test_blocked_user_hook_built_before_database_connects_rejects_blocked_customer( + mock_prisma_client: MagicMock, +) -> None: + from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList + + blocked_row: Final = PrismaEndUserRow(user_id="config-loaded-customer", blocked=True, spend=0, models=[]) + read_customer: Final = AsyncMock(return_value=blocked_row) + mock_prisma_client.db = SimpleNamespace(litellm_endusertable=SimpleNamespace(find_unique=read_customer)) + with patch("litellm.blocked_user_list", None): + hook: Final = ENTERPRISE_BlockedUserList(prisma_client=None) + with pytest.raises(HTTPException, match="User blocked from making LLM API Calls") as error: + await hook.async_pre_call_hook( + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + UserApiKeyCache(), + {"user": blocked_row.user_id}, + "completion", + ) + assert error.value.status_code == 400 + read_customer.assert_awaited_once_with(where={"user_id": blocked_row.user_id}) + + @pytest.mark.asyncio async def test_customer_unblock_database_only_hook_refreshes_cached_block(mock_prisma_client: MagicMock) -> None: from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList From 90a6cff00dc12877566a551e4091843efae73642 Mon Sep 17 00:00:00 2001 From: lewis Date: Fri, 16 Oct 2026 07:19:14 -0400 Subject: [PATCH 05/10] refactor(proxy): read unblock legacy ids from the loaded hook --- .../customer_endpoints.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index dacde45239f..d27f3c95c26 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -60,7 +60,6 @@ from litellm.types.proxy.management_endpoints.customer_endpoints import ( _RowT_co: Final = TypeVar("_RowT_co", covariant=True) _STR_OBJECT_DICT: Final = TypeAdapter(dict[str, object]) -_STRING_IDS: Final = TypeAdapter(tuple[str, ...]) _CLEARABLE_LIST_FIELDS: Final = frozenset({"models"}) @@ -268,27 +267,26 @@ async def unblock_user(data: BlockUsers) -> UnblockUsersResponse: configured_hook: Final = _configured_blocked_user_hook() requested_ids: Final = frozenset(data.user_ids) - file_ids: Final = ( - _STRING_IDS.validate_python(configured_hook.blocked_user_list or ()) - if isinstance(litellm.blocked_user_list, str) - else () - ) - if requested_ids.intersection(file_ids): + if isinstance(litellm.blocked_user_list, str) and requested_ids.intersection( + configured_hook.blocked_user_list or () + ): raise HTTPException( status_code=400, detail={ "error": "These customers are blocked by a file. Update the file and restart the proxy to unblock them." }, ) - table: Final = _typed_table(EndUserRepository(prisma_client)) if prisma_client is not None else None - if table is not None: - await update_many_in(table, "user_id", data.user_ids, data={"blocked": False}, atomicity="per_chunk_ok") - configured_ids: Final = ( - file_ids - if isinstance(litellm.blocked_user_list, str) - else _STRING_IDS.validate_python(litellm.blocked_user_list or ()) + if prisma_client is not None: + await update_many_in( + _typed_table(EndUserRepository(prisma_client)), + "user_id", + data.user_ids, + data={"blocked": False}, + atomicity="per_chunk_ok", + ) + remaining_legacy_ids: Final = tuple( + user_id for user_id in configured_hook.blocked_user_list or () if user_id not in requested_ids ) - remaining_legacy_ids: Final = tuple(user_id for user_id in configured_ids if user_id not in requested_ids) if isinstance(litellm.blocked_user_list, list): litellm.blocked_user_list[:] = remaining_legacy_ids await _evict_end_user_cache_keys(_end_user_cache_keys(data.user_ids)) From d8da29fb1194a5a7d3f98e4562419a1bb31df6ba Mon Sep 17 00:00:00 2001 From: lewis Date: Fri, 16 Oct 2026 08:48:40 -0400 Subject: [PATCH 06/10] fix(proxy): enforce cached customer blocks --- .../enterprise_hooks/blocked_user_list.py | 5 +++-- .../test_customer_endpoints.py | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/enterprise/enterprise_hooks/blocked_user_list.py b/enterprise/enterprise_hooks/blocked_user_list.py index 339223d9b39..c1c57705095 100644 --- a/enterprise/enterprise_hooks/blocked_user_list.py +++ b/enterprise/enterprise_hooks/blocked_user_list.py @@ -17,6 +17,7 @@ from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth +from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, end_user_block_cache_key from litellm.proxy.utils import PrismaClient @@ -97,8 +98,8 @@ class ENTERPRISE_BlockedUserList(CustomLogger): ) cache_key: Final = end_user_block_cache_key(f"{user}") - end_user_cache_obj: Optional[LiteLLM_EndUserTable] = cache.get_cache( # type: ignore - key=cache_key + end_user_cache_obj: Final = CacheCodec.deserialize( + cache.get_cache(key=cache_key), model_type=LiteLLM_EndUserTable ) database: Final = self._database() if end_user_cache_obj is None and database is not None: diff --git a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py index 826c7831e03..33bf627a772 100644 --- a/tests/unit/proxy/management_endpoints/test_customer_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_customer_endpoints.py @@ -1420,14 +1420,16 @@ async def test_blocked_user_hook_built_before_database_connects_rejects_blocked_ mock_prisma_client.db = SimpleNamespace(litellm_endusertable=SimpleNamespace(find_unique=read_customer)) with patch("litellm.blocked_user_list", None): hook: Final = ENTERPRISE_BlockedUserList(prisma_client=None) - with pytest.raises(HTTPException, match="User blocked from making LLM API Calls") as error: - await hook.async_pre_call_hook( - UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), - UserApiKeyCache(), - {"user": blocked_row.user_id}, - "completion", - ) - assert error.value.status_code == 400 + cache: Final = UserApiKeyCache() + for _ in range(2): + with pytest.raises(HTTPException, match="User blocked from making LLM API Calls") as error: + await hook.async_pre_call_hook( + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + cache, + {"user": blocked_row.user_id}, + "completion", + ) + assert error.value.status_code == 400 read_customer.assert_awaited_once_with(where={"user_id": blocked_row.user_id}) From 689b7169bf432632862da20e4bde5b5f3a878fe4 Mon Sep 17 00:00:00 2001 From: lewis Date: Fri, 16 Oct 2026 21:49:43 -0400 Subject: [PATCH 07/10] fix(proxy): type the blocked user hook without an enterprise import --- litellm/proxy/management_endpoints/customer_endpoints.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index d27f3c95c26..df64246238e 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -24,7 +24,6 @@ if TYPE_CHECKING: from prisma.models import LiteLLM_BudgetTable as PrismaBudgetRow from prisma.models import LiteLLM_EndUserTable as PrismaEndUserRow - from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList from litellm.proxy.utils import PrismaClient import litellm @@ -156,7 +155,11 @@ def _to_customer_response(record: BaseModel) -> CustomerResponse: return CustomerResponse.model_validate(record.model_dump()) -def _configured_blocked_user_hook() -> "ENTERPRISE_BlockedUserList": +class _BlockedUserHook(Protocol): + blocked_user_list: list[str] | None + + +def _configured_blocked_user_hook() -> _BlockedUserHook: try: from enterprise.enterprise_hooks.blocked_user_list import ENTERPRISE_BlockedUserList except ImportError: From 6f9d717227e7d3589ed25bc1eba7a2cddefc676a Mon Sep 17 00:00:00 2001 From: lewis Date: Fri, 16 Oct 2026 22:10:42 -0400 Subject: [PATCH 08/10] fix(proxy): read the blocked user hook list through a read-only protocol --- litellm/proxy/management_endpoints/customer_endpoints.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index df64246238e..6e613208a48 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -156,7 +156,8 @@ def _to_customer_response(record: BaseModel) -> CustomerResponse: class _BlockedUserHook(Protocol): - blocked_user_list: list[str] | None + @property + def blocked_user_list(self) -> Sequence[str] | None: ... def _configured_blocked_user_hook() -> _BlockedUserHook: From d7a4a2746ac979197a8ae187d20175ac782b88bc Mon Sep 17 00:00:00 2001 From: lewis Date: Fri, 16 Oct 2026 22:26:29 -0400 Subject: [PATCH 09/10] test(integration): run customer block cases through a proxy with the blocked user check --- .../authorization/test_end_user_block.py | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/tests/integration/authorization/test_end_user_block.py b/tests/integration/authorization/test_end_user_block.py index 8b18bddfd00..2817d1ceae8 100644 --- a/tests/integration/authorization/test_end_user_block.py +++ b/tests/integration/authorization/test_end_user_block.py @@ -133,12 +133,13 @@ def test_block_route_marks_the_customer_blocked_and_update_unblocks(gateway: Gat ) -def _assert_unseen_customer_is_blocked(gateway: Gateway, block_route: str) -> None: - with gateway.scenario() as scenario: +def _assert_unseen_customer_is_blocked(gateway: Gateway, tmp_path: Path, block_route: str) -> None: + with ExitStack() as stack: + proxy: Final = _one_callback_proxy(gateway, tmp_path, stack) end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" - scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + stack.callback(proxy.request, "POST", "/customer/delete", {"user_ids": [end_user]}) - blocked: Final = gateway.request("POST", block_route, {"user_ids": [end_user]}) + blocked: Final = proxy.request("POST", block_route, {"user_ids": [end_user]}) assert blocked.status_code == 200, blocked.text blocked_response: Final = _BlockUsersResponse.model_validate_json(blocked.content) assert blocked_response == _BlockUsersResponse( @@ -147,26 +148,26 @@ def _assert_unseen_customer_is_blocked(gateway: Gateway, block_route: str) -> No assert eventually(lambda: _blocked_row(end_user), lambda value: value == [{"blocked": True}], seconds=15) == [ {"blocked": True} ] - info: Final = gateway.request("GET", "/customer/info", params={"end_user_id": end_user}) + info: Final = proxy.request("GET", "/customer/info", params={"end_user_id": end_user}) assert info.status_code == 200, info.text assert _BlockedCustomerInfo.model_validate(info.json()).blocked is True - updated: Final = gateway.request("POST", "/customer/update", {"user_id": end_user, "blocked": False}) + updated: Final = proxy.request("POST", "/customer/update", {"user_id": end_user, "blocked": False}) assert updated.status_code == 200, updated.text assert eventually(lambda: _blocked_row(end_user), lambda value: value == [{"blocked": False}], seconds=15) == [ {"blocked": False} ] - info_after_update: Final = gateway.request("GET", "/customer/info", params={"end_user_id": end_user}) + info_after_update: Final = proxy.request("GET", "/customer/info", params={"end_user_id": end_user}) assert info_after_update.status_code == 200, info_after_update.text assert _BlockedCustomerInfo.model_validate(info_after_update.json()).blocked is False -def test_end_user_block_route_creates_and_blocks_an_unseen_customer(gateway: Gateway) -> None: - _assert_unseen_customer_is_blocked(gateway, "/end_user/block") +def test_end_user_block_route_creates_and_blocks_an_unseen_customer(gateway: Gateway, tmp_path: Path) -> None: + _assert_unseen_customer_is_blocked(gateway, tmp_path, "/end_user/block") -def test_customer_block_route_creates_and_blocks_an_unseen_customer(gateway: Gateway) -> None: - _assert_unseen_customer_is_blocked(gateway, "/customer/block") +def test_customer_block_route_creates_and_blocks_an_unseen_customer(gateway: Gateway, tmp_path: Path) -> None: + _assert_unseen_customer_is_blocked(gateway, tmp_path, "/customer/block") def test_listed_end_user_is_refused_and_unblock_restores_serving(gateway: Gateway, tmp_path: Path) -> None: @@ -206,7 +207,7 @@ def test_listed_end_user_is_refused_and_unblock_restores_serving(gateway: Gatewa def test_blocked_end_user_is_refused_on_every_proxy_and_unblock_restores( gateway: Gateway, tmp_path: Path, block_route: str ) -> None: - pytest.skip("BUG: blocked end-user chat requests return HTTP 200 instead of HTTP 400") + pytest.skip("BUG: another proxy keeps a customer's previous block state for up to 60 seconds after unblock") with ( ExitStack() as stack, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, @@ -286,7 +287,7 @@ def test_blocked_end_user_is_refused_on_every_proxy_and_unblock_restores( assert _observed(upstream) == expected_observations -def test_unblock_route_requires_the_blocked_user_check_callback(gateway: Gateway, tmp_path: Path) -> None: +def test_block_and_unblock_require_the_blocked_user_check_callback(gateway: Gateway, tmp_path: Path) -> None: with ExitStack() as stack: callback_proxy: Final = _one_callback_proxy(gateway, tmp_path, stack) end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" @@ -304,21 +305,26 @@ def test_unblock_route_requires_the_blocked_user_check_callback(gateway: Gateway ) assert _blocked_row(end_user) == [{"blocked": True}] + unseen_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + shared_block: Final = gateway.request("POST", "/customer/block", {"user_ids": [unseen_user]}) + assert shared_block.status_code == 400, shared_block.text + assert shared_block.json() == shared.json(), shared_block.text + assert _blocked_row(unseen_user) == [] + callback_response: Final = callback_proxy.request("POST", "/customer/unblock", {"user_ids": [end_user]}) - assert callback_response.status_code == 400, callback_response.text - assert callback_response.json() == { - "detail": {"error": "Blocked user check was never set. This call has no effect."} - }, callback_response.text - assert _blocked_row(end_user) == [{"blocked": True}] + assert callback_response.status_code == 200, callback_response.text + assert callback_response.json() == {"blocked_users": []}, callback_response.text + assert _blocked_row(end_user) == [{"blocked": False}] -def test_block_route_returns_the_blocked_records(gateway: Gateway) -> None: +def test_block_route_returns_the_blocked_records(gateway: Gateway, tmp_path: Path) -> None: end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" - with gateway.scenario() as scenario: - created: Final = gateway.request("POST", "/customer/new", {"user_id": end_user}) + with ExitStack() as stack: + proxy: Final = _one_callback_proxy(gateway, tmp_path, stack) + created: Final = proxy.request("POST", "/customer/new", {"user_id": end_user}) assert created.status_code == 200, created.text - scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) - blocked: Final = gateway.request("POST", "/customer/block", {"user_ids": [end_user]}) + stack.callback(proxy.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + blocked: Final = proxy.request("POST", "/customer/block", {"user_ids": [end_user]}) assert blocked.status_code == 200, blocked.text payload: Final = object_value(blocked.json()) blocked_users: Final = payload["blocked_users"] From eb24414619453208d6573a84d845d73ef04d3369 Mon Sep 17 00:00:00 2001 From: lewis Date: Sat, 17 Oct 2026 22:14:29 -0400 Subject: [PATCH 10/10] test(proxy): configure enforcement for customer block persistence checks --- .../authorization/test_end_user_block.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/integration/authorization/test_end_user_block.py b/tests/integration/authorization/test_end_user_block.py index 2817d1ceae8..b36315a6c83 100644 --- a/tests/integration/authorization/test_end_user_block.py +++ b/tests/integration/authorization/test_end_user_block.py @@ -107,28 +107,31 @@ def _one_callback_proxy(gateway: Gateway, tmp_path: Path, stack: ExitStack) -> G @pytest.mark.parametrize("block_route", ("/customer/block", "/end_user/block")) -def test_block_route_marks_the_customer_blocked_and_update_unblocks(gateway: Gateway, block_route: str) -> None: - with gateway.scenario() as scenario: +def test_block_route_marks_the_customer_blocked_and_update_unblocks( + gateway: Gateway, tmp_path: Path, block_route: str +) -> None: + with ExitStack() as stack: + proxy: Final = _one_callback_proxy(gateway, tmp_path, stack) end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" - created: Final = gateway.request("POST", "/customer/new", {"user_id": end_user, "max_budget": 100.0}) + created: Final = proxy.request("POST", "/customer/new", {"user_id": end_user, "max_budget": 100.0}) assert created.status_code == 200, created.text - scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + stack.callback(proxy.request, "POST", "/customer/delete", {"user_ids": [end_user]}) - blocked: Final = gateway.request("POST", block_route, {"user_ids": [end_user]}) + blocked: Final = proxy.request("POST", block_route, {"user_ids": [end_user]}) assert blocked.status_code == 200, blocked.text assert eventually(lambda: _blocked_row(end_user), lambda value: value == [{"blocked": True}], seconds=15) == [ {"blocked": True} ] - info: Final = gateway.get("/customer/info", {"end_user_id": end_user}) + info: Final = proxy.get("/customer/info", {"end_user_id": end_user}) assert _BlockedCustomerInfo.model_validate(info).blocked is True - updated: Final = gateway.request("POST", "/customer/update", {"user_id": end_user, "blocked": False}) + updated: Final = proxy.request("POST", "/customer/update", {"user_id": end_user, "blocked": False}) assert updated.status_code == 200, updated.text assert eventually(lambda: _blocked_row(end_user), lambda value: value == [{"blocked": False}], seconds=15) == [ {"blocked": False} ] assert ( - _BlockedCustomerInfo.model_validate(gateway.get("/customer/info", {"end_user_id": end_user})).blocked + _BlockedCustomerInfo.model_validate(proxy.get("/customer/info", {"end_user_id": end_user})).blocked is False )