diff --git a/litellm/__init__.py b/litellm/__init__.py index 73714cd0c9c..3668e6efb0c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -219,7 +219,6 @@ redact_user_api_key_info: Optional[bool] = False # major release; opt in early with `litellm.expose_router_debug_in_errors # = False`. expose_router_debug_in_errors: bool = True -model_access_denied_message: str | None = None filter_invalid_headers: Optional[bool] = False add_user_information_to_llm_headers: Optional[bool] = ( None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers diff --git a/litellm/constants.py b/litellm/constants.py index 7d546fdccdf..745a4d9294e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -98,7 +98,6 @@ BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024 REDACTED_BY_LITELLM: Final = "redacted-by-litellm" # in-memory stand-in handed to provider converters for redacted arguments; never stored REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}" -MODEL_ACCESS_DENIED_MESSAGE_MODEL_PLACEHOLDER: Final = "{model}" MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096) @@ -1801,8 +1800,6 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "max_ui_session_budget", "budget_rollover", "mcp_tool_search", - "model_access_denied_message", - "expose_router_debug_in_errors", ] SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 6b67fc0cb28..ba68dc8a17f 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -72,7 +72,7 @@ from litellm.proxy.auth.budget_throttle import ( budget_throttle_percentage, should_throttle_budget_exceeded, ) -from litellm.proxy.auth.model_access_denied import client_facing_model_access_denied_message +from litellm.proxy.auth.model_access_denied import model_access_denied_client_message from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import publish_auth_cache_invalidation from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec @@ -4177,7 +4177,7 @@ def _can_object_call_model( f"Tried to access {model}" ) raise ModelAccessDeniedProxyException( - message=client_facing_model_access_denied_message(internal_message=internal_message, model=model), + message=model_access_denied_client_message(model=model), internal_message=internal_message, type=ProxyErrorTypes.get_model_access_error_type_for_object(object_type=object_type), param="model", @@ -4808,7 +4808,7 @@ async def can_user_call_model( f"Tried to access {model}" ) raise ModelAccessDeniedProxyException( - message=client_facing_model_access_denied_message(internal_message=internal_message, model=model), + message=model_access_denied_client_message(model=model), internal_message=internal_message, type=ProxyErrorTypes.key_model_access_denied, param="model", @@ -5415,7 +5415,7 @@ async def _check_team_member_model_access( f"Model={model}. Allowed member models = {member_allowed_models}" ) raise ModelAccessDeniedProxyException( - message=client_facing_model_access_denied_message(internal_message=internal_message, model=model), + message=model_access_denied_client_message(model=model), internal_message=internal_message, type=ProxyErrorTypes.team_model_access_denied, param="model", diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index ea6d52b28f0..0389f69cfeb 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -54,7 +54,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_checks import can_team_access_model from litellm.proxy.auth.model_access_denied import ( ModelAccessDeniedHTTPException, - client_facing_model_access_denied_message, + model_access_denied_client_message, ) from litellm.proxy.auth.resolvers.grants import GrantResolver, UserLookup, canonical_user_id from litellm.proxy.auth.route_checks import RouteChecks @@ -1347,7 +1347,7 @@ class JWTAuthManager: raise ModelAccessDeniedHTTPException( internal_message=internal_message, status_code=403, - detail=client_facing_model_access_denied_message(internal_message=internal_message, model=model), + detail=model_access_denied_client_message(model=model), ) return True @@ -1380,11 +1380,7 @@ class JWTAuthManager: raise ModelAccessDeniedHTTPException( internal_message=internal_message, status_code=403, - detail={ - "error": client_facing_model_access_denied_message( - internal_message=internal_message, model=requested_model - ) - }, + detail={"error": model_access_denied_client_message(model=requested_model)}, ) return diff --git a/litellm/proxy/auth/model_access_denied.py b/litellm/proxy/auth/model_access_denied.py index 8164e06c42a..ffb73b343cd 100644 --- a/litellm/proxy/auth/model_access_denied.py +++ b/litellm/proxy/auth/model_access_denied.py @@ -2,15 +2,14 @@ from typing import Final from fastapi import HTTPException -import litellm -from litellm.constants import MODEL_ACCESS_DENIED_MESSAGE_MODEL_PLACEHOLDER +MODEL_ACCESS_DENIED_CLIENT_MESSAGE: Final = ( + "The requested model '{model}' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." +) -def client_facing_model_access_denied_message(internal_message: str, model: str | list[str]) -> str: - template: Final = litellm.model_access_denied_message - if not template: - return internal_message - return template.replace(MODEL_ACCESS_DENIED_MESSAGE_MODEL_PLACEHOLDER, str(model)) +def model_access_denied_client_message(model: str | list[str]) -> str: + return MODEL_ACCESS_DENIED_CLIENT_MESSAGE.format(model=model) class ModelAccessDeniedHTTPException(HTTPException): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 12d62f6620d..23bb8b6225b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1682,7 +1682,7 @@ async def openai_exception_handler(request: Request, exc: ProxyException): def _log_model_access_denial(exc: ProxyException) -> None: - if not litellm.model_access_denied_message or not isinstance(exc, ModelAccessDeniedProxyException): + if not isinstance(exc, ModelAccessDeniedProxyException): return verbose_proxy_logger.warning(exc.sanitized_internal_message()) @@ -17455,13 +17455,11 @@ GeneralSettingsUILiteLLMValue = float | bool | str | None class GeneralSettingsUILiteLLMFieldSpec(TypedDict): - type: ReadOnly[Literal["Float", "Dollar", "Boolean", "Select", "String"]] - description: ReadOnly[str] - options: ReadOnly[NotRequired[tuple[str, ...]]] - tab: ReadOnly[NotRequired[str]] # Admin UI sub-tab this field renders under; None groups it with the rest - default: ReadOnly[ - NotRequired[float | bool] - ] # reset/clear restores this instead of None; fields whose None means fail-open set it + type: Literal["Float", "Dollar", "Boolean", "Select"] + description: str + options: NotRequired[tuple[str, ...]] + tab: NotRequired[str] # Admin UI sub-tab this field renders under; None groups it with the rest + default: NotRequired[float] # reset/clear restores this instead of None; fields whose None means fail-open set it _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFieldSpec]] = { @@ -17513,24 +17511,6 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie "with this budget. Clearing restores the $1 default." ), }, - "model_access_denied_message": { - "type": "String", - "description": ( - "Client-facing error message returned when a key, team, user, org or project is not allowed " - "to call the requested model. {model} is replaced with the requested model name. The full " - "denial reason (allowed models and access groups) is still written to the proxy logs. " - "Leave empty to return the detailed message to clients." - ), - }, - "expose_router_debug_in_errors": { - "type": "Boolean", - "default": True, - "description": ( - "Append router debug details (model group, configured fallbacks, fallback errors, cooldown " - "info) to error messages returned to clients. Turn off to keep those details in the proxy " - "logs only." - ), - }, } @@ -17578,13 +17558,6 @@ def _validate_general_settings_ui_litellm_value(field_name: str, value: object) detail={"error": f"{field_name} must be a positive dollar amount or empty"}, ) return float(value) - case "String": - if not isinstance(value, str): - raise HTTPException( - status_code=400, - detail={"error": f"{field_name} must be a string or empty"}, - ) - return value case _: assert_never(field_type) diff --git a/tests/otel_tests/test_e2e_model_access.py b/tests/otel_tests/test_e2e_model_access.py index e5e93c0b179..6017a820299 100644 --- a/tests/otel_tests/test_e2e_model_access.py +++ b/tests/otel_tests/test_e2e_model_access.py @@ -101,7 +101,7 @@ async def test_model_access_patterns(key_models, test_model, expect_success): assert _error_body["type"] == "key_model_access_denied" assert _error_body["param"] == "model" assert _error_body["code"] == "403" - assert "key not allowed to access model" in _error_body["message"] + assert "is not available for this API key" in _error_body["message"] @pytest.mark.asyncio @@ -299,7 +299,5 @@ def _validate_model_access_exception( assert _error_body["type"] == expected_type assert _error_body["param"] == "model" assert _error_body["code"] == "403" - if expected_type == "key_model_access_denied": - assert "key not allowed to access model" in _error_body["message"] - elif expected_type == "team_model_access_denied": - assert "eam not allowed to access model" in _error_body["message"] + assert "is not available for this API key" in _error_body["message"] + assert "not allowed to access model" not in _error_body["message"] diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index d436c99cd20..2538556d3b5 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -163,7 +163,7 @@ async def test_can_key_call_model(model, expect_to_work): if expect_to_work: await can_key_call_model(**args) else: - with pytest.raises(Exception, match='key not allowed to access model\\. This key can only access') as e: + with pytest.raises(Exception, match='is not available for this API key') as e: await can_key_call_model(**args) print(e) @@ -943,7 +943,7 @@ async def test_can_key_call_model_with_aliases(model, alias_map, expect_to_work) llm_router=router, ) else: - with pytest.raises(Exception, match='key not allowed to access model\\. This key can only access') as e: + with pytest.raises(Exception, match='is not available for this API key') as e: await can_key_call_model( model=model, llm_model_list=llm_model_list, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py index d98db5518c3..7eebf1eb436 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_sampling_model_access.py @@ -138,11 +138,9 @@ class TestCheckModelAccess: assert "claude-3-opus-20240229" in result.message @pytest.mark.asyncio - async def test_should_log_internal_denial_reason_when_client_message_is_configured(self, monkeypatch, caplog): - import litellm + async def test_should_log_internal_denial_reason_and_hide_allowlist_from_client(self, caplog): from litellm.proxy._types import UserAPIKeyAuth - monkeypatch.setattr(litellm, "model_access_denied_message", "The model `{model}` is unavailable for this API key.") auth = UserAPIKeyAuth(api_key="sk-test-key", models=["gpt-3.5-turbo"]) with caplog.at_level("WARNING", logger="LiteLLM"): diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 0677fb8af29..26ae28a57d2 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -531,12 +531,14 @@ async def test_can_team_access_model_error_lists_direct_and_access_group_models( assert await can_team_access_model("direct-model", team_object, None) is True assert await can_team_access_model("group-model", team_object, None) is True - with pytest.raises(ProxyException) as exc_info: + with pytest.raises(ModelAccessDeniedProxyException) as exc_info: await can_team_access_model("blocked-model", team_object, None) assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied - assert "direct-model" in exc_info.value.message - assert "group-model" in exc_info.value.message + assert "direct-model" in exc_info.value.internal_message + assert "group-model" in exc_info.value.internal_message + assert "direct-model" not in exc_info.value.message + assert "group-model" not in exc_info.value.message @pytest.mark.asyncio @@ -1676,16 +1678,17 @@ def test_can_object_call_model_no_access_to_alias_or_underlying(): # Should raise ProxyException with appropriate error type assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied - assert "key not allowed to access model" in str(exc_info.value.message) + assert "is not available for this API key" in str(exc_info.value.message) assert "my-fake-gpt" in str(exc_info.value.message) -_DENIED_MESSAGE_TEMPLATE: Final = "The model `{model}` is unavailable for this API key or does not exist." +_DENIED_MESSAGE_TEMPLATE: Final = ( + "The requested model '{model}' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." +) -def test_can_object_call_model_denial_uses_configured_message_and_keeps_detail_on_exception(monkeypatch, caplog): - monkeypatch.setattr(litellm, "model_access_denied_message", _DENIED_MESSAGE_TEMPLATE) - +def test_can_object_call_model_denial_hides_allowlist_and_keeps_detail_on_exception(caplog): with caplog.at_level("DEBUG", logger="LiteLLM Proxy"): with pytest.raises(ModelAccessDeniedProxyException) as exc_info: _can_object_call_model( @@ -1695,9 +1698,8 @@ def test_can_object_call_model_denial_uses_configured_message_and_keeps_detail_o object_type="key", ) - assert ( - exc_info.value.message == "The model `anthropic-sonnet-4-5` is unavailable for this API key or does not exist." - ) + assert exc_info.value.message == _DENIED_MESSAGE_TEMPLATE.format(model="anthropic-sonnet-4-5") + assert "internal-models" not in exc_info.value.message assert exc_info.value.type == ProxyErrorTypes.key_model_access_denied assert exc_info.value.param == "model" assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN @@ -1709,10 +1711,9 @@ def test_can_object_call_model_denial_uses_configured_message_and_keeps_detail_o @pytest.mark.asyncio -async def test_access_group_fallback_grant_does_not_log_a_denial(monkeypatch, caplog): +async def test_access_group_fallback_grant_does_not_log_a_denial(caplog): from litellm.proxy.auth.auth_checks import can_team_access_model - monkeypatch.setattr(litellm, "model_access_denied_message", _DENIED_MESSAGE_TEMPLATE) team_object = LiteLLM_TeamTable(team_id="team-123", models=["direct-model"], access_group_ids=["ag-1"]) with ( @@ -1727,46 +1728,49 @@ async def test_access_group_fallback_grant_does_not_log_a_denial(monkeypatch, ca assert "not allowed to access model" not in caplog.text -@pytest.mark.parametrize("unset_value", [None, ""]) -def test_can_object_call_model_denial_unchanged_when_message_not_configured(monkeypatch, unset_value): - monkeypatch.setattr(litellm, "model_access_denied_message", unset_value) - - with pytest.raises(ProxyException) as exc_info: +@pytest.mark.parametrize( + "object_type, expected_type", + [ + ("team", ProxyErrorTypes.team_model_access_denied), + ("user", ProxyErrorTypes.user_model_access_denied), + ("org", ProxyErrorTypes.org_model_access_denied), + ], +) +def test_can_object_call_model_denial_same_client_message_for_every_object_type(object_type, expected_type): + with pytest.raises(ModelAccessDeniedProxyException) as exc_info: _can_object_call_model( model="anthropic-sonnet-4-5", llm_router=None, models=["internal-models"], - object_type="team", + object_type=object_type, ) - assert exc_info.value.message == ( - "team not allowed to access model. This team can only access models=['internal-models']. " - "Tried to access anthropic-sonnet-4-5" - ) + assert exc_info.value.message == _DENIED_MESSAGE_TEMPLATE.format(model="anthropic-sonnet-4-5") + assert exc_info.value.type == expected_type + assert f"{object_type} not allowed to access model" in exc_info.value.internal_message @pytest.mark.asyncio -async def test_can_user_call_model_no_default_models_uses_configured_message(monkeypatch): +async def test_can_user_call_model_no_default_models_hides_policy_detail(): from litellm.proxy._types import SpecialModelNames from litellm.proxy.auth.auth_checks import can_user_call_model - monkeypatch.setattr(litellm, "model_access_denied_message", _DENIED_MESSAGE_TEMPLATE) user_object = LiteLLM_UserTable(user_id="test-user", models=[SpecialModelNames.no_default_models.value]) - with pytest.raises(ProxyException) as exc_info: + with pytest.raises(ModelAccessDeniedProxyException) as exc_info: await can_user_call_model(model="restricted-model", llm_router=None, user_object=user_object) - assert exc_info.value.message == "The model `restricted-model` is unavailable for this API key or does not exist." + assert exc_info.value.message == _DENIED_MESSAGE_TEMPLATE.format(model="restricted-model") + assert "only team models allowed" in exc_info.value.internal_message assert int(exc_info.value.code) == status.HTTP_403_FORBIDDEN @pytest.mark.asyncio -async def test_check_team_member_model_access_denied_uses_configured_message(monkeypatch): +async def test_check_team_member_model_access_denied_hides_member_allowlist(): from litellm.proxy._types import LiteLLM_TeamMembership from litellm.proxy.auth.auth_checks import _check_team_member_model_access from litellm.proxy.common_utils.user_api_key_cache import team_membership_reservation_cache_key - monkeypatch.setattr(litellm, "model_access_denied_message", _DENIED_MESSAGE_TEMPLATE) membership = LiteLLM_TeamMembership( user_id="alice", team_id="team-a", @@ -1779,7 +1783,7 @@ async def test_check_team_member_model_access_denied_uses_configured_message(mon model_type=LiteLLM_TeamMembership, ) - with pytest.raises(ProxyException) as exc_info: + with pytest.raises(ModelAccessDeniedProxyException) as exc_info: await _check_team_member_model_access( model="mock-vision", team_object=LiteLLM_TeamTable(team_id="team-a"), @@ -1790,7 +1794,9 @@ async def test_check_team_member_model_access_denied_uses_configured_message(mon proxy_logging_obj=MagicMock(), ) - assert exc_info.value.message == "The model `mock-vision` is unavailable for this API key or does not exist." + assert exc_info.value.message == _DENIED_MESSAGE_TEMPLATE.format(model="mock-vision") + assert "fast-models" not in exc_info.value.message + assert "Allowed member models = ['fast-models']" in exc_info.value.internal_message assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index d5ef71cd1d1..125b8862dfc 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -26,7 +26,6 @@ from prisma.errors import ( ) -import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import INVALID_VIRTUAL_KEY_ERROR_MARKER from litellm.exceptions import BudgetExceededError @@ -991,12 +990,15 @@ async def test_handle_authentication_error_traceback_only_for_unexpected_errors( assert records[0].name == expected_logger_name -_DENIED_MESSAGE_TEMPLATE = "The model `{model}` is unavailable for this API key or does not exist." +_DENIED_CLIENT_MESSAGE = ( + "The requested model 'gpt-5.6' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." +) def _denied_proxy_exception() -> ModelAccessDeniedProxyException: return ModelAccessDeniedProxyException( - message="The model `gpt-5.6\r\nWARNING forged log line` is unavailable for this API key or does not exist.", + message=_DENIED_CLIENT_MESSAGE, internal_message="key not allowed to access model. This key can only access models=['internal-models']. " "Tried to access gpt-5.6\r\nWARNING forged log line", type=ProxyErrorTypes.key_model_access_denied, @@ -1010,7 +1012,7 @@ def _denied_jwt_exception() -> ModelAccessDeniedHTTPException: internal_message="Role=engineer not allowed to call model=gpt-5.6\r\nWARNING forged log line. " "Allowed models=['internal-models']", status_code=status.HTTP_403_FORBIDDEN, - detail="The model `gpt-5.6` is unavailable for this API key or does not exist.", + detail=_DENIED_CLIENT_MESSAGE, ) @@ -1022,10 +1024,7 @@ def _denied_jwt_exception() -> ModelAccessDeniedHTTPException: pytest.param(_denied_jwt_exception, id="jwt_http_exception"), ], ) -async def test_handle_authentication_error_keeps_internal_message_on_model_access_denial( - monkeypatch, make_denial, caplog -): - monkeypatch.setattr(litellm, "model_access_denied_message", _DENIED_MESSAGE_TEMPLATE) +async def test_handle_authentication_error_keeps_internal_message_on_model_access_denial(make_denial, caplog): handler = UserAPIKeyAuthExceptionHandler() denial = make_denial() @@ -1054,7 +1053,7 @@ async def test_handle_authentication_error_keeps_internal_message_on_model_acces def test_as_proxy_exception_keeps_jwt_scope_denial_message_shape(): - detail = {"error": "The model `gpt-5.6` is unavailable for this API key or does not exist."} + detail = {"error": _DENIED_CLIENT_MESSAGE} denial = ModelAccessDeniedHTTPException( internal_message="model=gpt-5.6 not allowed. Allowed_models=['internal-models']", status_code=status.HTTP_403_FORBIDDEN, @@ -1066,42 +1065,3 @@ def test_as_proxy_exception_keeps_jwt_scope_denial_message_shape(): assert converted.to_dict() == plain.to_dict() assert converted.internal_message == denial.internal_message - - -@pytest.mark.asyncio -@pytest.mark.parametrize("unset_value", [None, ""]) -async def test_handle_authentication_error_no_extra_denial_log_when_message_not_configured( - monkeypatch, unset_value, caplog -): - monkeypatch.setattr(litellm, "model_access_denied_message", unset_value) - handler = UserAPIKeyAuthExceptionHandler() - denial = ModelAccessDeniedProxyException( - message="key not allowed to access model. This key can only access models=['internal-models']. " - "Tried to access gpt-5.6", - internal_message="key not allowed to access model. This key can only access models=['internal-models']. " - "Tried to access gpt-5.6", - type=ProxyErrorTypes.key_model_access_denied, - param="model", - code=status.HTTP_403_FORBIDDEN, - ) - - with ( - patch( # test-quality-ok: handler reads proxy_server globals at call time - "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", - new_callable=AsyncMock, - return_value=None, - ), - patch( # test-quality-ok: handler reads proxy_server globals at call time - "litellm.proxy.auth.auth_exception_handler.seed_request_identity", - ), - patch( # test-quality-ok: handler reads proxy_server globals at call time - "litellm.proxy.proxy_server.general_settings", - {"allow_requests_on_db_unavailable": False}, - ), - caplog.at_level("WARNING", logger="LiteLLM Proxy"), - pytest.raises(ProxyException) as exc_info, - ): - await handler._handle_authentication_error(denial, MagicMock(), {}, "/v1/chat/completions", None, "sk-bad-key") - - assert "internal-models" in str(exc_info.value.message) - assert [r for r in caplog.records if r.levelname == "WARNING" and "internal-models" in r.getMessage()] == [] diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index bd6a14cad21..db16da7237c 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -1053,7 +1053,7 @@ async def test_managed_batch_routes_pass_team_model_access_check(route, request_ is True ) - with pytest.raises(Exception, match="team not allowed to access model"): + with pytest.raises(Exception, match="is not available for this API key"): await can_team_access_model( model=model, team_object=LiteLLM_TeamTable(team_id="team-other", models=["some-other-model"]), diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index 9fab1e1785a..a8385eadc59 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -6972,19 +6972,13 @@ async def test_auth_builder_denies_jwt_naming_unregistered_agent_before_admin_ch assert exc_info.value.status_code == 403 -_JWT_DENIED_MESSAGE_TEMPLATE = "The model `{model}` is unavailable for this identity." - - -@pytest.mark.parametrize( - "configured_message, expected_detail", - [ - (None, "Role=internal_user not allowed to call model=gpt-5.6. Allowed models=['gpt-5.6-mini']"), - ("", "Role=internal_user not allowed to call model=gpt-5.6. Allowed models=['gpt-5.6-mini']"), - (_JWT_DENIED_MESSAGE_TEMPLATE, "The model `gpt-5.6` is unavailable for this identity."), - ], +_JWT_DENIED_CLIENT_MESSAGE = ( + "The requested model 'gpt-5.6' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." ) -def test_can_rbac_role_call_model_denial_honors_configured_message(monkeypatch, configured_message, expected_detail): - monkeypatch.setattr(litellm, "model_access_denied_message", configured_message) + + +def test_can_rbac_role_call_model_denial_hides_role_allowlist_from_client(): general_settings = { "role_permissions": [ RoleBasedPermissions(role=LitellmUserRoles.INTERNAL_USER, models=["gpt-5.6-mini"]), @@ -6999,23 +6993,13 @@ def test_can_rbac_role_call_model_denial_honors_configured_message(monkeypatch, ) assert exc_info.value.status_code == 403 - assert exc_info.value.detail == expected_detail + assert exc_info.value.detail == _JWT_DENIED_CLIENT_MESSAGE assert exc_info.value.internal_message == ( "Role=internal_user not allowed to call model=gpt-5.6. Allowed models=['gpt-5.6-mini']" ) -@pytest.mark.parametrize( - "configured_message, expected_error", - [ - (None, "model=gpt-5.6 not allowed. Allowed_models=['gpt-5.6-mini']"), - ("", "model=gpt-5.6 not allowed. Allowed_models=['gpt-5.6-mini']"), - (_JWT_DENIED_MESSAGE_TEMPLATE, "The model `gpt-5.6` is unavailable for this identity."), - ], -) -def test_check_scope_based_access_denial_honors_configured_message(monkeypatch, configured_message, expected_error): - monkeypatch.setattr(litellm, "model_access_denied_message", configured_message) - +def test_check_scope_based_access_denial_hides_scope_allowlist_from_client(): with pytest.raises(ModelAccessDeniedHTTPException) as exc_info: JWTAuthManager.check_scope_based_access( scope_mappings=[ScopeMapping(scope="litellm.api.consumer", models=["gpt-5.6-mini"])], @@ -7025,5 +7009,5 @@ def test_check_scope_based_access_denial_honors_configured_message(monkeypatch, ) assert exc_info.value.status_code == 403 - assert exc_info.value.detail == {"error": expected_error} + assert exc_info.value.detail == {"error": _JWT_DENIED_CLIENT_MESSAGE} assert exc_info.value.internal_message == "model=gpt-5.6 not allowed. Allowed_models=['gpt-5.6-mini']" diff --git a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 82f2ef097aa..f5c97142dde 100644 --- a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -287,7 +287,7 @@ async def test_client_secrets_transcription_rejects_disallowed_nested_model( ) assert response.status_code == 403 - assert "Tried to access gpt-realtime-whisper" in response.text + assert "The requested model 'gpt-realtime-whisper' is not available for this API key" in response.text mock_route_request.assert_not_called() finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) @@ -611,7 +611,7 @@ async def test_transcription_sessions_rejects_disallowed_resolved_model( ) assert response.status_code == 403 - assert "Tried to access gpt-realtime-whisper" in response.text + assert "The requested model 'gpt-realtime-whisper' is not available for this API key" in response.text mock_route_request.assert_not_called() finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) @@ -658,7 +658,7 @@ async def test_transcription_sessions_rejects_disallowed_team_model_scope( assert response.status_code == 403 assert "team" in response.text.lower() - assert "Tried to access gpt-realtime-whisper" in response.text + assert "The requested model 'gpt-realtime-whisper' is not available for this API key" in response.text mock_route_request.assert_not_called() finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) @@ -703,7 +703,7 @@ async def test_transcription_sessions_rejects_disallowed_project_model_scope( assert response.status_code == 403 assert "project" in response.text.lower() - assert "Tried to access gpt-realtime-whisper" in response.text + assert "The requested model 'gpt-realtime-whisper' is not available for this API key" in response.text mock_route_request.assert_not_called() finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) @@ -757,7 +757,7 @@ async def test_transcription_sessions_rejects_disallowed_team_member_model_scope ) assert response.status_code == 403 - assert "Team member not allowed to access model" in response.text + assert "is not available for this API key" in response.text mock_route_request.assert_not_called() finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) @@ -783,7 +783,7 @@ async def test_realtime_transcription_websocket_default_model_checks_key_scope() websocket.close.assert_awaited_once() _, close_kwargs = websocket.close.call_args assert close_kwargs["code"] == 1008 - assert "not allowed to access model" in close_kwargs["reason"] + assert "is not available for this API key" in close_kwargs["reason"] @pytest.mark.asyncio @@ -825,7 +825,7 @@ async def test_realtime_transcription_websocket_default_model_checks_team_scope( websocket.close.assert_awaited_once() _, close_kwargs = websocket.close.call_args assert close_kwargs["code"] == 1008 - assert "not allowed to access model" in close_kwargs["reason"] + assert "is not available for this API key" in close_kwargs["reason"] @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index ee85c9ba6a5..d1928b9cd52 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -10955,60 +10955,10 @@ def test_validate_max_ui_session_budget_empty_restores_default(empty_value): assert _validate_general_settings_ui_litellm_value("max_ui_session_budget", empty_value) == 1.0 -@pytest.mark.asyncio -async def test_update_config_field_model_access_denied_message_sets_live_value(monkeypatch): - from unittest.mock import AsyncMock, MagicMock - - import litellm.proxy.proxy_server as ps - from litellm.proxy._types import ConfigFieldUpdate, LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.proxy_server import update_config_general_settings - - save_config = AsyncMock() - monkeypatch.setattr(ps.proxy_config, "get_config", AsyncMock(return_value={"litellm_settings": {}})) - monkeypatch.setattr(ps.proxy_config, "save_config", save_config) - monkeypatch.setattr(ps, "prisma_client", MagicMock()) - monkeypatch.setattr(litellm, "store_audit_logs", False) - monkeypatch.setattr(litellm, "model_access_denied_message", None) - - admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN) - await update_config_general_settings( - data=ConfigFieldUpdate( - field_name="model_access_denied_message", - field_value="Model `{model}` is unavailable for this key.", - config_type="general_settings", - ), - user_api_key_dict=admin, - ) - - assert litellm.model_access_denied_message == "Model `{model}` is unavailable for this key." - save_config.assert_awaited_once() - saved_config = save_config.await_args.kwargs["new_config"] - assert saved_config["litellm_settings"]["model_access_denied_message"] == ( - "Model `{model}` is unavailable for this key." - ) - - -@pytest.mark.parametrize("bad_value", [True, 3, 1.5, ["x"], {"a": "b"}]) -def test_validate_model_access_denied_message_rejects_non_strings(bad_value): - from fastapi import HTTPException - - from litellm.proxy.proxy_server import _validate_general_settings_ui_litellm_value - - with pytest.raises(HTTPException) as exc_info: - _validate_general_settings_ui_litellm_value("model_access_denied_message", bad_value) - assert exc_info.value.status_code == 400 - - -@pytest.mark.parametrize("empty_value", [None, ""]) -def test_validate_model_access_denied_message_empty_restores_detailed_default(empty_value): - from litellm.proxy.proxy_server import _validate_general_settings_ui_litellm_value - - assert _validate_general_settings_ui_litellm_value("model_access_denied_message", empty_value) is None - - def _model_access_denied_proxy_exception(): return ModelAccessDeniedProxyException( - message="The model `gpt-5.6\r\nWARNING forged log line` is unavailable for this API key or does not exist.", + message="The requested model 'gpt-5.6\r\nWARNING forged log line' is not available for this API key, " + "or the model name is invalid. Check the models available to you and try again.", internal_message="key not allowed to access model. This key can only access models=['internal-models']. " "Tried to access gpt-5.6\r\nWARNING forged log line", type=ProxyErrorTypes.key_model_access_denied, @@ -11022,11 +10972,7 @@ def _http_request_scope(): @pytest.mark.asyncio -async def test_openai_exception_handler_logs_sanitized_model_access_denial(monkeypatch, caplog): - monkeypatch.setattr( - litellm, "model_access_denied_message", "The model `{model}` is unavailable for this API key or does not exist." - ) - +async def test_openai_exception_handler_logs_sanitized_model_access_denial(caplog): with caplog.at_level("WARNING", logger="LiteLLM Proxy"): response = await openai_exception_handler(_http_request_scope(), _model_access_denied_proxy_exception()) @@ -11042,22 +10988,7 @@ async def test_openai_exception_handler_logs_sanitized_model_access_denial(monke @pytest.mark.asyncio -@pytest.mark.parametrize("unset_value", [None, ""]) -async def test_openai_exception_handler_no_denial_log_when_message_not_configured(monkeypatch, unset_value, caplog): - monkeypatch.setattr(litellm, "model_access_denied_message", unset_value) - - with caplog.at_level("WARNING", logger="LiteLLM Proxy"): - response = await openai_exception_handler(_http_request_scope(), _model_access_denied_proxy_exception()) - - assert response.status_code == 403 - assert [r for r in caplog.records if r.levelname == "WARNING" and "internal-models" in r.getMessage()] == [] - - -@pytest.mark.asyncio -async def test_openai_exception_handler_no_denial_log_for_plain_proxy_exception(monkeypatch, caplog): - monkeypatch.setattr( - litellm, "model_access_denied_message", "The model `{model}` is unavailable for this API key or does not exist." - ) +async def test_openai_exception_handler_no_denial_log_for_plain_proxy_exception(caplog): denial = ProxyException( message="Authentication Error, Invalid proxy server token passed", type=ProxyErrorTypes.auth_error, @@ -11073,10 +11004,7 @@ async def test_openai_exception_handler_no_denial_log_for_plain_proxy_exception( @pytest.mark.asyncio -async def test_realtime_model_access_denial_logs_sanitized_internal_message(monkeypatch, caplog): - monkeypatch.setattr( - litellm, "model_access_denied_message", "The model `{model}` is unavailable for this API key or does not exist." - ) +async def test_realtime_model_access_denial_logs_sanitized_internal_message(caplog): reservation = {"reserved_cost": 0.0, "input_cost": 0.0, "finalized": False, "entries": []} with caplog.at_level("WARNING", logger="LiteLLM Proxy"): @@ -11095,43 +11023,6 @@ async def test_realtime_model_access_denial_logs_sanitized_internal_message(monk assert "gpt-5.6WARNING forged log line" in denial_records[0].getMessage() -@pytest.mark.parametrize("empty_value", [None, ""]) -def test_validate_expose_router_debug_in_errors_empty_restores_true_default(empty_value): - from litellm.proxy.proxy_server import _validate_general_settings_ui_litellm_value - - assert _validate_general_settings_ui_litellm_value("expose_router_debug_in_errors", empty_value) is True - assert _validate_general_settings_ui_litellm_value("expose_router_debug_in_errors", False) is False - - -@pytest.mark.parametrize( - "field_name, booted_value, db_value, read_setting", - [ - ( - "model_access_denied_message", - None, - "Model `{model}` is unavailable for this key.", - lambda: litellm.model_access_denied_message, - ), - ("expose_router_debug_in_errors", True, False, lambda: litellm.expose_router_debug_in_errors), - ], -) -def test_model_access_denied_settings_propagate_on_config_reload( - monkeypatch, field_name, booted_value, db_value, read_setting -): - import litellm.proxy.proxy_server as ps - - monkeypatch.setattr(litellm, field_name, booted_value) - assert read_setting() == booted_value - - ps.ProxyConfig()._update_config_fields( - current_config={"litellm_settings": {}}, - param_name="litellm_settings", - db_param_value={field_name: db_value}, - ) - - assert read_setting() == db_value - - def test_general_settings_ui_defaults_unchanged_for_existing_fields(): """The spec-default mechanism added for max_ui_session_budget must not change what clearing the pre-existing fields restores (None for Float/Select, False for Boolean).""" diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index fb42ab6c893..1e6636ec3d6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -16424,7 +16424,7 @@ class TestMemberAutoRouterInference: project_id="router-project", team_id="router-team", models=["restricted-model"], ), model_type=LiteLLM_ProjectTableCachedObj, ) - with pytest.raises(ProxyException, match="not allowed to access model"): + with pytest.raises(ProxyException, match="is not available for this API key"): await self._route(self._router(), self._request(actor=self.actor.model_copy(update={ "models": ["member-router"] if ceiling == "key" else self.actor.models, "project_id": "router-project" if ceiling == "project" else None, @@ -16453,7 +16453,7 @@ class TestMemberAutoRouterInference: assert self.database.db.litellm_accessgrouptable.find_unique.await_count == 1 self.database.db.litellm_accessgrouptable.find_unique.return_value = group.model_copy(update={"access_model_names": []}) await evict_and_broadcast(cache_keys=("access_group_id:router-group",), user_api_key_cache=self.cache) - with pytest.raises(ProxyException, match="not allowed to access model"): + with pytest.raises(ProxyException, match="is not available for this API key"): await self._route(router, request) assert self.database.db.litellm_accessgrouptable.find_unique.await_count == 2 @@ -16471,7 +16471,7 @@ class TestMemberAutoRouterInference: key="team_id:router-team", model_type=LiteLLM_TeamTable, value=self.team.model_copy(update={"models": ["member-router"]}), ) - with pytest.raises(ProxyException, match="not allowed to access model"): + with pytest.raises(ProxyException, match="is not available for this API key"): await self._route(router, self._request()) self.database.db.litellm_teamtable.find_unique.reset_mock() admin: Final = self._request(tag="admin") diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index ab43d1acb00..e8a7732e4cb 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -307,7 +307,7 @@ async def test_chat_completion(): model="gpt-4", messages=[{"role": "user", "content": "Hello!"}], ) - assert "key not allowed to access model." in str(e) + assert "is not available for this API key" in str(e) @pytest.mark.asyncio diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx index 2dc6ecf09e5..b4df567e250 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, renderWithProviders, screen, within } from "../../../../../tests/test-utils"; +import { renderWithProviders, screen, within } from "../../../../../tests/test-utils"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import GeneralSettings from "./general_settings"; @@ -62,14 +62,6 @@ const SETTINGS_FIXTURE = [ stored_in_db: true, field_default_value: 1.0, }, - { - field_name: "model_access_denied_message", - field_type: "String", - field_value: null, - field_description: "client-facing denial message", - stored_in_db: null, - field_default_value: null, - }, ]; const settingsRow = async (fieldName: string) => { @@ -116,54 +108,6 @@ describe("GeneralSettings General tab", () => { expect(deleteConfigFieldSetting).toHaveBeenCalledWith("token", "max_ui_session_budget"); expect(numericValueIn(row)).toBe(1); }); - - it("saves a typed model_access_denied_message and resets it when cleared", async () => { - const user = userEvent.setup(); - renderWithProviders(); - - await user.click(screen.getByText("General")); - const row = await settingsRow("model_access_denied_message"); - const input = within(row).getByRole("textbox") as HTMLInputElement; - expect(input.value).toBe(""); - - fireEvent.change(input, { target: { value: "Model `{model}` is unavailable for this key." } }); - await user.click(within(row).getByRole("button", { name: /update/i })); - expect(updateConfigFieldSetting).toHaveBeenCalledWith( - "token", - "model_access_denied_message", - "Model `{model}` is unavailable for this key.", - ); - - fireEvent.change(input, { target: { value: "" } }); - await user.click(within(row).getByRole("button", { name: /update/i })); - expect(deleteConfigFieldSetting).toHaveBeenCalledWith("token", "model_access_denied_message"); - expect(vi.mocked(updateConfigFieldSetting).mock.calls).toHaveLength(1); - }); - - it("keeps the stored value visible when the reset request fails", async () => { - vi.mocked(getGeneralSettingsCall).mockResolvedValue( - SETTINGS_FIXTURE.map((s) => - s.field_name === "model_access_denied_message" - ? { ...s, field_value: "Model `{model}` is unavailable.", stored_in_db: true } - : { ...s }, - ), - ); - vi.mocked(deleteConfigFieldSetting).mockRejectedValueOnce(new Error("proxy unreachable")); - const user = userEvent.setup(); - renderWithProviders(); - - await user.click(screen.getByText("General")); - const row = await settingsRow("model_access_denied_message"); - const input = within(row).getByRole("textbox") as HTMLInputElement; - expect(within(row).getByText("In DB")).toBeInTheDocument(); - - fireEvent.change(input, { target: { value: "" } }); - await user.click(within(row).getByRole("button", { name: /update/i })); - - expect(deleteConfigFieldSetting).toHaveBeenCalledWith("token", "model_access_denied_message"); - expect(within(row).getByText("In DB")).toBeInTheDocument(); - expect(within(row).queryByText("Not Set")).not.toBeInTheDocument(); - }); }); describe("GeneralSettings Prompt Caching tab", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index ca2d80b856a..9a718cbe9b8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -42,8 +42,6 @@ export interface generalSettingsItem { const NUMERIC_INPUT_WIDTH = "w-36"; const toNumericValue = (raw: string): number | null => (raw === "" ? null : Number(raw)); -const toStringValue = (raw: string): string | null => (raw === "" ? null : raw); -const RESETS_WHEN_CLEARED: ReadonlySet = new Set(["Select", "String"]); const SettingValueEditor: React.FC<{ setting: generalSettingsItem; @@ -112,16 +110,6 @@ const SettingValueEditor: React.FC<{ ); } - if (setting.field_type === "String") { - return ( - onChange(setting.field_name, toStringValue(event.target.value))} - /> - ); - } return null; }; @@ -231,7 +219,7 @@ const GeneralSettings: React.FC = ({ accessToken, user setGeneralSettings(updatedSettings); }; - const handleUpdateField = async (fieldName: string) => { + const handleUpdateField = (fieldName: string) => { if (!accessToken) { return; } @@ -240,33 +228,37 @@ const GeneralSettings: React.FC = ({ accessToken, user const fieldValue = setting?.field_value; if (fieldValue == null) { - if (setting && RESETS_WHEN_CLEARED.has(setting.field_type)) await handleResetField(fieldName); + if (setting?.field_type === "Select") handleResetField(fieldName); return; } try { - await updateConfigFieldSetting(accessToken, fieldName, fieldValue); - setGeneralSettings((current) => - current.map((setting) => (setting.field_name === fieldName ? { ...setting, stored_in_db: true } : setting)), + updateConfigFieldSetting(accessToken, fieldName, fieldValue); + // update value in state + + const updatedSettings = generalSettings.map((setting) => + setting.field_name === fieldName ? { ...setting, stored_in_db: true } : setting, ); + setGeneralSettings(updatedSettings); } catch (error) { // do something } }; - const handleResetField = async (fieldName: string) => { + const handleResetField = (fieldName: string) => { if (!accessToken) { return; } try { - await deleteConfigFieldSetting(accessToken, fieldName); - setGeneralSettings((current) => - current.map((setting) => - setting.field_name === fieldName - ? { ...setting, stored_in_db: null, field_value: setting.field_default_value ?? null } - : setting, - ), + deleteConfigFieldSetting(accessToken, fieldName); + // update value in state + + const updatedSettings = generalSettings.map((setting) => + setting.field_name === fieldName + ? { ...setting, stored_in_db: null, field_value: setting.field_default_value ?? null } + : setting, ); + setGeneralSettings(updatedSettings); } catch (error) { // do something }