diff --git a/litellm/proxy/auth/login_throttle.py b/litellm/proxy/auth/login_throttle.py index 0536e293f13..27f049a950e 100644 --- a/litellm/proxy/auth/login_throttle.py +++ b/litellm/proxy/auth/login_throttle.py @@ -431,6 +431,10 @@ class LoginAttempt: throttle: LoginThrottle username: str + async def refuse_if_blocked_while_waiting(self) -> None: + """Refuses a login that waited in line while failures ahead of it blocked this key.""" + await self.throttle.attempt(self.username) + async def succeeded(self) -> None: if not self.throttle.enabled: return diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 9783f44b41f..7d413121b56 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -5,6 +5,7 @@ This module contains the core login logic that can be reused across different login endpoints (e.g., /login and /v2/login). """ +import asyncio import os import secrets from collections.abc import Mapping @@ -31,6 +32,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle from litellm.proxy.auth.password_policy import is_breach_check_enabled, is_password_breached +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, @@ -42,6 +44,7 @@ from litellm.proxy.utils import ( PrismaClient, get_server_root_path, hash_password, + run_off_loop_if_fips, verify_password, ) from litellm.repositories.user_repository import UserRepository @@ -105,9 +108,21 @@ async def screen_login_password_for_breach( return breached +_ONE_FIPS_LOGIN_AT_A_TIME: Final = asyncio.Lock() + + +async def _password_matches(password: str, stored: str, attempt: LoginAttempt) -> bool: + """Checks inline, or in FIPS mode one login at a time, refusing a login whose user got blocked while it waited.""" + if not fips_mode_active(): + return verify_password(password, stored) + async with _ONE_FIPS_LOGIN_AT_A_TIME: + await attempt.refuse_if_blocked_while_waiting() + return await run_off_loop_if_fips(verify_password, password, stored) + + async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None: """Rehash legacy password (SHA256) to scrypt on successful login.""" - if stored.startswith("scrypt:"): + if stored.startswith(("scrypt:", "pbkdf2:")): return from litellm.proxy.proxy_server import prisma_client @@ -411,7 +426,7 @@ async def _sign_in( code=401, ) - if verify_password(password, _password): + if await _password_matches(password, _password, attempt): await _rehash_password_if_needed(_user_row.user_id, password, _password) breached_now: Final = prisma_client is not None and await screen_login_password_for_breach( user_id=_user_row.user_id, diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 1a4d7ddeadb..6cd23e5c0ff 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -1,3 +1,4 @@ +import threading from collections.abc import Mapping from typing import Final @@ -5,6 +6,7 @@ from litellm.secret_managers.main import str_to_bool FIPS_MODE_SETTING: Final = "fips_mode" _SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}" +_FIPS_MODE_ACTIVE: Final = threading.Event() class FipsModeError(Exception): @@ -87,3 +89,12 @@ def enforce_fips_mode(general_settings: Mapping[str, object]) -> None: refusal: Final = fips_mode_refusal(general_settings) if refusal is not None: raise FipsModeError(refusal) + if _fips_mode_setting(general_settings): + _FIPS_MODE_ACTIVE.set() + else: + _FIPS_MODE_ACTIVE.clear() + + +def fips_mode_active() -> bool: + """Whether general_settings.fips_mode was on when enforce_fips_mode last accepted it.""" + return _FIPS_MODE_ACTIVE.is_set() diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index b30d6f01a8a..a76dad8d02a 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -85,7 +85,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( # noqa: ) from litellm.proxy.management_helpers.team_roster_sync import user_tx_db from litellm.proxy.management_helpers.utils import management_endpoint_wrapper -from litellm.proxy.utils import handle_exception_on_proxy, hash_password +from litellm.proxy.utils import handle_exception_on_proxy, hash_password, run_off_loop_if_fips from litellm.redis.stores.spend_counters import ( user_counter, ) @@ -210,7 +210,7 @@ async def _hash_password_in_dict( if not password_prevalidated: validate_password_policy(data["password"], general_settings) await validate_password_not_breached(data["password"], general_settings, hibp_client) - data["password"] = hash_password(data["password"]) + data["password"] = await run_off_loop_if_fips(hash_password, data["password"]) data["password_reset_required"] = True data["last_breach_check_at"] = None diff --git a/litellm/proxy/management_endpoints/password_endpoints.py b/litellm/proxy/management_endpoints/password_endpoints.py index 16f3e7dfcfb..21172e30639 100644 --- a/litellm/proxy/management_endpoints/password_endpoints.py +++ b/litellm/proxy/management_endpoints/password_endpoints.py @@ -33,7 +33,7 @@ from litellm.proxy.auth.password_policy import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.session_endpoints import revoke_ui_session_keys from litellm.proxy.management_helpers.audit_logs import create_object_audit_log -from litellm.proxy.utils import hash_password, verify_password +from litellm.proxy.utils import hash_password, run_off_loop_if_fips, verify_password from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.user_repository import UserRepository @@ -129,7 +129,7 @@ async def change_password( ), ) - if not verify_password(data.current_password, stored_password): + if not await run_off_loop_if_fips(verify_password, data.current_password, stored_password): raise HTTPException(status_code=400, detail=_error_detail("Current password is incorrect.")) if data.new_password == data.current_password: @@ -142,7 +142,7 @@ async def change_password( await validate_password_not_breached(data.new_password, general_settings, hibp_client) password_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = { - "password": hash_password(data.new_password), + "password": await run_off_loop_if_fips(hash_password, data.new_password), "password_reset_required": False, "last_breach_check_at": None, } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ffd2dc305e7..f5369bc34b0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1041,6 +1041,7 @@ from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports migrate_passwords_to_scrypt_async, model_dump_with_preserved_fields, prefetch_config_params, + run_off_loop_if_fips, update_spend, ) from litellm.proxy.video_endpoints.endpoints import router as video_router @@ -19796,7 +19797,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): validate_password_policy(data.password, general_settings) await validate_password_not_breached(data.password, general_settings) - hashed_pw: Final = hash_password(data.password) + hashed_pw: Final = await run_off_loop_if_fips(hash_password, data.password) current_time = litellm.utils.get_utc_datetime() tx_manager: Final[_InvitationAcceptTxManager] = prisma_client.db.tx() async with tx_manager as tx: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3102674376a..64bce0d21f2 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6,6 +6,7 @@ import inspect import json import math import os +import re import smtplib import ssl import sys @@ -24,6 +25,7 @@ from collections.abc import ( Mapping, Sequence, ) +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import date, datetime, timedelta, timezone from email.mime.multipart import MIMEMultipart @@ -85,7 +87,9 @@ from litellm.proxy._types import ( SpendLogsMetadata, SpendLogsPayload, ) +from litellm.proxy.auth.password_policy import MIN_ALLOWED_LENGTH from litellm.proxy.bug_report_config import build_proxy_bug_report +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.common_utils.openai_error_payload import ( attribute_of, error_status_code, @@ -8399,23 +8403,42 @@ def hash_token(token: str): return hashed_token +def _is_pbkdf2_row(stored: str) -> bool: + return re.fullmatch(r"pbkdf2:[A-Za-z0-9+/]{64}", stored) is not None + + def hash_password(password: str) -> str: - """Hash a password using scrypt with a random salt.""" + """Hash a password using scrypt with a random salt, or PBKDF2 in FIPS mode.""" import base64 import hashlib import os + if fips_mode_active(): + fips_salt: Final = ssl.RAND_bytes(16) + fips_dk: Final = hashlib.pbkdf2_hmac("sha256", password.encode(), fips_salt, iterations=600_000, dklen=32) + return "pbkdf2:" + base64.b64encode(fips_salt + fips_dk).decode() salt: Final = os.urandom(16) dk: Final = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) return "scrypt:" + base64.b64encode(salt + dk).decode() def verify_password(password: str, stored: str) -> bool: - """Verify a password against a stored hash. Supports scrypt and SHA256.""" + """Verify a password against a stored hash. Supports scrypt and SHA256, or only PBKDF2 in FIPS mode.""" import base64 import hashlib import secrets + if fips_mode_active(): + if not _is_pbkdf2_row(stored): + verbose_proxy_logger.error("FIPS mode refuses a non-PBKDF2 password row. Reset it with /user/update.") + return False + if len(password) < MIN_ALLOWED_LENGTH: + return False + fips_row: Final = base64.b64decode(stored[7:]) + fips_dk: Final = hashlib.pbkdf2_hmac( + "sha256", password.encode(errors="surrogatepass"), fips_row[:16], iterations=600_000, dklen=32 + ) + return secrets.compare_digest(fips_row[16:], fips_dk) if stored.startswith("scrypt:"): try: raw: Final = base64.b64decode(stored[7:]) @@ -8430,12 +8453,24 @@ def verify_password(password: str, stored: str) -> bool: return False +_FIPS_KDF_POOL: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="fips-password-kdf") + + +async def run_off_loop_if_fips(kdf: Callable[..., _T], *args: str) -> _T: + """Run a password KDF inline, or on the single KDF thread in FIPS mode.""" + if not fips_mode_active(): + return kdf(*args) + return await asyncio.get_running_loop().run_in_executor(_FIPS_KDF_POOL, kdf, *args) + + async def migrate_passwords_to_scrypt_async(prisma_client) -> str: """ Migrate plaintext passwords in the DB to scrypt. SHA256 passwords are left alone (they migrate on next login via the SHA256 fallback). Skips quickly if no plaintext passwords exist. """ + if fips_mode_active(): + return "Skipped, FIPS mode has no password migration" all_with_pw: Final = await UserRepository(prisma_client).table.find_many( where={"password": {"not": None}}, ) @@ -8446,7 +8481,10 @@ async def migrate_passwords_to_scrypt_async(prisma_client) -> str: plaintext_users: Final = [ (u.user_id, u.password) for u in all_with_pw - if u.password and not u.password.startswith("scrypt:") and not _is_sha256_hex(u.password) + if u.password + and not u.password.startswith("scrypt:") + and not _is_sha256_hex(u.password) + and not _is_pbkdf2_row(u.password) ] if not plaintext_users: return "No plaintext passwords found" diff --git a/tests/unit/proxy/auth/test_login_utils.py b/tests/unit/proxy/auth/test_login_utils.py index 5148e38aa61..bd5bdeafe95 100644 --- a/tests/unit/proxy/auth/test_login_utils.py +++ b/tests/unit/proxy/auth/test_login_utils.py @@ -5,6 +5,7 @@ This module tests the refactored login logic that was moved from proxy_server.py to login_utils.py for better reusability. """ +import asyncio import hashlib import os from collections.abc import Mapping @@ -2365,3 +2366,160 @@ class TestScreenLoginPasswordForBreach: ) is True ) + + +_PBKDF2_PASSWORD: Final = "Seed-Lamp-4-River" +_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T" +_SCRYPT_ROW: Final = "scrypt:AAECAwQFBgcICQoLDA0OD7hIwlM/k8ytnffjFDt6AuO+BvqtmZvDkWXJP+A3bf0N" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "stored, schemes_written", + [ + (_SCRYPT_ROW, []), + (_PBKDF2_ROW, []), + (hashlib.sha256(_PBKDF2_PASSWORD.encode()).hexdigest(), ["scrypt"]), + ], + ids=["scrypt", "pbkdf2", "sha256"], +) +async def test_login_rehash_rewrites_only_a_legacy_row(stored: str, schemes_written: list[str]) -> None: + from litellm.proxy.auth.login_utils import _rehash_password_if_needed + + prisma: Final = MagicMock() + prisma.db.litellm_usertable.update = AsyncMock() + with patch( # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ): + await _rehash_password_if_needed("u-1", _PBKDF2_PASSWORD, stored) + + updates: Final = prisma.db.litellm_usertable.update.await_args_list + assert [update.kwargs["data"]["password"].split(":")[0] for update in updates] == schemes_written + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_login_runs_one_pbkdf2_off_the_event_loop_and_writes_no_row( + monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool] +) -> None: + from litellm.proxy.auth.login_utils import authenticate_user + + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + repository: Final = _known_user("user@corp.com") + repository.return_value.table.find_first.return_value.password = _PBKDF2_ROW + repository.return_value.table.update = AsyncMock() + with ( + patch( # test-quality-ok: the user lookup and rehash write are the database boundary + "litellm.proxy.auth.login_utils.UserRepository", repository + ), + patch( # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ), + patch( # test-quality-ok: success mints a UI key; faked so no DB is needed + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ), + ): + result: Final = await authenticate_user( + username="user@corp.com", + password=_PBKDF2_PASSWORD, + master_key="sk-master", + prisma_client=MagicMock(), + throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, + ) + + assert result.key == "sk-ui" + assert pbkdf2_on_event_loop == [False] + repository.return_value.table.update.assert_not_awaited() + + +async def _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch: pytest.MonkeyPatch, row: str): + from litellm.proxy.auth.login_utils import authenticate_user + + throttle: Final = _throttle(user_limit=1) + repository: Final = _known_user() + user: Final = repository.return_value.table.find_first.return_value + user.password = row + + async def block_the_user_then_return_the_row(**_: object) -> MagicMock: + await throttle.record_failure("known@example.com") + await throttle.record_failure("known@example.com") + return user + + repository.return_value.table.find_first = AsyncMock(side_effect=block_the_user_then_return_the_row) + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + with ( + patch( # test-quality-ok: the user lookup is the database boundary + "litellm.proxy.auth.login_utils.UserRepository", repository + ), + patch( # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ), + patch( # test-quality-ok: success mints a UI key; faked so no DB is needed + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ), + ): + return await authenticate_user( + username="known@example.com", + password=_PBKDF2_PASSWORD, + master_key="sk-master", + prisma_client=MagicMock(), + throttle=throttle, + general_settings=_POLICY_NO_BREACH_CHECK, + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_login_hides_the_right_password_when_its_user_got_blocked_while_it_waited( + monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool] +) -> None: + from litellm.proxy._types import ProxyException + + with pytest.raises(ProxyException) as blocked: + await _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch, _PBKDF2_ROW) + + assert blocked.value.code == "429" + assert pbkdf2_on_event_loop == [] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_burst_of_wrong_passwords_stops_deriving_once_its_user_is_blocked( + monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool] +) -> None: + from litellm.proxy.auth.login_utils import authenticate_user + + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + repository: Final = _known_user() + repository.return_value.table.find_first.return_value.password = _PBKDF2_ROW + throttle: Final = _throttle(user_limit=1) + with patch( # test-quality-ok: the user lookup is the database boundary + "litellm.proxy.auth.login_utils.UserRepository", repository + ): + refusals: Final = await asyncio.gather( + *( + authenticate_user( + username="known@example.com", + password="Wrong-Lamp-4-River", + master_key="sk-master", + prisma_client=MagicMock(), + throttle=throttle, + general_settings=_POLICY_NO_BREACH_CHECK, + ) + for _ in range(6) + ), + return_exceptions=True, + ) + + assert [refusal.code for refusal in refusals] == ["401", "401", "429", "429", "429", "429"] + assert pbkdf2_on_event_loop == [False, False] + + +@pytest.mark.asyncio +async def test_standard_login_past_the_throttle_keeps_its_answer_when_its_user_gets_blocked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result: Final = await _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch, _SCRYPT_ROW) + + assert result.key == "sk-ui" diff --git a/tests/unit/proxy/auth/test_onboarding.py b/tests/unit/proxy/auth/test_onboarding.py index 46a48c21353..c449ffed806 100644 --- a/tests/unit/proxy/auth/test_onboarding.py +++ b/tests/unit/proxy/auth/test_onboarding.py @@ -462,7 +462,7 @@ async def test_claim_token_sets_accepted_at_after_password_written(): prisma.db.litellm_usertable.update.assert_called_once() call_kwargs = prisma.db.litellm_usertable.update.call_args assert call_kwargs.kwargs["where"] == {"user_id": "user-123"} - assert "password" in call_kwargs.kwargs["data"] + assert call_kwargs.kwargs["data"]["password"].startswith(("scrypt:", "pbkdf2:")) # A freshly claimed, policy-screened password lifts any pending forced # reset and re-arms the login-time breach screen. assert call_kwargs.kwargs["data"]["password_reset_required"] is False @@ -477,6 +477,13 @@ async def test_claim_token_sets_accepted_at_after_password_written(): assert outer_claims["key"] == "sk-generated-key" +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_claim_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None: + await test_claim_token_sets_accepted_at_after_password_written() + assert pbkdf2_on_event_loop == [False] + + @pytest.mark.asyncio async def test_claim_token_revokes_existing_ui_sessions(): """A claimed invite/reset link changes the password; any UI session minted diff --git a/tests/unit/proxy/auth/test_password_policy.py b/tests/unit/proxy/auth/test_password_policy.py index 0ad1083ebe0..ea60c08a4ea 100644 --- a/tests/unit/proxy/auth/test_password_policy.py +++ b/tests/unit/proxy/auth/test_password_policy.py @@ -9,6 +9,7 @@ httpx.MockTransport, so no network is touched and nothing is monkeypatched. import asyncio import hashlib +from typing import Final import httpx import pytest @@ -20,6 +21,7 @@ from litellm.proxy.auth.password_policy import ( MIN_ALLOWED_LENGTH, PasswordPolicy, get_password_policy, + is_password_breached, validate_password_not_breached, validate_password_policy, validate_passwords_bulk, @@ -355,3 +357,10 @@ async def test_validate_passwords_bulk_mixed_verdicts(): async def test_validate_passwords_bulk_empty_batch_makes_no_lookups(): verdicts = await validate_passwords_bulk((), {}, client=_client_never_called()) assert verdicts == {} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_mode_keeps_the_breach_check() -> None: + client_says_breached: Final = _client_returning(f"{_sha1_upper(STRONG_PASSWORD)[5:]}:1387") + assert await is_password_breached(STRONG_PASSWORD, {}, client_says_breached) is True diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 0884d0bbbed..d22684ea051 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,5 +1,5 @@ import _hashlib -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from contextlib import nullcontext from unittest.mock import Mock @@ -12,10 +12,17 @@ from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, enforce_fips_mode, + fips_mode_active, openssl_enforces_fips, ) +@pytest.fixture(autouse=True) +def _fips_mode_off_after_each_test() -> Iterator[None]: + yield + enforce_fips_mode({}) + + def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, scrypt_refused: bool) -> None: monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) if scrypt_refused: @@ -119,3 +126,29 @@ def test_enforce_fips_mode( with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode(general_settings) + + +@pytest.mark.parametrize( + "general_settings, active", + [({"fips_mode": True}, True), ({"fips_mode": "true"}, True), ({"fips_mode": False}, False), ({}, False)], + ids=["on", "on-from-a-string", "off", "unset"], +) +def test_fips_mode_active_follows_the_setting_the_gate_accepted( + monkeypatch: pytest.MonkeyPatch, general_settings: Mapping[str, object], active: bool +) -> None: + _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) + enforce_fips_mode(general_settings) + assert fips_mode_active() is active + + +def test_fips_mode_stays_inactive_on_a_refused_boot(monkeypatch: pytest.MonkeyPatch) -> None: + _fake_openssl(monkeypatch, fips_flag=0, scrypt_refused=False) + with pytest.raises(FipsModeError): + enforce_fips_mode({"fips_mode": True}) + assert fips_mode_active() is False + + +def test_a_fips_openssl_alone_does_not_activate_fips_mode(monkeypatch: pytest.MonkeyPatch) -> None: + _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True) + assert fips_mode_active() is False diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 298d32def7d..d08a98242ef 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -1,7 +1,9 @@ # conftest.py +import _hashlib import asyncio import copy +import hashlib import inspect import logging import os @@ -21,6 +23,7 @@ from prisma.errors import ClientNotConnectedError import litellm import litellm.proxy.proxy_server from litellm._logging import verbose_proxy_logger +from litellm.proxy.common_utils.fips import enforce_fips_mode from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( InitPassThroughEndpointHelpers, _registered_pass_through_routes, @@ -514,3 +517,31 @@ def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]: with patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock(service_logging_obj=service_logging)): yield rendered + + +@pytest.fixture +def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) + monkeypatch.setattr(_hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported"))) + + +@pytest.fixture +def fips_mode(fips_openssl: None, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True) + enforce_fips_mode({"fips_mode": True}) + yield + enforce_fips_mode({}) + + +@pytest.fixture +def pbkdf2_on_event_loop(monkeypatch: pytest.MonkeyPatch) -> list[bool]: + """One entry per PBKDF2 call: whether it ran on the event loop's thread.""" + real_pbkdf2: Final = hashlib.pbkdf2_hmac + on_event_loop: Final[list[bool]] = [] + + def recording_pbkdf2(hash_name: str, password: bytes, salt: bytes, iterations: int, dklen: int) -> bytes: + on_event_loop.append(asyncio._get_running_loop() is not None) + return real_pbkdf2(hash_name, password, salt, iterations, dklen) + + monkeypatch.setattr(hashlib, "pbkdf2_hmac", recording_pbkdf2) + return on_event_loop diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index b96a5b606eb..120cf8a2b84 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -5483,3 +5483,16 @@ async def test_user_update_reports_a_bad_budget_duration_as_a_request_error_not_ assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error") assert "Invalid budget_duration" in exc_info.value.message assert "Authentication Error" not in exc_info.value.message + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_user_update_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None: + from litellm.proxy.management_endpoints.internal_user_endpoints import _hash_password_in_dict + + data: Final = {"password": "Str0ng!Passw0rd"} + + await _hash_password_in_dict(data, {"password_policy_check_breached_passwords": False}) + + assert pbkdf2_on_event_loop == [False] + assert data["password"].startswith("pbkdf2:") diff --git a/tests/unit/proxy/management_endpoints/test_password_endpoints.py b/tests/unit/proxy/management_endpoints/test_password_endpoints.py index adff4eda47c..d1e9f80129b 100644 --- a/tests/unit/proxy/management_endpoints/test_password_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_password_endpoints.py @@ -490,3 +490,30 @@ async def test_change_password_requires_db(): ) assert exc_info.value.status_code == 500 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_password_change_runs_both_pbkdf2_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None: + from litellm.proxy._types import ChangePasswordRequest + + prisma: Final = _make_prisma( + _make_user_row("pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T") + ) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + await change_password( + data=ChangePasswordRequest(current_password="Seed-Lamp-4-River", new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + hibp_client=_hibp_client_never_called(), + ) + + assert pbkdf2_on_event_loop == [False, False] + assert prisma.db.litellm_usertable.update.call_args.kwargs["data"]["password"].startswith("pbkdf2:") diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py b/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py index f241cd00cf5..2df009a72a9 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py @@ -4,6 +4,7 @@ Symbols pinned here: - ``hash_token`` - ``hash_password`` - ``verify_password`` + - ``run_off_loop_if_fips`` - ``migrate_passwords_to_scrypt_async`` - ``_hash_token_if_needed`` - ``PrismaClient._is_sha256_hex`` (a nested helper inside @@ -13,9 +14,12 @@ Symbols pinned here: from __future__ import annotations +import asyncio import hashlib +import ssl +import threading from types import SimpleNamespace -from typing import List +from typing import Final, List from unittest.mock import AsyncMock, MagicMock import pytest @@ -25,9 +29,17 @@ from litellm.proxy.utils import ( hash_password, hash_token, migrate_passwords_to_scrypt_async, + run_off_loop_if_fips, verify_password, ) +PASSWORD: Final = "Seed-Lamp-4-River" +PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T" +EIGHT_CHARACTER_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0ODxhpezknSiGV0cY8wgaSXYf/xzxcPw49uGdOt/TBvPPH" +SEVEN_CHARACTER_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0ODxFIwl/XKIAeyWEnfqz8H9vNQX4uo7/u1HiJhB7Ro+N7" +SCRYPT_ROW: Final = "scrypt:AAECAwQFBgcICQoLDA0OD7hIwlM/k8ytnffjFDt6AuO+BvqtmZvDkWXJP+A3bf0N" +SHA256_ROW: Final = hashlib.sha256(PASSWORD.encode()).hexdigest() + def test_hash_token_returns_sha256_hex_of_input() -> None: token = "sk-abcDEF12345" @@ -217,3 +229,133 @@ async def test_migrate_passwords_raises_on_db_failure() -> None: pc.db.litellm_usertable.find_many = AsyncMock(side_effect=RuntimeError("db unavailable")) with pytest.raises(RuntimeError, match="db unavailable"): await migrate_passwords_to_scrypt_async(pc) + + +@pytest.mark.usefixtures("fips_openssl") +def test_a_fips_openssl_without_fips_mode_stays_on_scrypt() -> None: + assert hash_password(PASSWORD).startswith("scrypt:") + + +@pytest.mark.usefixtures("fips_mode") +def test_fips_hash_is_pbkdf2_sha256_at_600k_salted_by_openssl(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ssl, "RAND_bytes", lambda size: bytes(range(size))) + assert hash_password(PASSWORD) == PBKDF2_ROW + + +@pytest.mark.usefixtures("fips_mode") +def test_fips_hash_salts_every_row_and_verifies() -> None: + first: Final = hash_password(PASSWORD) + assert first != hash_password(PASSWORD) + assert verify_password(PASSWORD, first) + + +@pytest.mark.usefixtures("fips_mode") +def test_fips_verify_needs_the_right_password_of_at_least_eight_characters() -> None: + outcome: Final = { + "right": verify_password(PASSWORD, PBKDF2_ROW), + "wrong": verify_password("Seed-Lamp-4-Rivet", PBKDF2_ROW), + "lone_surrogate": verify_password("\ud800" + PASSWORD, PBKDF2_ROW), + "eight_characters": verify_password("Seed-Lam", EIGHT_CHARACTER_PBKDF2_ROW), + "seven_characters": verify_password("Seed-La", SEVEN_CHARACTER_PBKDF2_ROW), + } + assert outcome == { + "right": True, + "wrong": False, + "lone_surrogate": False, + "eight_characters": True, + "seven_characters": False, + } + + +@pytest.mark.usefixtures("fips_mode") +@pytest.mark.parametrize( + "row", + [ + SCRYPT_ROW, + SHA256_ROW, + PASSWORD, + "pbkdf2:", + PBKDF2_ROW[:-1], + PBKDF2_ROW + "A", + PBKDF2_ROW + "\n", + "pbkdf2:" + "@" * 64, + "pbkdf2:sha256:1000:AAECAwQFBgcICQoLDA0ODw==:67GE2pGj/dkUsRooBGoUFP43cqp6T9IYz/kC+hxI3RM=", + ], + ids=[ + "scrypt", + "sha256", + "plaintext", + "prefix-only", + "short", + "long", + "newline", + "not-base64", + "stored-iterations", + ], +) +def test_fips_refuses_any_other_row_with_one_error_that_hides_it(caplog: pytest.LogCaptureFixture, row: str) -> None: + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + assert verify_password(PASSWORD, row) is False + assert [record.levelname for record in caplog.records] == ["ERROR"] + assert row not in caplog.text + + +@pytest.mark.parametrize( + "row, verifies", + [(SCRYPT_ROW, True), (SHA256_ROW, True), (PBKDF2_ROW, False)], + ids=["scrypt", "sha256", "pbkdf2"], +) +def test_standard_mode_verifies_the_rows_it_always_did(row: str, verifies: bool) -> None: + assert verify_password(PASSWORD, row) is verifies + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_boot_migration_writes_no_row() -> None: + pc: Final = MagicMock() + pc.db.litellm_usertable.find_many = AsyncMock( + return_value=[_make_user("plaintext-user", PASSWORD), _make_user("fips-user", PBKDF2_ROW)] + ) + pc.db.litellm_usertable.update = AsyncMock() + + await migrate_passwords_to_scrypt_async(pc) + + pc.db.litellm_usertable.update.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_standard_boot_migration_hashes_a_plaintext_row_but_not_a_pbkdf2_row() -> None: + pc: Final = MagicMock() + pc.db.litellm_usertable.find_many = AsyncMock( + return_value=[_make_user("plaintext-user", "pbkdf2:plaintext"), _make_user("fips-user", PBKDF2_ROW)] + ) + pc.db.litellm_usertable.update = AsyncMock() + + await migrate_passwords_to_scrypt_async(pc) + + updated: Final = pc.db.litellm_usertable.update.await_args_list + assert [update.kwargs["where"]["user_id"] for update in updated] == ["plaintext-user"] + + +def _on_event_loop() -> bool: + return asyncio._get_running_loop() is not None + + +@pytest.mark.asyncio +async def test_password_kdf_runs_on_the_event_loop_in_standard_mode() -> None: + assert await run_off_loop_if_fips(_on_event_loop) is True + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_password_kdfs_leave_the_event_loop_on_a_single_thread_in_fips_mode() -> None: + release_first: Final = threading.Event() + first_kdf: Final = asyncio.ensure_future(run_off_loop_if_fips(lambda: release_first.wait(timeout=5))) + second_kdf: Final = asyncio.ensure_future(run_off_loop_if_fips(_on_event_loop)) + await asyncio.sleep(0) + kdf_threads: Final = [thread for thread in threading.enumerate() if thread.name.startswith("fips-password-kdf")] + release_first.set() + + assert len(kdf_threads) == 1 + assert await first_kdf is True + assert await second_kdf is False