From 4086d5ccaabe43fae4c08995b341ef208818a407 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Fri, 6 Nov 2026 01:31:59 -0800 Subject: [PATCH] feat(proxy): hash UI passwords with PBKDF2 in FIPS mode (#49874) * feat(proxy): hash UI passwords with PBKDF2 in FIPS mode A FIPS OpenSSL refuses scrypt, so every password write fails there and nobody can sign in with a password. When OpenSSL enforces FIPS, passwords now use PBKDF2-HMAC-SHA256 at 600k iterations with a salt from OpenSSL's DRBG, stored as "pbkdf2:" plus base64 of salt and key. Any other row is refused with one error log and left as it is, and the boot migration does nothing, because FIPS mode is for new deployments. PBKDF2 at 600k takes about five times as long as scrypt, so in FIPS mode the five request paths run it on one thread instead of the event loop. A single thread keeps a login burst from using up a CPU quota, which is what stalls the loop with asyncio.to_thread. A login that queued for that thread asks the throttle again before it answers, so a burst of guesses can't get past a block that the failures ahead of it just set. The breached password check is off in FIPS mode, because it runs the password through SHA-1. A standard OpenSSL reads False at every new branch, so standard mode keeps scrypt, inline, on the same rows. Resolves LIT-8425 * fix(proxy): take FIPS logins one at a time and recheck the throttle first A login that waited for the KDF thread asked the throttle again only after deriving, so a burst for one user kept that thread busy long after the user was blocked. Logins now wait for their turn on a lock and ask the throttle when it comes, so a blocked login costs no PBKDF2 and only one login at a time holds a place in the KDF queue. Password changes and invite claims no longer wait behind a whole burst of logins. * fix(proxy): keep PBKDF2 rows on a standard boot and warn when FIPS skips the breach check A standard boot treated a pbkdf2: row as plaintext and hashed its text, so the real password stopped working and the old row text signed in. The migration now leaves a row of that exact shape alone. Any other pbkdf2: text still migrates. FIPS mode now logs a warning when it skips a breach check that is turned on. The one-thread test no longer depends on timing, and the new test variables are annotated Final. * fix(proxy): keep the breached password check on in FIPS mode * fix(proxy): choose the password KDF from fips_mode instead of the host OpenSSL * test(proxy): fake the scrypt probe in the FIPS fixture now that the gate checks scrypt --- litellm/proxy/auth/login_throttle.py | 4 + litellm/proxy/auth/login_utils.py | 19 ++- litellm/proxy/common_utils/fips.py | 11 ++ .../internal_user_endpoints.py | 4 +- .../password_endpoints.py | 6 +- litellm/proxy/proxy_server.py | 3 +- litellm/proxy/utils.py | 44 ++++- tests/unit/proxy/auth/test_login_utils.py | 158 ++++++++++++++++++ tests/unit/proxy/auth/test_onboarding.py | 9 +- tests/unit/proxy/auth/test_password_policy.py | 9 + tests/unit/proxy/common_utils/test_fips.py | 35 +++- tests/unit/proxy/conftest.py | 31 ++++ .../test_internal_user_endpoints.py | 13 ++ .../test_password_endpoints.py | 27 +++ .../prisma_and_spend/test_password_helpers.py | 144 +++++++++++++++- 15 files changed, 503 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/auth/login_throttle.py b/litellm/proxy/auth/login_throttle.py index 0536e293f13..27f049a950e 100644 --- a/litellm/proxy/auth/login_throttle.py +++ b/litellm/proxy/auth/login_throttle.py @@ -431,6 +431,10 @@ class LoginAttempt: throttle: LoginThrottle username: str + async def refuse_if_blocked_while_waiting(self) -> None: + """Refuses a login that waited in line while failures ahead of it blocked this key.""" + await self.throttle.attempt(self.username) + async def succeeded(self) -> None: if not self.throttle.enabled: return diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 9783f44b41f..7d413121b56 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -5,6 +5,7 @@ This module contains the core login logic that can be reused across different login endpoints (e.g., /login and /v2/login). """ +import asyncio import os import secrets from collections.abc import Mapping @@ -31,6 +32,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle from litellm.proxy.auth.password_policy import is_breach_check_enabled, is_password_breached +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, @@ -42,6 +44,7 @@ from litellm.proxy.utils import ( PrismaClient, get_server_root_path, hash_password, + run_off_loop_if_fips, verify_password, ) from litellm.repositories.user_repository import UserRepository @@ -105,9 +108,21 @@ async def screen_login_password_for_breach( return breached +_ONE_FIPS_LOGIN_AT_A_TIME: Final = asyncio.Lock() + + +async def _password_matches(password: str, stored: str, attempt: LoginAttempt) -> bool: + """Checks inline, or in FIPS mode one login at a time, refusing a login whose user got blocked while it waited.""" + if not fips_mode_active(): + return verify_password(password, stored) + async with _ONE_FIPS_LOGIN_AT_A_TIME: + await attempt.refuse_if_blocked_while_waiting() + return await run_off_loop_if_fips(verify_password, password, stored) + + async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None: """Rehash legacy password (SHA256) to scrypt on successful login.""" - if stored.startswith("scrypt:"): + if stored.startswith(("scrypt:", "pbkdf2:")): return from litellm.proxy.proxy_server import prisma_client @@ -411,7 +426,7 @@ async def _sign_in( code=401, ) - if verify_password(password, _password): + if await _password_matches(password, _password, attempt): await _rehash_password_if_needed(_user_row.user_id, password, _password) breached_now: Final = prisma_client is not None and await screen_login_password_for_breach( user_id=_user_row.user_id, diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 1a4d7ddeadb..6cd23e5c0ff 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -1,3 +1,4 @@ +import threading from collections.abc import Mapping from typing import Final @@ -5,6 +6,7 @@ from litellm.secret_managers.main import str_to_bool FIPS_MODE_SETTING: Final = "fips_mode" _SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}" +_FIPS_MODE_ACTIVE: Final = threading.Event() class FipsModeError(Exception): @@ -87,3 +89,12 @@ def enforce_fips_mode(general_settings: Mapping[str, object]) -> None: refusal: Final = fips_mode_refusal(general_settings) if refusal is not None: raise FipsModeError(refusal) + if _fips_mode_setting(general_settings): + _FIPS_MODE_ACTIVE.set() + else: + _FIPS_MODE_ACTIVE.clear() + + +def fips_mode_active() -> bool: + """Whether general_settings.fips_mode was on when enforce_fips_mode last accepted it.""" + return _FIPS_MODE_ACTIVE.is_set() diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index b30d6f01a8a..a76dad8d02a 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -85,7 +85,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( # noqa: ) from litellm.proxy.management_helpers.team_roster_sync import user_tx_db from litellm.proxy.management_helpers.utils import management_endpoint_wrapper -from litellm.proxy.utils import handle_exception_on_proxy, hash_password +from litellm.proxy.utils import handle_exception_on_proxy, hash_password, run_off_loop_if_fips from litellm.redis.stores.spend_counters import ( user_counter, ) @@ -210,7 +210,7 @@ async def _hash_password_in_dict( if not password_prevalidated: validate_password_policy(data["password"], general_settings) await validate_password_not_breached(data["password"], general_settings, hibp_client) - data["password"] = hash_password(data["password"]) + data["password"] = await run_off_loop_if_fips(hash_password, data["password"]) data["password_reset_required"] = True data["last_breach_check_at"] = None diff --git a/litellm/proxy/management_endpoints/password_endpoints.py b/litellm/proxy/management_endpoints/password_endpoints.py index 16f3e7dfcfb..21172e30639 100644 --- a/litellm/proxy/management_endpoints/password_endpoints.py +++ b/litellm/proxy/management_endpoints/password_endpoints.py @@ -33,7 +33,7 @@ from litellm.proxy.auth.password_policy import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.session_endpoints import revoke_ui_session_keys from litellm.proxy.management_helpers.audit_logs import create_object_audit_log -from litellm.proxy.utils import hash_password, verify_password +from litellm.proxy.utils import hash_password, run_off_loop_if_fips, verify_password from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.user_repository import UserRepository @@ -129,7 +129,7 @@ async def change_password( ), ) - if not verify_password(data.current_password, stored_password): + if not await run_off_loop_if_fips(verify_password, data.current_password, stored_password): raise HTTPException(status_code=400, detail=_error_detail("Current password is incorrect.")) if data.new_password == data.current_password: @@ -142,7 +142,7 @@ async def change_password( await validate_password_not_breached(data.new_password, general_settings, hibp_client) password_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = { - "password": hash_password(data.new_password), + "password": await run_off_loop_if_fips(hash_password, data.new_password), "password_reset_required": False, "last_breach_check_at": None, } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ffd2dc305e7..f5369bc34b0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1041,6 +1041,7 @@ from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports migrate_passwords_to_scrypt_async, model_dump_with_preserved_fields, prefetch_config_params, + run_off_loop_if_fips, update_spend, ) from litellm.proxy.video_endpoints.endpoints import router as video_router @@ -19796,7 +19797,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): validate_password_policy(data.password, general_settings) await validate_password_not_breached(data.password, general_settings) - hashed_pw: Final = hash_password(data.password) + hashed_pw: Final = await run_off_loop_if_fips(hash_password, data.password) current_time = litellm.utils.get_utc_datetime() tx_manager: Final[_InvitationAcceptTxManager] = prisma_client.db.tx() async with tx_manager as tx: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3102674376a..64bce0d21f2 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6,6 +6,7 @@ import inspect import json import math import os +import re import smtplib import ssl import sys @@ -24,6 +25,7 @@ from collections.abc import ( Mapping, Sequence, ) +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import date, datetime, timedelta, timezone from email.mime.multipart import MIMEMultipart @@ -85,7 +87,9 @@ from litellm.proxy._types import ( SpendLogsMetadata, SpendLogsPayload, ) +from litellm.proxy.auth.password_policy import MIN_ALLOWED_LENGTH from litellm.proxy.bug_report_config import build_proxy_bug_report +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.common_utils.openai_error_payload import ( attribute_of, error_status_code, @@ -8399,23 +8403,42 @@ def hash_token(token: str): return hashed_token +def _is_pbkdf2_row(stored: str) -> bool: + return re.fullmatch(r"pbkdf2:[A-Za-z0-9+/]{64}", stored) is not None + + def hash_password(password: str) -> str: - """Hash a password using scrypt with a random salt.""" + """Hash a password using scrypt with a random salt, or PBKDF2 in FIPS mode.""" import base64 import hashlib import os + if fips_mode_active(): + fips_salt: Final = ssl.RAND_bytes(16) + fips_dk: Final = hashlib.pbkdf2_hmac("sha256", password.encode(), fips_salt, iterations=600_000, dklen=32) + return "pbkdf2:" + base64.b64encode(fips_salt + fips_dk).decode() salt: Final = os.urandom(16) dk: Final = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) return "scrypt:" + base64.b64encode(salt + dk).decode() def verify_password(password: str, stored: str) -> bool: - """Verify a password against a stored hash. Supports scrypt and SHA256.""" + """Verify a password against a stored hash. Supports scrypt and SHA256, or only PBKDF2 in FIPS mode.""" import base64 import hashlib import secrets + if fips_mode_active(): + if not _is_pbkdf2_row(stored): + verbose_proxy_logger.error("FIPS mode refuses a non-PBKDF2 password row. Reset it with /user/update.") + return False + if len(password) < MIN_ALLOWED_LENGTH: + return False + fips_row: Final = base64.b64decode(stored[7:]) + fips_dk: Final = hashlib.pbkdf2_hmac( + "sha256", password.encode(errors="surrogatepass"), fips_row[:16], iterations=600_000, dklen=32 + ) + return secrets.compare_digest(fips_row[16:], fips_dk) if stored.startswith("scrypt:"): try: raw: Final = base64.b64decode(stored[7:]) @@ -8430,12 +8453,24 @@ def verify_password(password: str, stored: str) -> bool: return False +_FIPS_KDF_POOL: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="fips-password-kdf") + + +async def run_off_loop_if_fips(kdf: Callable[..., _T], *args: str) -> _T: + """Run a password KDF inline, or on the single KDF thread in FIPS mode.""" + if not fips_mode_active(): + return kdf(*args) + return await asyncio.get_running_loop().run_in_executor(_FIPS_KDF_POOL, kdf, *args) + + async def migrate_passwords_to_scrypt_async(prisma_client) -> str: """ Migrate plaintext passwords in the DB to scrypt. SHA256 passwords are left alone (they migrate on next login via the SHA256 fallback). Skips quickly if no plaintext passwords exist. """ + if fips_mode_active(): + return "Skipped, FIPS mode has no password migration" all_with_pw: Final = await UserRepository(prisma_client).table.find_many( where={"password": {"not": None}}, ) @@ -8446,7 +8481,10 @@ async def migrate_passwords_to_scrypt_async(prisma_client) -> str: plaintext_users: Final = [ (u.user_id, u.password) for u in all_with_pw - if u.password and not u.password.startswith("scrypt:") and not _is_sha256_hex(u.password) + if u.password + and not u.password.startswith("scrypt:") + and not _is_sha256_hex(u.password) + and not _is_pbkdf2_row(u.password) ] if not plaintext_users: return "No plaintext passwords found" diff --git a/tests/unit/proxy/auth/test_login_utils.py b/tests/unit/proxy/auth/test_login_utils.py index 5148e38aa61..bd5bdeafe95 100644 --- a/tests/unit/proxy/auth/test_login_utils.py +++ b/tests/unit/proxy/auth/test_login_utils.py @@ -5,6 +5,7 @@ This module tests the refactored login logic that was moved from proxy_server.py to login_utils.py for better reusability. """ +import asyncio import hashlib import os from collections.abc import Mapping @@ -2365,3 +2366,160 @@ class TestScreenLoginPasswordForBreach: ) is True ) + + +_PBKDF2_PASSWORD: Final = "Seed-Lamp-4-River" +_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T" +_SCRYPT_ROW: Final = "scrypt:AAECAwQFBgcICQoLDA0OD7hIwlM/k8ytnffjFDt6AuO+BvqtmZvDkWXJP+A3bf0N" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "stored, schemes_written", + [ + (_SCRYPT_ROW, []), + (_PBKDF2_ROW, []), + (hashlib.sha256(_PBKDF2_PASSWORD.encode()).hexdigest(), ["scrypt"]), + ], + ids=["scrypt", "pbkdf2", "sha256"], +) +async def test_login_rehash_rewrites_only_a_legacy_row(stored: str, schemes_written: list[str]) -> None: + from litellm.proxy.auth.login_utils import _rehash_password_if_needed + + prisma: Final = MagicMock() + prisma.db.litellm_usertable.update = AsyncMock() + with patch( # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ): + await _rehash_password_if_needed("u-1", _PBKDF2_PASSWORD, stored) + + updates: Final = prisma.db.litellm_usertable.update.await_args_list + assert [update.kwargs["data"]["password"].split(":")[0] for update in updates] == schemes_written + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_login_runs_one_pbkdf2_off_the_event_loop_and_writes_no_row( + monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool] +) -> None: + from litellm.proxy.auth.login_utils import authenticate_user + + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + repository: Final = _known_user("user@corp.com") + repository.return_value.table.find_first.return_value.password = _PBKDF2_ROW + repository.return_value.table.update = AsyncMock() + with ( + patch( # test-quality-ok: the user lookup and rehash write are the database boundary + "litellm.proxy.auth.login_utils.UserRepository", repository + ), + patch( # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ), + patch( # test-quality-ok: success mints a UI key; faked so no DB is needed + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ), + ): + result: Final = await authenticate_user( + username="user@corp.com", + password=_PBKDF2_PASSWORD, + master_key="sk-master", + prisma_client=MagicMock(), + throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, + ) + + assert result.key == "sk-ui" + assert pbkdf2_on_event_loop == [False] + repository.return_value.table.update.assert_not_awaited() + + +async def _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch: pytest.MonkeyPatch, row: str): + from litellm.proxy.auth.login_utils import authenticate_user + + throttle: Final = _throttle(user_limit=1) + repository: Final = _known_user() + user: Final = repository.return_value.table.find_first.return_value + user.password = row + + async def block_the_user_then_return_the_row(**_: object) -> MagicMock: + await throttle.record_failure("known@example.com") + await throttle.record_failure("known@example.com") + return user + + repository.return_value.table.find_first = AsyncMock(side_effect=block_the_user_then_return_the_row) + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + with ( + patch( # test-quality-ok: the user lookup is the database boundary + "litellm.proxy.auth.login_utils.UserRepository", repository + ), + patch( # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ), + patch( # test-quality-ok: success mints a UI key; faked so no DB is needed + "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"}) + ), + ): + return await authenticate_user( + username="known@example.com", + password=_PBKDF2_PASSWORD, + master_key="sk-master", + prisma_client=MagicMock(), + throttle=throttle, + general_settings=_POLICY_NO_BREACH_CHECK, + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_login_hides_the_right_password_when_its_user_got_blocked_while_it_waited( + monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool] +) -> None: + from litellm.proxy._types import ProxyException + + with pytest.raises(ProxyException) as blocked: + await _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch, _PBKDF2_ROW) + + assert blocked.value.code == "429" + assert pbkdf2_on_event_loop == [] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_burst_of_wrong_passwords_stops_deriving_once_its_user_is_blocked( + monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool] +) -> None: + from litellm.proxy.auth.login_utils import authenticate_user + + monkeypatch.setenv("DATABASE_URL", "postgresql://stub") + repository: Final = _known_user() + repository.return_value.table.find_first.return_value.password = _PBKDF2_ROW + throttle: Final = _throttle(user_limit=1) + with patch( # test-quality-ok: the user lookup is the database boundary + "litellm.proxy.auth.login_utils.UserRepository", repository + ): + refusals: Final = await asyncio.gather( + *( + authenticate_user( + username="known@example.com", + password="Wrong-Lamp-4-River", + master_key="sk-master", + prisma_client=MagicMock(), + throttle=throttle, + general_settings=_POLICY_NO_BREACH_CHECK, + ) + for _ in range(6) + ), + return_exceptions=True, + ) + + assert [refusal.code for refusal in refusals] == ["401", "401", "429", "429", "429", "429"] + assert pbkdf2_on_event_loop == [False, False] + + +@pytest.mark.asyncio +async def test_standard_login_past_the_throttle_keeps_its_answer_when_its_user_gets_blocked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result: Final = await _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch, _SCRYPT_ROW) + + assert result.key == "sk-ui" diff --git a/tests/unit/proxy/auth/test_onboarding.py b/tests/unit/proxy/auth/test_onboarding.py index 46a48c21353..c449ffed806 100644 --- a/tests/unit/proxy/auth/test_onboarding.py +++ b/tests/unit/proxy/auth/test_onboarding.py @@ -462,7 +462,7 @@ async def test_claim_token_sets_accepted_at_after_password_written(): prisma.db.litellm_usertable.update.assert_called_once() call_kwargs = prisma.db.litellm_usertable.update.call_args assert call_kwargs.kwargs["where"] == {"user_id": "user-123"} - assert "password" in call_kwargs.kwargs["data"] + assert call_kwargs.kwargs["data"]["password"].startswith(("scrypt:", "pbkdf2:")) # A freshly claimed, policy-screened password lifts any pending forced # reset and re-arms the login-time breach screen. assert call_kwargs.kwargs["data"]["password_reset_required"] is False @@ -477,6 +477,13 @@ async def test_claim_token_sets_accepted_at_after_password_written(): assert outer_claims["key"] == "sk-generated-key" +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_claim_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None: + await test_claim_token_sets_accepted_at_after_password_written() + assert pbkdf2_on_event_loop == [False] + + @pytest.mark.asyncio async def test_claim_token_revokes_existing_ui_sessions(): """A claimed invite/reset link changes the password; any UI session minted diff --git a/tests/unit/proxy/auth/test_password_policy.py b/tests/unit/proxy/auth/test_password_policy.py index 0ad1083ebe0..ea60c08a4ea 100644 --- a/tests/unit/proxy/auth/test_password_policy.py +++ b/tests/unit/proxy/auth/test_password_policy.py @@ -9,6 +9,7 @@ httpx.MockTransport, so no network is touched and nothing is monkeypatched. import asyncio import hashlib +from typing import Final import httpx import pytest @@ -20,6 +21,7 @@ from litellm.proxy.auth.password_policy import ( MIN_ALLOWED_LENGTH, PasswordPolicy, get_password_policy, + is_password_breached, validate_password_not_breached, validate_password_policy, validate_passwords_bulk, @@ -355,3 +357,10 @@ async def test_validate_passwords_bulk_mixed_verdicts(): async def test_validate_passwords_bulk_empty_batch_makes_no_lookups(): verdicts = await validate_passwords_bulk((), {}, client=_client_never_called()) assert verdicts == {} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_mode_keeps_the_breach_check() -> None: + client_says_breached: Final = _client_returning(f"{_sha1_upper(STRONG_PASSWORD)[5:]}:1387") + assert await is_password_breached(STRONG_PASSWORD, {}, client_says_breached) is True diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 0884d0bbbed..d22684ea051 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,5 +1,5 @@ import _hashlib -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from contextlib import nullcontext from unittest.mock import Mock @@ -12,10 +12,17 @@ from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, enforce_fips_mode, + fips_mode_active, openssl_enforces_fips, ) +@pytest.fixture(autouse=True) +def _fips_mode_off_after_each_test() -> Iterator[None]: + yield + enforce_fips_mode({}) + + def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, scrypt_refused: bool) -> None: monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) if scrypt_refused: @@ -119,3 +126,29 @@ def test_enforce_fips_mode( with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode(general_settings) + + +@pytest.mark.parametrize( + "general_settings, active", + [({"fips_mode": True}, True), ({"fips_mode": "true"}, True), ({"fips_mode": False}, False), ({}, False)], + ids=["on", "on-from-a-string", "off", "unset"], +) +def test_fips_mode_active_follows_the_setting_the_gate_accepted( + monkeypatch: pytest.MonkeyPatch, general_settings: Mapping[str, object], active: bool +) -> None: + _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) + enforce_fips_mode(general_settings) + assert fips_mode_active() is active + + +def test_fips_mode_stays_inactive_on_a_refused_boot(monkeypatch: pytest.MonkeyPatch) -> None: + _fake_openssl(monkeypatch, fips_flag=0, scrypt_refused=False) + with pytest.raises(FipsModeError): + enforce_fips_mode({"fips_mode": True}) + assert fips_mode_active() is False + + +def test_a_fips_openssl_alone_does_not_activate_fips_mode(monkeypatch: pytest.MonkeyPatch) -> None: + _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True) + assert fips_mode_active() is False diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 298d32def7d..d08a98242ef 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -1,7 +1,9 @@ # conftest.py +import _hashlib import asyncio import copy +import hashlib import inspect import logging import os @@ -21,6 +23,7 @@ from prisma.errors import ClientNotConnectedError import litellm import litellm.proxy.proxy_server from litellm._logging import verbose_proxy_logger +from litellm.proxy.common_utils.fips import enforce_fips_mode from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( InitPassThroughEndpointHelpers, _registered_pass_through_routes, @@ -514,3 +517,31 @@ def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]: with patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock(service_logging_obj=service_logging)): yield rendered + + +@pytest.fixture +def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) + monkeypatch.setattr(_hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported"))) + + +@pytest.fixture +def fips_mode(fips_openssl: None, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True) + enforce_fips_mode({"fips_mode": True}) + yield + enforce_fips_mode({}) + + +@pytest.fixture +def pbkdf2_on_event_loop(monkeypatch: pytest.MonkeyPatch) -> list[bool]: + """One entry per PBKDF2 call: whether it ran on the event loop's thread.""" + real_pbkdf2: Final = hashlib.pbkdf2_hmac + on_event_loop: Final[list[bool]] = [] + + def recording_pbkdf2(hash_name: str, password: bytes, salt: bytes, iterations: int, dklen: int) -> bytes: + on_event_loop.append(asyncio._get_running_loop() is not None) + return real_pbkdf2(hash_name, password, salt, iterations, dklen) + + monkeypatch.setattr(hashlib, "pbkdf2_hmac", recording_pbkdf2) + return on_event_loop diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index b96a5b606eb..120cf8a2b84 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -5483,3 +5483,16 @@ async def test_user_update_reports_a_bad_budget_duration_as_a_request_error_not_ assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error") assert "Invalid budget_duration" in exc_info.value.message assert "Authentication Error" not in exc_info.value.message + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_user_update_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None: + from litellm.proxy.management_endpoints.internal_user_endpoints import _hash_password_in_dict + + data: Final = {"password": "Str0ng!Passw0rd"} + + await _hash_password_in_dict(data, {"password_policy_check_breached_passwords": False}) + + assert pbkdf2_on_event_loop == [False] + assert data["password"].startswith("pbkdf2:") diff --git a/tests/unit/proxy/management_endpoints/test_password_endpoints.py b/tests/unit/proxy/management_endpoints/test_password_endpoints.py index adff4eda47c..d1e9f80129b 100644 --- a/tests/unit/proxy/management_endpoints/test_password_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_password_endpoints.py @@ -490,3 +490,30 @@ async def test_change_password_requires_db(): ) assert exc_info.value.status_code == 500 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_password_change_runs_both_pbkdf2_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None: + from litellm.proxy._types import ChangePasswordRequest + + prisma: Final = _make_prisma( + _make_user_row("pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T") + ) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + await change_password( + data=ChangePasswordRequest(current_password="Seed-Lamp-4-River", new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + hibp_client=_hibp_client_never_called(), + ) + + assert pbkdf2_on_event_loop == [False, False] + assert prisma.db.litellm_usertable.update.call_args.kwargs["data"]["password"].startswith("pbkdf2:") diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py b/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py index f241cd00cf5..2df009a72a9 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py @@ -4,6 +4,7 @@ Symbols pinned here: - ``hash_token`` - ``hash_password`` - ``verify_password`` + - ``run_off_loop_if_fips`` - ``migrate_passwords_to_scrypt_async`` - ``_hash_token_if_needed`` - ``PrismaClient._is_sha256_hex`` (a nested helper inside @@ -13,9 +14,12 @@ Symbols pinned here: from __future__ import annotations +import asyncio import hashlib +import ssl +import threading from types import SimpleNamespace -from typing import List +from typing import Final, List from unittest.mock import AsyncMock, MagicMock import pytest @@ -25,9 +29,17 @@ from litellm.proxy.utils import ( hash_password, hash_token, migrate_passwords_to_scrypt_async, + run_off_loop_if_fips, verify_password, ) +PASSWORD: Final = "Seed-Lamp-4-River" +PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T" +EIGHT_CHARACTER_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0ODxhpezknSiGV0cY8wgaSXYf/xzxcPw49uGdOt/TBvPPH" +SEVEN_CHARACTER_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0ODxFIwl/XKIAeyWEnfqz8H9vNQX4uo7/u1HiJhB7Ro+N7" +SCRYPT_ROW: Final = "scrypt:AAECAwQFBgcICQoLDA0OD7hIwlM/k8ytnffjFDt6AuO+BvqtmZvDkWXJP+A3bf0N" +SHA256_ROW: Final = hashlib.sha256(PASSWORD.encode()).hexdigest() + def test_hash_token_returns_sha256_hex_of_input() -> None: token = "sk-abcDEF12345" @@ -217,3 +229,133 @@ async def test_migrate_passwords_raises_on_db_failure() -> None: pc.db.litellm_usertable.find_many = AsyncMock(side_effect=RuntimeError("db unavailable")) with pytest.raises(RuntimeError, match="db unavailable"): await migrate_passwords_to_scrypt_async(pc) + + +@pytest.mark.usefixtures("fips_openssl") +def test_a_fips_openssl_without_fips_mode_stays_on_scrypt() -> None: + assert hash_password(PASSWORD).startswith("scrypt:") + + +@pytest.mark.usefixtures("fips_mode") +def test_fips_hash_is_pbkdf2_sha256_at_600k_salted_by_openssl(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ssl, "RAND_bytes", lambda size: bytes(range(size))) + assert hash_password(PASSWORD) == PBKDF2_ROW + + +@pytest.mark.usefixtures("fips_mode") +def test_fips_hash_salts_every_row_and_verifies() -> None: + first: Final = hash_password(PASSWORD) + assert first != hash_password(PASSWORD) + assert verify_password(PASSWORD, first) + + +@pytest.mark.usefixtures("fips_mode") +def test_fips_verify_needs_the_right_password_of_at_least_eight_characters() -> None: + outcome: Final = { + "right": verify_password(PASSWORD, PBKDF2_ROW), + "wrong": verify_password("Seed-Lamp-4-Rivet", PBKDF2_ROW), + "lone_surrogate": verify_password("\ud800" + PASSWORD, PBKDF2_ROW), + "eight_characters": verify_password("Seed-Lam", EIGHT_CHARACTER_PBKDF2_ROW), + "seven_characters": verify_password("Seed-La", SEVEN_CHARACTER_PBKDF2_ROW), + } + assert outcome == { + "right": True, + "wrong": False, + "lone_surrogate": False, + "eight_characters": True, + "seven_characters": False, + } + + +@pytest.mark.usefixtures("fips_mode") +@pytest.mark.parametrize( + "row", + [ + SCRYPT_ROW, + SHA256_ROW, + PASSWORD, + "pbkdf2:", + PBKDF2_ROW[:-1], + PBKDF2_ROW + "A", + PBKDF2_ROW + "\n", + "pbkdf2:" + "@" * 64, + "pbkdf2:sha256:1000:AAECAwQFBgcICQoLDA0ODw==:67GE2pGj/dkUsRooBGoUFP43cqp6T9IYz/kC+hxI3RM=", + ], + ids=[ + "scrypt", + "sha256", + "plaintext", + "prefix-only", + "short", + "long", + "newline", + "not-base64", + "stored-iterations", + ], +) +def test_fips_refuses_any_other_row_with_one_error_that_hides_it(caplog: pytest.LogCaptureFixture, row: str) -> None: + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + assert verify_password(PASSWORD, row) is False + assert [record.levelname for record in caplog.records] == ["ERROR"] + assert row not in caplog.text + + +@pytest.mark.parametrize( + "row, verifies", + [(SCRYPT_ROW, True), (SHA256_ROW, True), (PBKDF2_ROW, False)], + ids=["scrypt", "sha256", "pbkdf2"], +) +def test_standard_mode_verifies_the_rows_it_always_did(row: str, verifies: bool) -> None: + assert verify_password(PASSWORD, row) is verifies + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_fips_boot_migration_writes_no_row() -> None: + pc: Final = MagicMock() + pc.db.litellm_usertable.find_many = AsyncMock( + return_value=[_make_user("plaintext-user", PASSWORD), _make_user("fips-user", PBKDF2_ROW)] + ) + pc.db.litellm_usertable.update = AsyncMock() + + await migrate_passwords_to_scrypt_async(pc) + + pc.db.litellm_usertable.update.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_standard_boot_migration_hashes_a_plaintext_row_but_not_a_pbkdf2_row() -> None: + pc: Final = MagicMock() + pc.db.litellm_usertable.find_many = AsyncMock( + return_value=[_make_user("plaintext-user", "pbkdf2:plaintext"), _make_user("fips-user", PBKDF2_ROW)] + ) + pc.db.litellm_usertable.update = AsyncMock() + + await migrate_passwords_to_scrypt_async(pc) + + updated: Final = pc.db.litellm_usertable.update.await_args_list + assert [update.kwargs["where"]["user_id"] for update in updated] == ["plaintext-user"] + + +def _on_event_loop() -> bool: + return asyncio._get_running_loop() is not None + + +@pytest.mark.asyncio +async def test_password_kdf_runs_on_the_event_loop_in_standard_mode() -> None: + assert await run_off_loop_if_fips(_on_event_loop) is True + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("fips_mode") +async def test_password_kdfs_leave_the_event_loop_on_a_single_thread_in_fips_mode() -> None: + release_first: Final = threading.Event() + first_kdf: Final = asyncio.ensure_future(run_off_loop_if_fips(lambda: release_first.wait(timeout=5))) + second_kdf: Final = asyncio.ensure_future(run_off_loop_if_fips(_on_event_loop)) + await asyncio.sleep(0) + kdf_threads: Final = [thread for thread in threading.enumerate() if thread.name.startswith("fips-password-kdf")] + release_first.set() + + assert len(kdf_threads) == 1 + assert await first_kdf is True + assert await second_kdf is False