feat(proxy): hash UI passwords with PBKDF2 in FIPS mode (#49874)

* feat(proxy): hash UI passwords with PBKDF2 in FIPS mode

A FIPS OpenSSL refuses scrypt, so every password write fails there and
nobody can sign in with a password. When OpenSSL enforces FIPS, passwords
now use PBKDF2-HMAC-SHA256 at 600k iterations with a salt from OpenSSL's
DRBG, stored as "pbkdf2:" plus base64 of salt and key. Any other row is
refused with one error log and left as it is, and the boot migration does
nothing, because FIPS mode is for new deployments.

PBKDF2 at 600k takes about five times as long as scrypt, so in FIPS mode the
five request paths run it on one thread instead of the event loop. A single
thread keeps a login burst from using up a CPU quota, which is what stalls
the loop with asyncio.to_thread. A login that queued for that thread asks the
throttle again before it answers, so a burst of guesses can't get past a block
that the failures ahead of it just set.

The breached password check is off in FIPS mode, because it runs the
password through SHA-1.

A standard OpenSSL reads False at every new branch, so standard mode keeps
scrypt, inline, on the same rows.

Resolves LIT-8425

* fix(proxy): take FIPS logins one at a time and recheck the throttle first

A login that waited for the KDF thread asked the throttle again only after
deriving, so a burst for one user kept that thread busy long after the user
was blocked. Logins now wait for their turn on a lock and ask the throttle
when it comes, so a blocked login costs no PBKDF2 and only one login at a time
holds a place in the KDF queue. Password changes and invite claims no longer
wait behind a whole burst of logins.

* fix(proxy): keep PBKDF2 rows on a standard boot and warn when FIPS skips the breach check

A standard boot treated a pbkdf2: row as plaintext and hashed its text, so the
real password stopped working and the old row text signed in. The migration now
leaves a row of that exact shape alone. Any other pbkdf2: text still migrates.

FIPS mode now logs a warning when it skips a breach check that is turned on.

The one-thread test no longer depends on timing, and the new test variables are
annotated Final.

* fix(proxy): keep the breached password check on in FIPS mode

* fix(proxy): choose the password KDF from fips_mode instead of the host OpenSSL

* test(proxy): fake the scrypt probe in the FIPS fixture now that the gate checks scrypt

litellm/proxy/auth/login_throttle.py

@@ -431,6 +431,10 @@ class LoginAttempt:
     throttle: LoginThrottle
     username: str
 
+    async def refuse_if_blocked_while_waiting(self) -> None:
+        """Refuses a login that waited in line while failures ahead of it blocked this key."""
+        await self.throttle.attempt(self.username)
+
     async def succeeded(self) -> None:
         if not self.throttle.enabled:
             return

litellm/proxy/auth/login_utils.py

@@ -5,6 +5,7 @@ This module contains the core login logic that can be reused across different
 login endpoints (e.g., /login and /v2/login).
 """
 
+import asyncio
 import os
 import secrets
 from collections.abc import Mapping
@@ -31,6 +32,7 @@ from litellm.proxy._types import (
 from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured
 from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle
 from litellm.proxy.auth.password_policy import is_breach_check_enabled, is_password_breached
+from litellm.proxy.common_utils.fips import fips_mode_active
 from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
 from litellm.proxy.management_endpoints.key_management_endpoints import (
     generate_key_helper_fn,
@@ -42,6 +44,7 @@ from litellm.proxy.utils import (
     PrismaClient,
     get_server_root_path,
     hash_password,
+    run_off_loop_if_fips,
     verify_password,
 )
 from litellm.repositories.user_repository import UserRepository
@@ -105,9 +108,21 @@ async def screen_login_password_for_breach(
     return breached
 
 
+_ONE_FIPS_LOGIN_AT_A_TIME: Final = asyncio.Lock()
+
+
+async def _password_matches(password: str, stored: str, attempt: LoginAttempt) -> bool:
+    """Checks inline, or in FIPS mode one login at a time, refusing a login whose user got blocked while it waited."""
+    if not fips_mode_active():
+        return verify_password(password, stored)
+    async with _ONE_FIPS_LOGIN_AT_A_TIME:
+        await attempt.refuse_if_blocked_while_waiting()
+        return await run_off_loop_if_fips(verify_password, password, stored)
+
+
 async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None:
     """Rehash legacy password (SHA256) to scrypt on successful login."""
-    if stored.startswith("scrypt:"):
+    if stored.startswith(("scrypt:", "pbkdf2:")):
         return
     from litellm.proxy.proxy_server import prisma_client
 
@@ -411,7 +426,7 @@ async def _sign_in(
                 code=401,
             )
 
-        if verify_password(password, _password):
+        if await _password_matches(password, _password, attempt):
             await _rehash_password_if_needed(_user_row.user_id, password, _password)
             breached_now: Final = prisma_client is not None and await screen_login_password_for_breach(
                 user_id=_user_row.user_id,

litellm/proxy/common_utils/fips.py

@@ -1,3 +1,4 @@
+import threading
 from collections.abc import Mapping
 from typing import Final
 
@@ -5,6 +6,7 @@ from litellm.secret_managers.main import str_to_bool
 
 FIPS_MODE_SETTING: Final = "fips_mode"
 _SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}"
+_FIPS_MODE_ACTIVE: Final = threading.Event()
 
 
 class FipsModeError(Exception):
@@ -87,3 +89,12 @@ def enforce_fips_mode(general_settings: Mapping[str, object]) -> None:
     refusal: Final = fips_mode_refusal(general_settings)
     if refusal is not None:
         raise FipsModeError(refusal)
+    if _fips_mode_setting(general_settings):
+        _FIPS_MODE_ACTIVE.set()
+    else:
+        _FIPS_MODE_ACTIVE.clear()
+
+
+def fips_mode_active() -> bool:
+    """Whether general_settings.fips_mode was on when enforce_fips_mode last accepted it."""
+    return _FIPS_MODE_ACTIVE.is_set()

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -85,7 +85,7 @@ from litellm.proxy.management_helpers.object_permission_utils import (  # noqa:
 )
 from litellm.proxy.management_helpers.team_roster_sync import user_tx_db
 from litellm.proxy.management_helpers.utils import management_endpoint_wrapper
-from litellm.proxy.utils import handle_exception_on_proxy, hash_password
+from litellm.proxy.utils import handle_exception_on_proxy, hash_password, run_off_loop_if_fips
 from litellm.redis.stores.spend_counters import (
     user_counter,
 )
@@ -210,7 +210,7 @@ async def _hash_password_in_dict(
         if not password_prevalidated:
             validate_password_policy(data["password"], general_settings)
             await validate_password_not_breached(data["password"], general_settings, hibp_client)
-        data["password"] = hash_password(data["password"])
+        data["password"] = await run_off_loop_if_fips(hash_password, data["password"])
         data["password_reset_required"] = True
         data["last_breach_check_at"] = None
 

litellm/proxy/management_endpoints/password_endpoints.py

@@ -33,7 +33,7 @@ from litellm.proxy.auth.password_policy import (
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.management_endpoints.session_endpoints import revoke_ui_session_keys
 from litellm.proxy.management_helpers.audit_logs import create_object_audit_log
-from litellm.proxy.utils import hash_password, verify_password
+from litellm.proxy.utils import hash_password, run_off_loop_if_fips, verify_password
 from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.user_repository import UserRepository
 
@@ -129,7 +129,7 @@ async def change_password(
             ),
         )
 
-    if not verify_password(data.current_password, stored_password):
+    if not await run_off_loop_if_fips(verify_password, data.current_password, stored_password):
         raise HTTPException(status_code=400, detail=_error_detail("Current password is incorrect."))
 
     if data.new_password == data.current_password:
@@ -142,7 +142,7 @@ async def change_password(
     await validate_password_not_breached(data.new_password, general_settings, hibp_client)
 
     password_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = {
-        "password": hash_password(data.new_password),
+        "password": await run_off_loop_if_fips(hash_password, data.new_password),
         "password_reset_required": False,
         "last_breach_check_at": None,
     }

litellm/proxy/proxy_server.py

@@ -1041,6 +1041,7 @@ from litellm.proxy.utils import (  # noqa: F401, RUF100  # legacy module exports
     migrate_passwords_to_scrypt_async,
     model_dump_with_preserved_fields,
     prefetch_config_params,
+    run_off_loop_if_fips,
     update_spend,
 )
 from litellm.proxy.video_endpoints.endpoints import router as video_router
@@ -19796,7 +19797,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request):
 
     validate_password_policy(data.password, general_settings)
     await validate_password_not_breached(data.password, general_settings)
-    hashed_pw: Final = hash_password(data.password)
+    hashed_pw: Final = await run_off_loop_if_fips(hash_password, data.password)
     current_time = litellm.utils.get_utc_datetime()
     tx_manager: Final[_InvitationAcceptTxManager] = prisma_client.db.tx()
     async with tx_manager as tx:

litellm/proxy/utils.py

@@ -6,6 +6,7 @@ import inspect
 import json
 import math
 import os
+import re
 import smtplib
 import ssl
 import sys
@@ -24,6 +25,7 @@ from collections.abc import (
     Mapping,
     Sequence,
 )
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import date, datetime, timedelta, timezone
 from email.mime.multipart import MIMEMultipart
@@ -85,7 +87,9 @@ from litellm.proxy._types import (
     SpendLogsMetadata,
     SpendLogsPayload,
 )
+from litellm.proxy.auth.password_policy import MIN_ALLOWED_LENGTH
 from litellm.proxy.bug_report_config import build_proxy_bug_report
+from litellm.proxy.common_utils.fips import fips_mode_active
 from litellm.proxy.common_utils.openai_error_payload import (
     attribute_of,
     error_status_code,
@@ -8399,23 +8403,42 @@ def hash_token(token: str):
     return hashed_token
 
 
+def _is_pbkdf2_row(stored: str) -> bool:
+    return re.fullmatch(r"pbkdf2:[A-Za-z0-9+/]{64}", stored) is not None
+
+
 def hash_password(password: str) -> str:
-    """Hash a password using scrypt with a random salt."""
+    """Hash a password using scrypt with a random salt, or PBKDF2 in FIPS mode."""
     import base64
     import hashlib
     import os
 
+    if fips_mode_active():
+        fips_salt: Final = ssl.RAND_bytes(16)
+        fips_dk: Final = hashlib.pbkdf2_hmac("sha256", password.encode(), fips_salt, iterations=600_000, dklen=32)
+        return "pbkdf2:" + base64.b64encode(fips_salt + fips_dk).decode()
     salt: Final = os.urandom(16)
     dk: Final = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32)
     return "scrypt:" + base64.b64encode(salt + dk).decode()
 
 
 def verify_password(password: str, stored: str) -> bool:
-    """Verify a password against a stored hash. Supports scrypt and SHA256."""
+    """Verify a password against a stored hash. Supports scrypt and SHA256, or only PBKDF2 in FIPS mode."""
     import base64
     import hashlib
     import secrets
 
+    if fips_mode_active():
+        if not _is_pbkdf2_row(stored):
+            verbose_proxy_logger.error("FIPS mode refuses a non-PBKDF2 password row. Reset it with /user/update.")
+            return False
+        if len(password) < MIN_ALLOWED_LENGTH:
+            return False
+        fips_row: Final = base64.b64decode(stored[7:])
+        fips_dk: Final = hashlib.pbkdf2_hmac(
+            "sha256", password.encode(errors="surrogatepass"), fips_row[:16], iterations=600_000, dklen=32
+        )
+        return secrets.compare_digest(fips_row[16:], fips_dk)
     if stored.startswith("scrypt:"):
         try:
             raw: Final = base64.b64decode(stored[7:])
@@ -8430,12 +8453,24 @@ def verify_password(password: str, stored: str) -> bool:
     return False
 
 
+_FIPS_KDF_POOL: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="fips-password-kdf")
+
+
+async def run_off_loop_if_fips(kdf: Callable[..., _T], *args: str) -> _T:
+    """Run a password KDF inline, or on the single KDF thread in FIPS mode."""
+    if not fips_mode_active():
+        return kdf(*args)
+    return await asyncio.get_running_loop().run_in_executor(_FIPS_KDF_POOL, kdf, *args)
+
+
 async def migrate_passwords_to_scrypt_async(prisma_client) -> str:
     """
     Migrate plaintext passwords in the DB to scrypt. SHA256 passwords
     are left alone (they migrate on next login via the SHA256 fallback).
     Skips quickly if no plaintext passwords exist.
     """
+    if fips_mode_active():
+        return "Skipped, FIPS mode has no password migration"
     all_with_pw: Final = await UserRepository(prisma_client).table.find_many(
         where={"password": {"not": None}},
     )
@@ -8446,7 +8481,10 @@ async def migrate_passwords_to_scrypt_async(prisma_client) -> str:
     plaintext_users: Final = [
         (u.user_id, u.password)
         for u in all_with_pw
-        if u.password and not u.password.startswith("scrypt:") and not _is_sha256_hex(u.password)
+        if u.password
+        and not u.password.startswith("scrypt:")
+        and not _is_sha256_hex(u.password)
+        and not _is_pbkdf2_row(u.password)
     ]
     if not plaintext_users:
         return "No plaintext passwords found"

tests/unit/proxy/auth/test_login_utils.py

@@ -5,6 +5,7 @@ This module tests the refactored login logic that was moved from proxy_server.py
 to login_utils.py for better reusability.
 """
 
+import asyncio
 import hashlib
 import os
 from collections.abc import Mapping
@@ -2365,3 +2366,160 @@ class TestScreenLoginPasswordForBreach:
             )
             is True
         )
+
+
+_PBKDF2_PASSWORD: Final = "Seed-Lamp-4-River"
+_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T"
+_SCRYPT_ROW: Final = "scrypt:AAECAwQFBgcICQoLDA0OD7hIwlM/k8ytnffjFDt6AuO+BvqtmZvDkWXJP+A3bf0N"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "stored, schemes_written",
+    [
+        (_SCRYPT_ROW, []),
+        (_PBKDF2_ROW, []),
+        (hashlib.sha256(_PBKDF2_PASSWORD.encode()).hexdigest(), ["scrypt"]),
+    ],
+    ids=["scrypt", "pbkdf2", "sha256"],
+)
+async def test_login_rehash_rewrites_only_a_legacy_row(stored: str, schemes_written: list[str]) -> None:
+    from litellm.proxy.auth.login_utils import _rehash_password_if_needed
+
+    prisma: Final = MagicMock()
+    prisma.db.litellm_usertable.update = AsyncMock()
+    with patch(  # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam
+        "litellm.proxy.proxy_server.prisma_client", prisma
+    ):
+        await _rehash_password_if_needed("u-1", _PBKDF2_PASSWORD, stored)
+
+    updates: Final = prisma.db.litellm_usertable.update.await_args_list
+    assert [update.kwargs["data"]["password"].split(":")[0] for update in updates] == schemes_written
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_login_runs_one_pbkdf2_off_the_event_loop_and_writes_no_row(
+    monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool]
+) -> None:
+    from litellm.proxy.auth.login_utils import authenticate_user
+
+    monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
+    repository: Final = _known_user("user@corp.com")
+    repository.return_value.table.find_first.return_value.password = _PBKDF2_ROW
+    repository.return_value.table.update = AsyncMock()
+    with (
+        patch(  # test-quality-ok: the user lookup and rehash write are the database boundary
+            "litellm.proxy.auth.login_utils.UserRepository", repository
+        ),
+        patch(  # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", MagicMock()
+        ),
+        patch(  # test-quality-ok: success mints a UI key; faked so no DB is needed
+            "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"})
+        ),
+    ):
+        result: Final = await authenticate_user(
+            username="user@corp.com",
+            password=_PBKDF2_PASSWORD,
+            master_key="sk-master",
+            prisma_client=MagicMock(),
+            throttle=_unlimited_throttle(),
+            general_settings=_POLICY_NO_BREACH_CHECK,
+        )
+
+    assert result.key == "sk-ui"
+    assert pbkdf2_on_event_loop == [False]
+    repository.return_value.table.update.assert_not_awaited()
+
+
+async def _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch: pytest.MonkeyPatch, row: str):
+    from litellm.proxy.auth.login_utils import authenticate_user
+
+    throttle: Final = _throttle(user_limit=1)
+    repository: Final = _known_user()
+    user: Final = repository.return_value.table.find_first.return_value
+    user.password = row
+
+    async def block_the_user_then_return_the_row(**_: object) -> MagicMock:
+        await throttle.record_failure("known@example.com")
+        await throttle.record_failure("known@example.com")
+        return user
+
+    repository.return_value.table.find_first = AsyncMock(side_effect=block_the_user_then_return_the_row)
+    monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
+    with (
+        patch(  # test-quality-ok: the user lookup is the database boundary
+            "litellm.proxy.auth.login_utils.UserRepository", repository
+        ),
+        patch(  # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", MagicMock()
+        ),
+        patch(  # test-quality-ok: success mints a UI key; faked so no DB is needed
+            "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"})
+        ),
+    ):
+        return await authenticate_user(
+            username="known@example.com",
+            password=_PBKDF2_PASSWORD,
+            master_key="sk-master",
+            prisma_client=MagicMock(),
+            throttle=throttle,
+            general_settings=_POLICY_NO_BREACH_CHECK,
+        )
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_login_hides_the_right_password_when_its_user_got_blocked_while_it_waited(
+    monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool]
+) -> None:
+    from litellm.proxy._types import ProxyException
+
+    with pytest.raises(ProxyException) as blocked:
+        await _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch, _PBKDF2_ROW)
+
+    assert blocked.value.code == "429"
+    assert pbkdf2_on_event_loop == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_burst_of_wrong_passwords_stops_deriving_once_its_user_is_blocked(
+    monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool]
+) -> None:
+    from litellm.proxy.auth.login_utils import authenticate_user
+
+    monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
+    repository: Final = _known_user()
+    repository.return_value.table.find_first.return_value.password = _PBKDF2_ROW
+    throttle: Final = _throttle(user_limit=1)
+    with patch(  # test-quality-ok: the user lookup is the database boundary
+        "litellm.proxy.auth.login_utils.UserRepository", repository
+    ):
+        refusals: Final = await asyncio.gather(
+            *(
+                authenticate_user(
+                    username="known@example.com",
+                    password="Wrong-Lamp-4-River",
+                    master_key="sk-master",
+                    prisma_client=MagicMock(),
+                    throttle=throttle,
+                    general_settings=_POLICY_NO_BREACH_CHECK,
+                )
+                for _ in range(6)
+            ),
+            return_exceptions=True,
+        )
+
+    assert [refusal.code for refusal in refusals] == ["401", "401", "429", "429", "429", "429"]
+    assert pbkdf2_on_event_loop == [False, False]
+
+
+@pytest.mark.asyncio
+async def test_standard_login_past_the_throttle_keeps_its_answer_when_its_user_gets_blocked(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    result: Final = await _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch, _SCRYPT_ROW)
+
+    assert result.key == "sk-ui"

tests/unit/proxy/auth/test_onboarding.py

@@ -462,7 +462,7 @@ async def test_claim_token_sets_accepted_at_after_password_written():
     prisma.db.litellm_usertable.update.assert_called_once()
     call_kwargs = prisma.db.litellm_usertable.update.call_args
     assert call_kwargs.kwargs["where"] == {"user_id": "user-123"}
-    assert "password" in call_kwargs.kwargs["data"]
+    assert call_kwargs.kwargs["data"]["password"].startswith(("scrypt:", "pbkdf2:"))
     # A freshly claimed, policy-screened password lifts any pending forced
     # reset and re-arms the login-time breach screen.
     assert call_kwargs.kwargs["data"]["password_reset_required"] is False
@@ -477,6 +477,13 @@ async def test_claim_token_sets_accepted_at_after_password_written():
     assert outer_claims["key"] == "sk-generated-key"
 
 
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_claim_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None:
+    await test_claim_token_sets_accepted_at_after_password_written()
+    assert pbkdf2_on_event_loop == [False]
+
+
 @pytest.mark.asyncio
 async def test_claim_token_revokes_existing_ui_sessions():
     """A claimed invite/reset link changes the password; any UI session minted

tests/unit/proxy/auth/test_password_policy.py

@@ -9,6 +9,7 @@ httpx.MockTransport, so no network is touched and nothing is monkeypatched.
 
 import asyncio
 import hashlib
+from typing import Final
 
 import httpx
 import pytest
@@ -20,6 +21,7 @@ from litellm.proxy.auth.password_policy import (
     MIN_ALLOWED_LENGTH,
     PasswordPolicy,
     get_password_policy,
+    is_password_breached,
     validate_password_not_breached,
     validate_password_policy,
     validate_passwords_bulk,
@@ -355,3 +357,10 @@ async def test_validate_passwords_bulk_mixed_verdicts():
 async def test_validate_passwords_bulk_empty_batch_makes_no_lookups():
     verdicts = await validate_passwords_bulk((), {}, client=_client_never_called())
     assert verdicts == {}
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_mode_keeps_the_breach_check() -> None:
+    client_says_breached: Final = _client_returning(f"{_sha1_upper(STRONG_PASSWORD)[5:]}:1387")
+    assert await is_password_breached(STRONG_PASSWORD, {}, client_says_breached) is True

tests/unit/proxy/common_utils/test_fips.py

@@ -1,5 +1,5 @@
 import _hashlib
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
 from contextlib import nullcontext
 from unittest.mock import Mock
 
@@ -12,10 +12,17 @@ from litellm.proxy.common_utils.fips import (
     FipsModeError,
     cryptography_enforces_fips,
     enforce_fips_mode,
+    fips_mode_active,
     openssl_enforces_fips,
 )
 
 
+@pytest.fixture(autouse=True)
+def _fips_mode_off_after_each_test() -> Iterator[None]:
+    yield
+    enforce_fips_mode({})
+
+
 def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, scrypt_refused: bool) -> None:
     monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag)
     if scrypt_refused:
@@ -119,3 +126,29 @@ def test_enforce_fips_mode(
 
     with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext():
         enforce_fips_mode(general_settings)
+
+
+@pytest.mark.parametrize(
+    "general_settings, active",
+    [({"fips_mode": True}, True), ({"fips_mode": "true"}, True), ({"fips_mode": False}, False), ({}, False)],
+    ids=["on", "on-from-a-string", "off", "unset"],
+)
+def test_fips_mode_active_follows_the_setting_the_gate_accepted(
+    monkeypatch: pytest.MonkeyPatch, general_settings: Mapping[str, object], active: bool
+) -> None:
+    _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True)
+    monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True)
+    enforce_fips_mode(general_settings)
+    assert fips_mode_active() is active
+
+
+def test_fips_mode_stays_inactive_on_a_refused_boot(monkeypatch: pytest.MonkeyPatch) -> None:
+    _fake_openssl(monkeypatch, fips_flag=0, scrypt_refused=False)
+    with pytest.raises(FipsModeError):
+        enforce_fips_mode({"fips_mode": True})
+    assert fips_mode_active() is False
+
+
+def test_a_fips_openssl_alone_does_not_activate_fips_mode(monkeypatch: pytest.MonkeyPatch) -> None:
+    _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True)
+    assert fips_mode_active() is False

tests/unit/proxy/conftest.py

@@ -1,7 +1,9 @@
 # conftest.py
 
+import _hashlib
 import asyncio
 import copy
+import hashlib
 import inspect
 import logging
 import os
@@ -21,6 +23,7 @@ from prisma.errors import ClientNotConnectedError
 import litellm
 import litellm.proxy.proxy_server
 from litellm._logging import verbose_proxy_logger
+from litellm.proxy.common_utils.fips import enforce_fips_mode
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     InitPassThroughEndpointHelpers,
     _registered_pass_through_routes,
@@ -514,3 +517,31 @@ def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]:
 
     with patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock(service_logging_obj=service_logging)):
         yield rendered
+
+
+@pytest.fixture
+def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1)
+    monkeypatch.setattr(_hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported")))
+
+
+@pytest.fixture
+def fips_mode(fips_openssl: None, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True)
+    enforce_fips_mode({"fips_mode": True})
+    yield
+    enforce_fips_mode({})
+
+
+@pytest.fixture
+def pbkdf2_on_event_loop(monkeypatch: pytest.MonkeyPatch) -> list[bool]:
+    """One entry per PBKDF2 call: whether it ran on the event loop's thread."""
+    real_pbkdf2: Final = hashlib.pbkdf2_hmac
+    on_event_loop: Final[list[bool]] = []
+
+    def recording_pbkdf2(hash_name: str, password: bytes, salt: bytes, iterations: int, dklen: int) -> bytes:
+        on_event_loop.append(asyncio._get_running_loop() is not None)
+        return real_pbkdf2(hash_name, password, salt, iterations, dklen)
+
+    monkeypatch.setattr(hashlib, "pbkdf2_hmac", recording_pbkdf2)
+    return on_event_loop

tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py

@@ -5483,3 +5483,16 @@ async def test_user_update_reports_a_bad_budget_duration_as_a_request_error_not_
     assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
     assert "Invalid budget_duration" in exc_info.value.message
     assert "Authentication Error" not in exc_info.value.message
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_user_update_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None:
+    from litellm.proxy.management_endpoints.internal_user_endpoints import _hash_password_in_dict
+
+    data: Final = {"password": "Str0ng!Passw0rd"}
+
+    await _hash_password_in_dict(data, {"password_policy_check_breached_passwords": False})
+
+    assert pbkdf2_on_event_loop == [False]
+    assert data["password"].startswith("pbkdf2:")

tests/unit/proxy/management_endpoints/test_password_endpoints.py

@@ -490,3 +490,30 @@ async def test_change_password_requires_db():
             )
 
     assert exc_info.value.status_code == 500
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_password_change_runs_both_pbkdf2_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None:
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma: Final = _make_prisma(
+        _make_user_row("pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T")
+    )
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+    ):
+        await change_password(
+            data=ChangePasswordRequest(current_password="Seed-Lamp-4-River", new_password=NEW_PASSWORD),
+            user_api_key_dict=_caller(),
+            hibp_client=_hibp_client_never_called(),
+        )
+
+    assert pbkdf2_on_event_loop == [False, False]
+    assert prisma.db.litellm_usertable.update.call_args.kwargs["data"]["password"].startswith("pbkdf2:")

tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py

@@ -4,6 +4,7 @@ Symbols pinned here:
   - ``hash_token``
   - ``hash_password``
   - ``verify_password``
+  - ``run_off_loop_if_fips``
   - ``migrate_passwords_to_scrypt_async``
   - ``_hash_token_if_needed``
   - ``PrismaClient._is_sha256_hex`` (a nested helper inside
@@ -13,9 +14,12 @@ Symbols pinned here:
 
 from __future__ import annotations
 
+import asyncio
 import hashlib
+import ssl
+import threading
 from types import SimpleNamespace
-from typing import List
+from typing import Final, List
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
@@ -25,9 +29,17 @@ from litellm.proxy.utils import (
     hash_password,
     hash_token,
     migrate_passwords_to_scrypt_async,
+    run_off_loop_if_fips,
     verify_password,
 )
 
+PASSWORD: Final = "Seed-Lamp-4-River"
+PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T"
+EIGHT_CHARACTER_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0ODxhpezknSiGV0cY8wgaSXYf/xzxcPw49uGdOt/TBvPPH"
+SEVEN_CHARACTER_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0ODxFIwl/XKIAeyWEnfqz8H9vNQX4uo7/u1HiJhB7Ro+N7"
+SCRYPT_ROW: Final = "scrypt:AAECAwQFBgcICQoLDA0OD7hIwlM/k8ytnffjFDt6AuO+BvqtmZvDkWXJP+A3bf0N"
+SHA256_ROW: Final = hashlib.sha256(PASSWORD.encode()).hexdigest()
+
 
 def test_hash_token_returns_sha256_hex_of_input() -> None:
     token = "sk-abcDEF12345"
@@ -217,3 +229,133 @@ async def test_migrate_passwords_raises_on_db_failure() -> None:
     pc.db.litellm_usertable.find_many = AsyncMock(side_effect=RuntimeError("db unavailable"))
     with pytest.raises(RuntimeError, match="db unavailable"):
         await migrate_passwords_to_scrypt_async(pc)
+
+
+@pytest.mark.usefixtures("fips_openssl")
+def test_a_fips_openssl_without_fips_mode_stays_on_scrypt() -> None:
+    assert hash_password(PASSWORD).startswith("scrypt:")
+
+
+@pytest.mark.usefixtures("fips_mode")
+def test_fips_hash_is_pbkdf2_sha256_at_600k_salted_by_openssl(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(ssl, "RAND_bytes", lambda size: bytes(range(size)))
+    assert hash_password(PASSWORD) == PBKDF2_ROW
+
+
+@pytest.mark.usefixtures("fips_mode")
+def test_fips_hash_salts_every_row_and_verifies() -> None:
+    first: Final = hash_password(PASSWORD)
+    assert first != hash_password(PASSWORD)
+    assert verify_password(PASSWORD, first)
+
+
+@pytest.mark.usefixtures("fips_mode")
+def test_fips_verify_needs_the_right_password_of_at_least_eight_characters() -> None:
+    outcome: Final = {
+        "right": verify_password(PASSWORD, PBKDF2_ROW),
+        "wrong": verify_password("Seed-Lamp-4-Rivet", PBKDF2_ROW),
+        "lone_surrogate": verify_password("\ud800" + PASSWORD, PBKDF2_ROW),
+        "eight_characters": verify_password("Seed-Lam", EIGHT_CHARACTER_PBKDF2_ROW),
+        "seven_characters": verify_password("Seed-La", SEVEN_CHARACTER_PBKDF2_ROW),
+    }
+    assert outcome == {
+        "right": True,
+        "wrong": False,
+        "lone_surrogate": False,
+        "eight_characters": True,
+        "seven_characters": False,
+    }
+
+
+@pytest.mark.usefixtures("fips_mode")
+@pytest.mark.parametrize(
+    "row",
+    [
+        SCRYPT_ROW,
+        SHA256_ROW,
+        PASSWORD,
+        "pbkdf2:",
+        PBKDF2_ROW[:-1],
+        PBKDF2_ROW + "A",
+        PBKDF2_ROW + "\n",
+        "pbkdf2:" + "@" * 64,
+        "pbkdf2:sha256:1000:AAECAwQFBgcICQoLDA0ODw==:67GE2pGj/dkUsRooBGoUFP43cqp6T9IYz/kC+hxI3RM=",
+    ],
+    ids=[
+        "scrypt",
+        "sha256",
+        "plaintext",
+        "prefix-only",
+        "short",
+        "long",
+        "newline",
+        "not-base64",
+        "stored-iterations",
+    ],
+)
+def test_fips_refuses_any_other_row_with_one_error_that_hides_it(caplog: pytest.LogCaptureFixture, row: str) -> None:
+    with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
+        assert verify_password(PASSWORD, row) is False
+    assert [record.levelname for record in caplog.records] == ["ERROR"]
+    assert row not in caplog.text
+
+
+@pytest.mark.parametrize(
+    "row, verifies",
+    [(SCRYPT_ROW, True), (SHA256_ROW, True), (PBKDF2_ROW, False)],
+    ids=["scrypt", "sha256", "pbkdf2"],
+)
+def test_standard_mode_verifies_the_rows_it_always_did(row: str, verifies: bool) -> None:
+    assert verify_password(PASSWORD, row) is verifies
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_boot_migration_writes_no_row() -> None:
+    pc: Final = MagicMock()
+    pc.db.litellm_usertable.find_many = AsyncMock(
+        return_value=[_make_user("plaintext-user", PASSWORD), _make_user("fips-user", PBKDF2_ROW)]
+    )
+    pc.db.litellm_usertable.update = AsyncMock()
+
+    await migrate_passwords_to_scrypt_async(pc)
+
+    pc.db.litellm_usertable.update.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_standard_boot_migration_hashes_a_plaintext_row_but_not_a_pbkdf2_row() -> None:
+    pc: Final = MagicMock()
+    pc.db.litellm_usertable.find_many = AsyncMock(
+        return_value=[_make_user("plaintext-user", "pbkdf2:plaintext"), _make_user("fips-user", PBKDF2_ROW)]
+    )
+    pc.db.litellm_usertable.update = AsyncMock()
+
+    await migrate_passwords_to_scrypt_async(pc)
+
+    updated: Final = pc.db.litellm_usertable.update.await_args_list
+    assert [update.kwargs["where"]["user_id"] for update in updated] == ["plaintext-user"]
+
+
+def _on_event_loop() -> bool:
+    return asyncio._get_running_loop() is not None
+
+
+@pytest.mark.asyncio
+async def test_password_kdf_runs_on_the_event_loop_in_standard_mode() -> None:
+    assert await run_off_loop_if_fips(_on_event_loop) is True
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_password_kdfs_leave_the_event_loop_on_a_single_thread_in_fips_mode() -> None:
+    release_first: Final = threading.Event()
+    first_kdf: Final = asyncio.ensure_future(run_off_loop_if_fips(lambda: release_first.wait(timeout=5)))
+    second_kdf: Final = asyncio.ensure_future(run_off_loop_if_fips(_on_event_loop))
+    await asyncio.sleep(0)
+    kdf_threads: Final = [thread for thread in threading.enumerate() if thread.name.startswith("fips-password-kdf")]
+    release_first.set()
+
+    assert len(kdf_threads) == 1
+    assert await first_kdf is True
+    assert await second_kdf is False