Merge 0b8d9ad890 into 9f489430a6

litellm/proxy/auth/login_throttle.py

@@ -431,6 +431,10 @@ class LoginAttempt:
     throttle: LoginThrottle
     username: str
 
+    async def refuse_if_blocked_while_waiting(self) -> None:
+        """Refuses a login that waited in line while failures ahead of it blocked this key."""
+        await self.throttle.attempt(self.username)
+
     async def succeeded(self) -> None:
         if not self.throttle.enabled:
             return

litellm/proxy/auth/login_utils.py

@@ -5,6 +5,7 @@ This module contains the core login logic that can be reused across different
 login endpoints (e.g., /login and /v2/login).
 """
 
+import asyncio
 import os
 import secrets
 from collections.abc import Mapping
@@ -31,6 +32,7 @@ from litellm.proxy._types import (
 from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured
 from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle
 from litellm.proxy.auth.password_policy import is_breach_check_enabled, is_password_breached
+from litellm.proxy.common_utils.fips import fips_mode_active
 from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
 from litellm.proxy.management_endpoints.key_management_endpoints import (
     generate_key_helper_fn,
@@ -42,6 +44,7 @@ from litellm.proxy.utils import (
     PrismaClient,
     get_server_root_path,
     hash_password,
+    run_off_loop_if_fips,
     verify_password,
 )
 from litellm.repositories.user_repository import UserRepository
@@ -105,9 +108,21 @@ async def screen_login_password_for_breach(
     return breached
 
 
+_ONE_FIPS_LOGIN_AT_A_TIME: Final = asyncio.Lock()
+
+
+async def _password_matches(password: str, stored: str, attempt: LoginAttempt) -> bool:
+    """Checks inline, or in FIPS mode one login at a time, refusing a login whose user got blocked while it waited."""
+    if not fips_mode_active():
+        return verify_password(password, stored)
+    async with _ONE_FIPS_LOGIN_AT_A_TIME:
+        await attempt.refuse_if_blocked_while_waiting()
+        return await run_off_loop_if_fips(verify_password, password, stored)
+
+
 async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None:
     """Rehash legacy password (SHA256) to scrypt on successful login."""
-    if stored.startswith("scrypt:"):
+    if stored.startswith(("scrypt:", "pbkdf2:")):
         return
     from litellm.proxy.proxy_server import prisma_client
 
@@ -411,7 +426,7 @@ async def _sign_in(
                 code=401,
             )
 
-        if verify_password(password, _password):
+        if await _password_matches(password, _password, attempt):
             await _rehash_password_if_needed(_user_row.user_id, password, _password)
             breached_now: Final = prisma_client is not None and await screen_login_password_for_breach(
                 user_id=_user_row.user_id,

litellm/proxy/common_utils/fips.py

@@ -1,3 +1,4 @@
+import threading
 from collections.abc import Mapping
 from typing import Final
 
@@ -5,6 +6,7 @@ from litellm.secret_managers.main import str_to_bool
 
 FIPS_MODE_SETTING: Final = "fips_mode"
 _SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}"
+_FIPS_MODE_ACTIVE: Final = threading.Event()
 
 
 class FipsModeError(Exception):
@@ -87,3 +89,12 @@ def enforce_fips_mode(general_settings: Mapping[str, object]) -> None:
     refusal: Final = fips_mode_refusal(general_settings)
     if refusal is not None:
         raise FipsModeError(refusal)
+    if _fips_mode_setting(general_settings):
+        _FIPS_MODE_ACTIVE.set()
+    else:
+        _FIPS_MODE_ACTIVE.clear()
+
+
+def fips_mode_active() -> bool:
+    """Whether general_settings.fips_mode was on when enforce_fips_mode last accepted it."""
+    return _FIPS_MODE_ACTIVE.is_set()

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -85,7 +85,7 @@ from litellm.proxy.management_helpers.object_permission_utils import (  # noqa:
 )
 from litellm.proxy.management_helpers.team_roster_sync import user_tx_db
 from litellm.proxy.management_helpers.utils import management_endpoint_wrapper
-from litellm.proxy.utils import handle_exception_on_proxy, hash_password
+from litellm.proxy.utils import handle_exception_on_proxy, hash_password, run_off_loop_if_fips
 from litellm.repositories.organization_repository import OrganizationRepository
 from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import (
@@ -207,7 +207,7 @@ async def _hash_password_in_dict(
         if not password_prevalidated:
             validate_password_policy(data["password"], general_settings)
             await validate_password_not_breached(data["password"], general_settings, hibp_client)
-        data["password"] = hash_password(data["password"])
+        data["password"] = await run_off_loop_if_fips(hash_password, data["password"])
         data["password_reset_required"] = True
         data["last_breach_check_at"] = None
 

litellm/proxy/management_endpoints/password_endpoints.py

@@ -33,7 +33,7 @@ from litellm.proxy.auth.password_policy import (
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.management_endpoints.session_endpoints import revoke_ui_session_keys
 from litellm.proxy.management_helpers.audit_logs import create_object_audit_log
-from litellm.proxy.utils import hash_password, verify_password
+from litellm.proxy.utils import hash_password, run_off_loop_if_fips, verify_password
 from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.user_repository import UserRepository
 
@@ -129,7 +129,7 @@ async def change_password(
             ),
         )
 
-    if not verify_password(data.current_password, stored_password):
+    if not await run_off_loop_if_fips(verify_password, data.current_password, stored_password):
         raise HTTPException(status_code=400, detail=_error_detail("Current password is incorrect."))
 
     if data.new_password == data.current_password:
@@ -142,7 +142,7 @@ async def change_password(
     await validate_password_not_breached(data.new_password, general_settings, hibp_client)
 
     password_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = {
-        "password": hash_password(data.new_password),
+        "password": await run_off_loop_if_fips(hash_password, data.new_password),
         "password_reset_required": False,
         "last_breach_check_at": None,
     }

litellm/proxy/proxy_server.py

@@ -1031,6 +1031,7 @@ from litellm.proxy.utils import (  # noqa: F401, RUF100  # legacy module exports
     migrate_passwords_to_scrypt_async,
     model_dump_with_preserved_fields,
     prefetch_config_params,
+    run_off_loop_if_fips,
     update_spend,
 )
 from litellm.proxy.video_endpoints.endpoints import router as video_router
@@ -19863,7 +19864,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request):
 
     validate_password_policy(data.password, general_settings)
     await validate_password_not_breached(data.password, general_settings)
-    hashed_pw: Final = hash_password(data.password)
+    hashed_pw: Final = await run_off_loop_if_fips(hash_password, data.password)
     current_time = litellm.utils.get_utc_datetime()
     tx_manager: Final[_InvitationAcceptTxManager] = prisma_client.db.tx()
     async with tx_manager as tx:

litellm/proxy/utils.py

@@ -6,6 +6,7 @@ import inspect
 import json
 import math
 import os
+import re
 import smtplib
 import ssl
 import sys
@@ -24,6 +25,7 @@ from collections.abc import (
     Mapping,
     Sequence,
 )
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import date, datetime, timedelta, timezone
 from email.mime.multipart import MIMEMultipart
@@ -85,7 +87,9 @@ from litellm.proxy._types import (
     SpendLogsMetadata,
     SpendLogsPayload,
 )
+from litellm.proxy.auth.password_policy import MIN_ALLOWED_LENGTH
 from litellm.proxy.bug_report_config import build_proxy_bug_report
+from litellm.proxy.common_utils.fips import fips_mode_active
 from litellm.proxy.common_utils.openai_error_payload import (
     attribute_of,
     error_status_code,
@@ -8399,23 +8403,42 @@ def hash_token(token: str):
     return hashed_token
 
 
+def _is_pbkdf2_row(stored: str) -> bool:
+    return re.fullmatch(r"pbkdf2:[A-Za-z0-9+/]{64}", stored) is not None
+
+
 def hash_password(password: str) -> str:
-    """Hash a password using scrypt with a random salt."""
+    """Hash a password using scrypt with a random salt, or PBKDF2 in FIPS mode."""
     import base64
     import hashlib
     import os
 
+    if fips_mode_active():
+        fips_salt: Final = ssl.RAND_bytes(16)
+        fips_dk: Final = hashlib.pbkdf2_hmac("sha256", password.encode(), fips_salt, iterations=600_000, dklen=32)
+        return "pbkdf2:" + base64.b64encode(fips_salt + fips_dk).decode()
     salt: Final = os.urandom(16)
     dk: Final = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32)
     return "scrypt:" + base64.b64encode(salt + dk).decode()
 
 
 def verify_password(password: str, stored: str) -> bool:
-    """Verify a password against a stored hash. Supports scrypt and SHA256."""
+    """Verify a password against a stored hash. Supports scrypt and SHA256, or only PBKDF2 in FIPS mode."""
     import base64
     import hashlib
     import secrets
 
+    if fips_mode_active():
+        if not _is_pbkdf2_row(stored):
+            verbose_proxy_logger.error("FIPS mode refuses a non-PBKDF2 password row. Reset it with /user/update.")
+            return False
+        if len(password) < MIN_ALLOWED_LENGTH:
+            return False
+        fips_row: Final = base64.b64decode(stored[7:])
+        fips_dk: Final = hashlib.pbkdf2_hmac(
+            "sha256", password.encode(errors="surrogatepass"), fips_row[:16], iterations=600_000, dklen=32
+        )
+        return secrets.compare_digest(fips_row[16:], fips_dk)
     if stored.startswith("scrypt:"):
         try:
             raw: Final = base64.b64decode(stored[7:])
@@ -8430,12 +8453,24 @@ def verify_password(password: str, stored: str) -> bool:
     return False
 
 
+_FIPS_KDF_POOL: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="fips-password-kdf")
+
+
+async def run_off_loop_if_fips(kdf: Callable[..., _T], *args: str) -> _T:
+    """Run a password KDF inline, or on the single KDF thread in FIPS mode."""
+    if not fips_mode_active():
+        return kdf(*args)
+    return await asyncio.get_running_loop().run_in_executor(_FIPS_KDF_POOL, kdf, *args)
+
+
 async def migrate_passwords_to_scrypt_async(prisma_client) -> str:
     """
     Migrate plaintext passwords in the DB to scrypt. SHA256 passwords
     are left alone (they migrate on next login via the SHA256 fallback).
     Skips quickly if no plaintext passwords exist.
     """
+    if fips_mode_active():
+        return "Skipped, FIPS mode has no password migration"
     all_with_pw: Final = await UserRepository(prisma_client).table.find_many(
         where={"password": {"not": None}},
     )
@@ -8446,7 +8481,10 @@ async def migrate_passwords_to_scrypt_async(prisma_client) -> str:
     plaintext_users: Final = [
         (u.user_id, u.password)
         for u in all_with_pw
-        if u.password and not u.password.startswith("scrypt:") and not _is_sha256_hex(u.password)
+        if u.password
+        and not u.password.startswith("scrypt:")
+        and not _is_sha256_hex(u.password)
+        and not _is_pbkdf2_row(u.password)
     ]
     if not plaintext_users:
         return "No plaintext passwords found"

tests/unit/proxy/auth/test_login_utils.py

@@ -5,6 +5,7 @@ This module tests the refactored login logic that was moved from proxy_server.py
 to login_utils.py for better reusability.
 """
 
+import asyncio
 import hashlib
 import os
 from collections.abc import Mapping
@@ -2365,3 +2366,160 @@ class TestScreenLoginPasswordForBreach:
             )
             is True
         )
+
+
+_PBKDF2_PASSWORD: Final = "Seed-Lamp-4-River"
+_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T"
+_SCRYPT_ROW: Final = "scrypt:AAECAwQFBgcICQoLDA0OD7hIwlM/k8ytnffjFDt6AuO+BvqtmZvDkWXJP+A3bf0N"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "stored, schemes_written",
+    [
+        (_SCRYPT_ROW, []),
+        (_PBKDF2_ROW, []),
+        (hashlib.sha256(_PBKDF2_PASSWORD.encode()).hexdigest(), ["scrypt"]),
+    ],
+    ids=["scrypt", "pbkdf2", "sha256"],
+)
+async def test_login_rehash_rewrites_only_a_legacy_row(stored: str, schemes_written: list[str]) -> None:
+    from litellm.proxy.auth.login_utils import _rehash_password_if_needed
+
+    prisma: Final = MagicMock()
+    prisma.db.litellm_usertable.update = AsyncMock()
+    with patch(  # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam
+        "litellm.proxy.proxy_server.prisma_client", prisma
+    ):
+        await _rehash_password_if_needed("u-1", _PBKDF2_PASSWORD, stored)
+
+    updates: Final = prisma.db.litellm_usertable.update.await_args_list
+    assert [update.kwargs["data"]["password"].split(":")[0] for update in updates] == schemes_written
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_login_runs_one_pbkdf2_off_the_event_loop_and_writes_no_row(
+    monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool]
+) -> None:
+    from litellm.proxy.auth.login_utils import authenticate_user
+
+    monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
+    repository: Final = _known_user("user@corp.com")
+    repository.return_value.table.find_first.return_value.password = _PBKDF2_ROW
+    repository.return_value.table.update = AsyncMock()
+    with (
+        patch(  # test-quality-ok: the user lookup and rehash write are the database boundary
+            "litellm.proxy.auth.login_utils.UserRepository", repository
+        ),
+        patch(  # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", MagicMock()
+        ),
+        patch(  # test-quality-ok: success mints a UI key; faked so no DB is needed
+            "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"})
+        ),
+    ):
+        result: Final = await authenticate_user(
+            username="user@corp.com",
+            password=_PBKDF2_PASSWORD,
+            master_key="sk-master",
+            prisma_client=MagicMock(),
+            throttle=_unlimited_throttle(),
+            general_settings=_POLICY_NO_BREACH_CHECK,
+        )
+
+    assert result.key == "sk-ui"
+    assert pbkdf2_on_event_loop == [False]
+    repository.return_value.table.update.assert_not_awaited()
+
+
+async def _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch: pytest.MonkeyPatch, row: str):
+    from litellm.proxy.auth.login_utils import authenticate_user
+
+    throttle: Final = _throttle(user_limit=1)
+    repository: Final = _known_user()
+    user: Final = repository.return_value.table.find_first.return_value
+    user.password = row
+
+    async def block_the_user_then_return_the_row(**_: object) -> MagicMock:
+        await throttle.record_failure("known@example.com")
+        await throttle.record_failure("known@example.com")
+        return user
+
+    repository.return_value.table.find_first = AsyncMock(side_effect=block_the_user_then_return_the_row)
+    monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
+    with (
+        patch(  # test-quality-ok: the user lookup is the database boundary
+            "litellm.proxy.auth.login_utils.UserRepository", repository
+        ),
+        patch(  # test-quality-ok: the rehash reads proxy_server's prisma_client global; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", MagicMock()
+        ),
+        patch(  # test-quality-ok: success mints a UI key; faked so no DB is needed
+            "litellm.proxy.auth.login_utils.generate_key_helper_fn", new=AsyncMock(return_value={"token": "sk-ui"})
+        ),
+    ):
+        return await authenticate_user(
+            username="known@example.com",
+            password=_PBKDF2_PASSWORD,
+            master_key="sk-master",
+            prisma_client=MagicMock(),
+            throttle=throttle,
+            general_settings=_POLICY_NO_BREACH_CHECK,
+        )
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_login_hides_the_right_password_when_its_user_got_blocked_while_it_waited(
+    monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool]
+) -> None:
+    from litellm.proxy._types import ProxyException
+
+    with pytest.raises(ProxyException) as blocked:
+        await _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch, _PBKDF2_ROW)
+
+    assert blocked.value.code == "429"
+    assert pbkdf2_on_event_loop == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_burst_of_wrong_passwords_stops_deriving_once_its_user_is_blocked(
+    monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool]
+) -> None:
+    from litellm.proxy.auth.login_utils import authenticate_user
+
+    monkeypatch.setenv("DATABASE_URL", "postgresql://stub")
+    repository: Final = _known_user()
+    repository.return_value.table.find_first.return_value.password = _PBKDF2_ROW
+    throttle: Final = _throttle(user_limit=1)
+    with patch(  # test-quality-ok: the user lookup is the database boundary
+        "litellm.proxy.auth.login_utils.UserRepository", repository
+    ):
+        refusals: Final = await asyncio.gather(
+            *(
+                authenticate_user(
+                    username="known@example.com",
+                    password="Wrong-Lamp-4-River",
+                    master_key="sk-master",
+                    prisma_client=MagicMock(),
+                    throttle=throttle,
+                    general_settings=_POLICY_NO_BREACH_CHECK,
+                )
+                for _ in range(6)
+            ),
+            return_exceptions=True,
+        )
+
+    assert [refusal.code for refusal in refusals] == ["401", "401", "429", "429", "429", "429"]
+    assert pbkdf2_on_event_loop == [False, False]
+
+
+@pytest.mark.asyncio
+async def test_standard_login_past_the_throttle_keeps_its_answer_when_its_user_gets_blocked(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    result: Final = await _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch, _SCRYPT_ROW)
+
+    assert result.key == "sk-ui"

tests/unit/proxy/auth/test_onboarding.py

@@ -462,7 +462,7 @@ async def test_claim_token_sets_accepted_at_after_password_written():
     prisma.db.litellm_usertable.update.assert_called_once()
     call_kwargs = prisma.db.litellm_usertable.update.call_args
     assert call_kwargs.kwargs["where"] == {"user_id": "user-123"}
-    assert "password" in call_kwargs.kwargs["data"]
+    assert call_kwargs.kwargs["data"]["password"].startswith(("scrypt:", "pbkdf2:"))
     # A freshly claimed, policy-screened password lifts any pending forced
     # reset and re-arms the login-time breach screen.
     assert call_kwargs.kwargs["data"]["password_reset_required"] is False
@@ -477,6 +477,13 @@ async def test_claim_token_sets_accepted_at_after_password_written():
     assert outer_claims["key"] == "sk-generated-key"
 
 
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_claim_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None:
+    await test_claim_token_sets_accepted_at_after_password_written()
+    assert pbkdf2_on_event_loop == [False]
+
+
 @pytest.mark.asyncio
 async def test_claim_token_revokes_existing_ui_sessions():
     """A claimed invite/reset link changes the password; any UI session minted

tests/unit/proxy/auth/test_password_policy.py

@@ -9,6 +9,7 @@ httpx.MockTransport, so no network is touched and nothing is monkeypatched.
 
 import asyncio
 import hashlib
+from typing import Final
 
 import httpx
 import pytest
@@ -20,6 +21,7 @@ from litellm.proxy.auth.password_policy import (
     MIN_ALLOWED_LENGTH,
     PasswordPolicy,
     get_password_policy,
+    is_password_breached,
     validate_password_not_breached,
     validate_password_policy,
     validate_passwords_bulk,
@@ -355,3 +357,10 @@ async def test_validate_passwords_bulk_mixed_verdicts():
 async def test_validate_passwords_bulk_empty_batch_makes_no_lookups():
     verdicts = await validate_passwords_bulk((), {}, client=_client_never_called())
     assert verdicts == {}
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_mode_keeps_the_breach_check() -> None:
+    client_says_breached: Final = _client_returning(f"{_sha1_upper(STRONG_PASSWORD)[5:]}:1387")
+    assert await is_password_breached(STRONG_PASSWORD, {}, client_says_breached) is True

tests/unit/proxy/common_utils/test_fips.py

@@ -1,5 +1,5 @@
 import _hashlib
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
 from contextlib import nullcontext
 from unittest.mock import Mock
 
@@ -12,10 +12,17 @@ from litellm.proxy.common_utils.fips import (
     FipsModeError,
     cryptography_enforces_fips,
     enforce_fips_mode,
+    fips_mode_active,
     openssl_enforces_fips,
 )
 
 
+@pytest.fixture(autouse=True)
+def _fips_mode_off_after_each_test() -> Iterator[None]:
+    yield
+    enforce_fips_mode({})
+
+
 def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, scrypt_refused: bool) -> None:
     monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag)
     if scrypt_refused:
@@ -119,3 +126,29 @@ def test_enforce_fips_mode(
 
     with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext():
         enforce_fips_mode(general_settings)
+
+
+@pytest.mark.parametrize(
+    "general_settings, active",
+    [({"fips_mode": True}, True), ({"fips_mode": "true"}, True), ({"fips_mode": False}, False), ({}, False)],
+    ids=["on", "on-from-a-string", "off", "unset"],
+)
+def test_fips_mode_active_follows_the_setting_the_gate_accepted(
+    monkeypatch: pytest.MonkeyPatch, general_settings: Mapping[str, object], active: bool
+) -> None:
+    _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True)
+    monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True)
+    enforce_fips_mode(general_settings)
+    assert fips_mode_active() is active
+
+
+def test_fips_mode_stays_inactive_on_a_refused_boot(monkeypatch: pytest.MonkeyPatch) -> None:
+    _fake_openssl(monkeypatch, fips_flag=0, scrypt_refused=False)
+    with pytest.raises(FipsModeError):
+        enforce_fips_mode({"fips_mode": True})
+    assert fips_mode_active() is False
+
+
+def test_a_fips_openssl_alone_does_not_activate_fips_mode(monkeypatch: pytest.MonkeyPatch) -> None:
+    _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True)
+    assert fips_mode_active() is False

tests/unit/proxy/conftest.py

@@ -1,7 +1,9 @@
 # conftest.py
 
+import _hashlib
 import asyncio
 import copy
+import hashlib
 import inspect
 import logging
 import os
@@ -21,6 +23,7 @@ from prisma.errors import ClientNotConnectedError
 import litellm
 import litellm.proxy.proxy_server
 from litellm._logging import verbose_proxy_logger
+from litellm.proxy.common_utils.fips import enforce_fips_mode
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     InitPassThroughEndpointHelpers,
     _registered_pass_through_routes,
@@ -514,3 +517,31 @@ def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]:
 
     with patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock(service_logging_obj=service_logging)):
         yield rendered
+
+
+@pytest.fixture
+def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1)
+    monkeypatch.setattr(_hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported")))
+
+
+@pytest.fixture
+def fips_mode(fips_openssl: None, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True)
+    enforce_fips_mode({"fips_mode": True})
+    yield
+    enforce_fips_mode({})
+
+
+@pytest.fixture
+def pbkdf2_on_event_loop(monkeypatch: pytest.MonkeyPatch) -> list[bool]:
+    """One entry per PBKDF2 call: whether it ran on the event loop's thread."""
+    real_pbkdf2: Final = hashlib.pbkdf2_hmac
+    on_event_loop: Final[list[bool]] = []
+
+    def recording_pbkdf2(hash_name: str, password: bytes, salt: bytes, iterations: int, dklen: int) -> bytes:
+        on_event_loop.append(asyncio._get_running_loop() is not None)
+        return real_pbkdf2(hash_name, password, salt, iterations, dklen)
+
+    monkeypatch.setattr(hashlib, "pbkdf2_hmac", recording_pbkdf2)
+    return on_event_loop

tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py

@@ -5483,3 +5483,16 @@ async def test_user_update_reports_a_bad_budget_duration_as_a_request_error_not_
     assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
     assert "Invalid budget_duration" in exc_info.value.message
     assert "Authentication Error" not in exc_info.value.message
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_user_update_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None:
+    from litellm.proxy.management_endpoints.internal_user_endpoints import _hash_password_in_dict
+
+    data: Final = {"password": "Str0ng!Passw0rd"}
+
+    await _hash_password_in_dict(data, {"password_policy_check_breached_passwords": False})
+
+    assert pbkdf2_on_event_loop == [False]
+    assert data["password"].startswith("pbkdf2:")

tests/unit/proxy/management_endpoints/test_password_endpoints.py

@@ -490,3 +490,30 @@ async def test_change_password_requires_db():
             )
 
     assert exc_info.value.status_code == 500
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_password_change_runs_both_pbkdf2_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None:
+    from litellm.proxy._types import ChangePasswordRequest
+
+    prisma: Final = _make_prisma(
+        _make_user_row("pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T")
+    )
+
+    with (
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.prisma_client", prisma
+        ),
+        patch(  # test-quality-ok: change_password reads proxy_server module globals; no injection seam
+            "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK
+        ),
+    ):
+        await change_password(
+            data=ChangePasswordRequest(current_password="Seed-Lamp-4-River", new_password=NEW_PASSWORD),
+            user_api_key_dict=_caller(),
+            hibp_client=_hibp_client_never_called(),
+        )
+
+    assert pbkdf2_on_event_loop == [False, False]
+    assert prisma.db.litellm_usertable.update.call_args.kwargs["data"]["password"].startswith("pbkdf2:")

tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py

@@ -4,6 +4,7 @@ Symbols pinned here:
   - ``hash_token``
   - ``hash_password``
   - ``verify_password``
+  - ``run_off_loop_if_fips``
   - ``migrate_passwords_to_scrypt_async``
   - ``_hash_token_if_needed``
   - ``PrismaClient._is_sha256_hex`` (a nested helper inside
@@ -13,9 +14,12 @@ Symbols pinned here:
 
 from __future__ import annotations
 
+import asyncio
 import hashlib
+import ssl
+import threading
 from types import SimpleNamespace
-from typing import List
+from typing import Final, List
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
@@ -25,9 +29,17 @@ from litellm.proxy.utils import (
     hash_password,
     hash_token,
     migrate_passwords_to_scrypt_async,
+    run_off_loop_if_fips,
     verify_password,
 )
 
+PASSWORD: Final = "Seed-Lamp-4-River"
+PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0OD+uxhNqRo/3ZFLEaKARqFBT+N3Kqek/SGM/5AvocSN0T"
+EIGHT_CHARACTER_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0ODxhpezknSiGV0cY8wgaSXYf/xzxcPw49uGdOt/TBvPPH"
+SEVEN_CHARACTER_PBKDF2_ROW: Final = "pbkdf2:AAECAwQFBgcICQoLDA0ODxFIwl/XKIAeyWEnfqz8H9vNQX4uo7/u1HiJhB7Ro+N7"
+SCRYPT_ROW: Final = "scrypt:AAECAwQFBgcICQoLDA0OD7hIwlM/k8ytnffjFDt6AuO+BvqtmZvDkWXJP+A3bf0N"
+SHA256_ROW: Final = hashlib.sha256(PASSWORD.encode()).hexdigest()
+
 
 def test_hash_token_returns_sha256_hex_of_input() -> None:
     token = "sk-abcDEF12345"
@@ -217,3 +229,133 @@ async def test_migrate_passwords_raises_on_db_failure() -> None:
     pc.db.litellm_usertable.find_many = AsyncMock(side_effect=RuntimeError("db unavailable"))
     with pytest.raises(RuntimeError, match="db unavailable"):
         await migrate_passwords_to_scrypt_async(pc)
+
+
+@pytest.mark.usefixtures("fips_openssl")
+def test_a_fips_openssl_without_fips_mode_stays_on_scrypt() -> None:
+    assert hash_password(PASSWORD).startswith("scrypt:")
+
+
+@pytest.mark.usefixtures("fips_mode")
+def test_fips_hash_is_pbkdf2_sha256_at_600k_salted_by_openssl(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(ssl, "RAND_bytes", lambda size: bytes(range(size)))
+    assert hash_password(PASSWORD) == PBKDF2_ROW
+
+
+@pytest.mark.usefixtures("fips_mode")
+def test_fips_hash_salts_every_row_and_verifies() -> None:
+    first: Final = hash_password(PASSWORD)
+    assert first != hash_password(PASSWORD)
+    assert verify_password(PASSWORD, first)
+
+
+@pytest.mark.usefixtures("fips_mode")
+def test_fips_verify_needs_the_right_password_of_at_least_eight_characters() -> None:
+    outcome: Final = {
+        "right": verify_password(PASSWORD, PBKDF2_ROW),
+        "wrong": verify_password("Seed-Lamp-4-Rivet", PBKDF2_ROW),
+        "lone_surrogate": verify_password("\ud800" + PASSWORD, PBKDF2_ROW),
+        "eight_characters": verify_password("Seed-Lam", EIGHT_CHARACTER_PBKDF2_ROW),
+        "seven_characters": verify_password("Seed-La", SEVEN_CHARACTER_PBKDF2_ROW),
+    }
+    assert outcome == {
+        "right": True,
+        "wrong": False,
+        "lone_surrogate": False,
+        "eight_characters": True,
+        "seven_characters": False,
+    }
+
+
+@pytest.mark.usefixtures("fips_mode")
+@pytest.mark.parametrize(
+    "row",
+    [
+        SCRYPT_ROW,
+        SHA256_ROW,
+        PASSWORD,
+        "pbkdf2:",
+        PBKDF2_ROW[:-1],
+        PBKDF2_ROW + "A",
+        PBKDF2_ROW + "\n",
+        "pbkdf2:" + "@" * 64,
+        "pbkdf2:sha256:1000:AAECAwQFBgcICQoLDA0ODw==:67GE2pGj/dkUsRooBGoUFP43cqp6T9IYz/kC+hxI3RM=",
+    ],
+    ids=[
+        "scrypt",
+        "sha256",
+        "plaintext",
+        "prefix-only",
+        "short",
+        "long",
+        "newline",
+        "not-base64",
+        "stored-iterations",
+    ],
+)
+def test_fips_refuses_any_other_row_with_one_error_that_hides_it(caplog: pytest.LogCaptureFixture, row: str) -> None:
+    with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
+        assert verify_password(PASSWORD, row) is False
+    assert [record.levelname for record in caplog.records] == ["ERROR"]
+    assert row not in caplog.text
+
+
+@pytest.mark.parametrize(
+    "row, verifies",
+    [(SCRYPT_ROW, True), (SHA256_ROW, True), (PBKDF2_ROW, False)],
+    ids=["scrypt", "sha256", "pbkdf2"],
+)
+def test_standard_mode_verifies_the_rows_it_always_did(row: str, verifies: bool) -> None:
+    assert verify_password(PASSWORD, row) is verifies
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_fips_boot_migration_writes_no_row() -> None:
+    pc: Final = MagicMock()
+    pc.db.litellm_usertable.find_many = AsyncMock(
+        return_value=[_make_user("plaintext-user", PASSWORD), _make_user("fips-user", PBKDF2_ROW)]
+    )
+    pc.db.litellm_usertable.update = AsyncMock()
+
+    await migrate_passwords_to_scrypt_async(pc)
+
+    pc.db.litellm_usertable.update.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_standard_boot_migration_hashes_a_plaintext_row_but_not_a_pbkdf2_row() -> None:
+    pc: Final = MagicMock()
+    pc.db.litellm_usertable.find_many = AsyncMock(
+        return_value=[_make_user("plaintext-user", "pbkdf2:plaintext"), _make_user("fips-user", PBKDF2_ROW)]
+    )
+    pc.db.litellm_usertable.update = AsyncMock()
+
+    await migrate_passwords_to_scrypt_async(pc)
+
+    updated: Final = pc.db.litellm_usertable.update.await_args_list
+    assert [update.kwargs["where"]["user_id"] for update in updated] == ["plaintext-user"]
+
+
+def _on_event_loop() -> bool:
+    return asyncio._get_running_loop() is not None
+
+
+@pytest.mark.asyncio
+async def test_password_kdf_runs_on_the_event_loop_in_standard_mode() -> None:
+    assert await run_off_loop_if_fips(_on_event_loop) is True
+
+
+@pytest.mark.asyncio
+@pytest.mark.usefixtures("fips_mode")
+async def test_password_kdfs_leave_the_event_loop_on_a_single_thread_in_fips_mode() -> None:
+    release_first: Final = threading.Event()
+    first_kdf: Final = asyncio.ensure_future(run_off_loop_if_fips(lambda: release_first.wait(timeout=5)))
+    second_kdf: Final = asyncio.ensure_future(run_off_loop_if_fips(_on_event_loop))
+    await asyncio.sleep(0)
+    kdf_threads: Final = [thread for thread in threading.enumerate() if thread.name.startswith("fips-password-kdf")]
+    release_first.set()
+
+    assert len(kdf_threads) == 1
+    assert await first_kdf is True
+    assert await second_kdf is False