fix(proxy): choose the password KDF from fips_mode instead of the host OpenSSL

This commit is contained in:
yucheng 2026-11-03 16:16:08 -08:00
parent 05e35d2122
commit 5e0e82bbaf
11 changed files with 78 additions and 20 deletions

View file

@ -32,7 +32,7 @@ from litellm.proxy._types import (
from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured
from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle
from litellm.proxy.auth.password_policy import is_breach_check_enabled, is_password_breached
from litellm.proxy.common_utils.fips import openssl_enforces_fips
from litellm.proxy.common_utils.fips import fips_mode_active
from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
from litellm.proxy.management_endpoints.key_management_endpoints import (
generate_key_helper_fn,
@ -113,7 +113,7 @@ _ONE_FIPS_LOGIN_AT_A_TIME: Final = asyncio.Lock()
async def _password_matches(password: str, stored: str, attempt: LoginAttempt) -> bool:
"""Checks inline, or in FIPS mode one login at a time, refusing a login whose user got blocked while it waited."""
if not openssl_enforces_fips():
if not fips_mode_active():
return verify_password(password, stored)
async with _ONE_FIPS_LOGIN_AT_A_TIME:
await attempt.refuse_if_blocked_while_waiting()

View file

@ -1,3 +1,4 @@
import threading
from collections.abc import Mapping
from typing import Final
@ -5,6 +6,7 @@ from litellm.secret_managers.main import str_to_bool
FIPS_MODE_SETTING: Final = "fips_mode"
_SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}"
_FIPS_MODE_ACTIVE: Final = threading.Event()
class FipsModeError(Exception):
@ -87,3 +89,12 @@ def enforce_fips_mode(general_settings: Mapping[str, object]) -> None:
refusal: Final = fips_mode_refusal(general_settings)
if refusal is not None:
raise FipsModeError(refusal)
if _fips_mode_setting(general_settings):
_FIPS_MODE_ACTIVE.set()
else:
_FIPS_MODE_ACTIVE.clear()
def fips_mode_active() -> bool:
"""Whether general_settings.fips_mode was on when enforce_fips_mode last accepted it."""
return _FIPS_MODE_ACTIVE.is_set()

View file

@ -89,7 +89,7 @@ from litellm.proxy._types import (
)
from litellm.proxy.auth.password_policy import MIN_ALLOWED_LENGTH
from litellm.proxy.bug_report_config import build_proxy_bug_report
from litellm.proxy.common_utils.fips import openssl_enforces_fips
from litellm.proxy.common_utils.fips import fips_mode_active
from litellm.proxy.common_utils.openai_error_payload import (
attribute_of,
error_status_code,
@ -8413,7 +8413,7 @@ def hash_password(password: str) -> str:
import hashlib
import os
if openssl_enforces_fips():
if fips_mode_active():
fips_salt: Final = ssl.RAND_bytes(16)
fips_dk: Final = hashlib.pbkdf2_hmac("sha256", password.encode(), fips_salt, iterations=600_000, dklen=32)
return "pbkdf2:" + base64.b64encode(fips_salt + fips_dk).decode()
@ -8428,7 +8428,7 @@ def verify_password(password: str, stored: str) -> bool:
import hashlib
import secrets
if openssl_enforces_fips():
if fips_mode_active():
if not _is_pbkdf2_row(stored):
verbose_proxy_logger.error("FIPS mode refuses a non-PBKDF2 password row. Reset it with /user/update.")
return False
@ -8458,7 +8458,7 @@ _FIPS_KDF_POOL: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="fi
async def run_off_loop_if_fips(kdf: Callable[..., _T], *args: str) -> _T:
"""Run a password KDF inline, or on the single KDF thread in FIPS mode."""
if not openssl_enforces_fips():
if not fips_mode_active():
return kdf(*args)
return await asyncio.get_running_loop().run_in_executor(_FIPS_KDF_POOL, kdf, *args)
@ -8469,7 +8469,7 @@ async def migrate_passwords_to_scrypt_async(prisma_client) -> str:
are left alone (they migrate on next login via the SHA256 fallback).
Skips quickly if no plaintext passwords exist.
"""
if openssl_enforces_fips():
if fips_mode_active():
return "Skipped, FIPS mode has no password migration"
all_with_pw: Final = await UserRepository(prisma_client).table.find_many(
where={"password": {"not": None}},

View file

@ -2398,7 +2398,7 @@ async def test_login_rehash_rewrites_only_a_legacy_row(stored: str, schemes_writ
@pytest.mark.asyncio
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
async def test_fips_login_runs_one_pbkdf2_off_the_event_loop_and_writes_no_row(
monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool]
) -> None:
@ -2470,7 +2470,7 @@ async def _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch: pyte
@pytest.mark.asyncio
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
async def test_fips_login_hides_the_right_password_when_its_user_got_blocked_while_it_waited(
monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool]
) -> None:
@ -2484,7 +2484,7 @@ async def test_fips_login_hides_the_right_password_when_its_user_got_blocked_whi
@pytest.mark.asyncio
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
async def test_fips_burst_of_wrong_passwords_stops_deriving_once_its_user_is_blocked(
monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool]
) -> None:

View file

@ -478,7 +478,7 @@ async def test_claim_token_sets_accepted_at_after_password_written():
@pytest.mark.asyncio
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
async def test_fips_claim_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None:
await test_claim_token_sets_accepted_at_after_password_written()
assert pbkdf2_on_event_loop == [False]

View file

@ -360,7 +360,7 @@ async def test_validate_passwords_bulk_empty_batch_makes_no_lookups():
@pytest.mark.asyncio
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
async def test_fips_mode_keeps_the_breach_check() -> None:
client_says_breached: Final = _client_returning(f"{_sha1_upper(STRONG_PASSWORD)[5:]}:1387")
assert await is_password_breached(STRONG_PASSWORD, {}, client_says_breached) is True

View file

@ -1,5 +1,5 @@
import _hashlib
from collections.abc import Mapping
from collections.abc import Iterator, Mapping
from contextlib import nullcontext
from unittest.mock import Mock
@ -12,10 +12,17 @@ from litellm.proxy.common_utils.fips import (
FipsModeError,
cryptography_enforces_fips,
enforce_fips_mode,
fips_mode_active,
openssl_enforces_fips,
)
@pytest.fixture(autouse=True)
def _fips_mode_off_after_each_test() -> Iterator[None]:
yield
enforce_fips_mode({})
def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, scrypt_refused: bool) -> None:
monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag)
if scrypt_refused:
@ -119,3 +126,29 @@ def test_enforce_fips_mode(
with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext():
enforce_fips_mode(general_settings)
@pytest.mark.parametrize(
"general_settings, active",
[({"fips_mode": True}, True), ({"fips_mode": "true"}, True), ({"fips_mode": False}, False), ({}, False)],
ids=["on", "on-from-a-string", "off", "unset"],
)
def test_fips_mode_active_follows_the_setting_the_gate_accepted(
monkeypatch: pytest.MonkeyPatch, general_settings: Mapping[str, object], active: bool
) -> None:
_fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True)
monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True)
enforce_fips_mode(general_settings)
assert fips_mode_active() is active
def test_fips_mode_stays_inactive_on_a_refused_boot(monkeypatch: pytest.MonkeyPatch) -> None:
_fake_openssl(monkeypatch, fips_flag=0, scrypt_refused=False)
with pytest.raises(FipsModeError):
enforce_fips_mode({"fips_mode": True})
assert fips_mode_active() is False
def test_a_fips_openssl_alone_does_not_activate_fips_mode(monkeypatch: pytest.MonkeyPatch) -> None:
_fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True)
assert fips_mode_active() is False

View file

@ -23,6 +23,7 @@ from prisma.errors import ClientNotConnectedError
import litellm
import litellm.proxy.proxy_server
from litellm._logging import verbose_proxy_logger
from litellm.proxy.common_utils.fips import enforce_fips_mode
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
InitPassThroughEndpointHelpers,
_registered_pass_through_routes,
@ -531,6 +532,14 @@ def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(_hashlib, "new", new)
@pytest.fixture
def fips_mode(fips_openssl: None, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True)
enforce_fips_mode({"fips_mode": True})
yield
enforce_fips_mode({})
@pytest.fixture
def pbkdf2_on_event_loop(monkeypatch: pytest.MonkeyPatch) -> list[bool]:
"""One entry per PBKDF2 call: whether it ran on the event loop's thread."""

View file

@ -5486,7 +5486,7 @@ async def test_user_update_reports_a_bad_budget_duration_as_a_request_error_not_
@pytest.mark.asyncio
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
async def test_fips_user_update_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None:
from litellm.proxy.management_endpoints.internal_user_endpoints import _hash_password_in_dict

View file

@ -493,7 +493,7 @@ async def test_change_password_requires_db():
@pytest.mark.asyncio
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
async def test_fips_password_change_runs_both_pbkdf2_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None:
from litellm.proxy._types import ChangePasswordRequest

View file

@ -232,19 +232,24 @@ async def test_migrate_passwords_raises_on_db_failure() -> None:
@pytest.mark.usefixtures("fips_openssl")
def test_a_fips_openssl_without_fips_mode_stays_on_scrypt() -> None:
assert hash_password(PASSWORD).startswith("scrypt:")
@pytest.mark.usefixtures("fips_mode")
def test_fips_hash_is_pbkdf2_sha256_at_600k_salted_by_openssl(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ssl, "RAND_bytes", lambda size: bytes(range(size)))
assert hash_password(PASSWORD) == PBKDF2_ROW
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
def test_fips_hash_salts_every_row_and_verifies() -> None:
first: Final = hash_password(PASSWORD)
assert first != hash_password(PASSWORD)
assert verify_password(PASSWORD, first)
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
def test_fips_verify_needs_the_right_password_of_at_least_eight_characters() -> None:
outcome: Final = {
"right": verify_password(PASSWORD, PBKDF2_ROW),
@ -262,7 +267,7 @@ def test_fips_verify_needs_the_right_password_of_at_least_eight_characters() ->
}
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
@pytest.mark.parametrize(
"row",
[
@ -305,7 +310,7 @@ def test_standard_mode_verifies_the_rows_it_always_did(row: str, verifies: bool)
@pytest.mark.asyncio
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
async def test_fips_boot_migration_writes_no_row() -> None:
pc: Final = MagicMock()
pc.db.litellm_usertable.find_many = AsyncMock(
@ -342,7 +347,7 @@ async def test_password_kdf_runs_on_the_event_loop_in_standard_mode() -> None:
@pytest.mark.asyncio
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.usefixtures("fips_mode")
async def test_password_kdfs_leave_the_event_loop_on_a_single_thread_in_fips_mode() -> None:
release_first: Final = threading.Event()
first_kdf: Final = asyncio.ensure_future(run_off_loop_if_fips(lambda: release_first.wait(timeout=5)))