From 5e0e82bbafda7387c7f74a5276f6d6afc970d055 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 3 Nov 2026 16:16:08 -0800 Subject: [PATCH] fix(proxy): choose the password KDF from fips_mode instead of the host OpenSSL --- litellm/proxy/auth/login_utils.py | 4 +-- litellm/proxy/common_utils/fips.py | 11 ++++++ litellm/proxy/utils.py | 10 +++--- tests/unit/proxy/auth/test_login_utils.py | 6 ++-- tests/unit/proxy/auth/test_onboarding.py | 2 +- tests/unit/proxy/auth/test_password_policy.py | 2 +- tests/unit/proxy/common_utils/test_fips.py | 35 ++++++++++++++++++- tests/unit/proxy/conftest.py | 9 +++++ .../test_internal_user_endpoints.py | 2 +- .../test_password_endpoints.py | 2 +- .../prisma_and_spend/test_password_helpers.py | 15 +++++--- 11 files changed, 78 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 5ff93eea7f2..7d413121b56 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -32,7 +32,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle from litellm.proxy.auth.password_policy import is_breach_check_enabled, is_password_breached -from litellm.proxy.common_utils.fips import openssl_enforces_fips +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, @@ -113,7 +113,7 @@ _ONE_FIPS_LOGIN_AT_A_TIME: Final = asyncio.Lock() async def _password_matches(password: str, stored: str, attempt: LoginAttempt) -> bool: """Checks inline, or in FIPS mode one login at a time, refusing a login whose user got blocked while it waited.""" - if not openssl_enforces_fips(): + if not fips_mode_active(): return verify_password(password, stored) async with _ONE_FIPS_LOGIN_AT_A_TIME: await attempt.refuse_if_blocked_while_waiting() diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 1a4d7ddeadb..6cd23e5c0ff 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -1,3 +1,4 @@ +import threading from collections.abc import Mapping from typing import Final @@ -5,6 +6,7 @@ from litellm.secret_managers.main import str_to_bool FIPS_MODE_SETTING: Final = "fips_mode" _SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}" +_FIPS_MODE_ACTIVE: Final = threading.Event() class FipsModeError(Exception): @@ -87,3 +89,12 @@ def enforce_fips_mode(general_settings: Mapping[str, object]) -> None: refusal: Final = fips_mode_refusal(general_settings) if refusal is not None: raise FipsModeError(refusal) + if _fips_mode_setting(general_settings): + _FIPS_MODE_ACTIVE.set() + else: + _FIPS_MODE_ACTIVE.clear() + + +def fips_mode_active() -> bool: + """Whether general_settings.fips_mode was on when enforce_fips_mode last accepted it.""" + return _FIPS_MODE_ACTIVE.is_set() diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 03b82658ec2..64bce0d21f2 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -89,7 +89,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.password_policy import MIN_ALLOWED_LENGTH from litellm.proxy.bug_report_config import build_proxy_bug_report -from litellm.proxy.common_utils.fips import openssl_enforces_fips +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.common_utils.openai_error_payload import ( attribute_of, error_status_code, @@ -8413,7 +8413,7 @@ def hash_password(password: str) -> str: import hashlib import os - if openssl_enforces_fips(): + if fips_mode_active(): fips_salt: Final = ssl.RAND_bytes(16) fips_dk: Final = hashlib.pbkdf2_hmac("sha256", password.encode(), fips_salt, iterations=600_000, dklen=32) return "pbkdf2:" + base64.b64encode(fips_salt + fips_dk).decode() @@ -8428,7 +8428,7 @@ def verify_password(password: str, stored: str) -> bool: import hashlib import secrets - if openssl_enforces_fips(): + if fips_mode_active(): if not _is_pbkdf2_row(stored): verbose_proxy_logger.error("FIPS mode refuses a non-PBKDF2 password row. Reset it with /user/update.") return False @@ -8458,7 +8458,7 @@ _FIPS_KDF_POOL: Final = ThreadPoolExecutor(max_workers=1, thread_name_prefix="fi async def run_off_loop_if_fips(kdf: Callable[..., _T], *args: str) -> _T: """Run a password KDF inline, or on the single KDF thread in FIPS mode.""" - if not openssl_enforces_fips(): + if not fips_mode_active(): return kdf(*args) return await asyncio.get_running_loop().run_in_executor(_FIPS_KDF_POOL, kdf, *args) @@ -8469,7 +8469,7 @@ async def migrate_passwords_to_scrypt_async(prisma_client) -> str: are left alone (they migrate on next login via the SHA256 fallback). Skips quickly if no plaintext passwords exist. """ - if openssl_enforces_fips(): + if fips_mode_active(): return "Skipped, FIPS mode has no password migration" all_with_pw: Final = await UserRepository(prisma_client).table.find_many( where={"password": {"not": None}}, diff --git a/tests/unit/proxy/auth/test_login_utils.py b/tests/unit/proxy/auth/test_login_utils.py index a74a0ff0641..bd5bdeafe95 100644 --- a/tests/unit/proxy/auth/test_login_utils.py +++ b/tests/unit/proxy/auth/test_login_utils.py @@ -2398,7 +2398,7 @@ async def test_login_rehash_rewrites_only_a_legacy_row(stored: str, schemes_writ @pytest.mark.asyncio -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") async def test_fips_login_runs_one_pbkdf2_off_the_event_loop_and_writes_no_row( monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool] ) -> None: @@ -2470,7 +2470,7 @@ async def _login_whose_user_gets_blocked_during_the_row_lookup(monkeypatch: pyte @pytest.mark.asyncio -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") async def test_fips_login_hides_the_right_password_when_its_user_got_blocked_while_it_waited( monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool] ) -> None: @@ -2484,7 +2484,7 @@ async def test_fips_login_hides_the_right_password_when_its_user_got_blocked_whi @pytest.mark.asyncio -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") async def test_fips_burst_of_wrong_passwords_stops_deriving_once_its_user_is_blocked( monkeypatch: pytest.MonkeyPatch, pbkdf2_on_event_loop: list[bool] ) -> None: diff --git a/tests/unit/proxy/auth/test_onboarding.py b/tests/unit/proxy/auth/test_onboarding.py index 6f907fab181..c449ffed806 100644 --- a/tests/unit/proxy/auth/test_onboarding.py +++ b/tests/unit/proxy/auth/test_onboarding.py @@ -478,7 +478,7 @@ async def test_claim_token_sets_accepted_at_after_password_written(): @pytest.mark.asyncio -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") async def test_fips_claim_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None: await test_claim_token_sets_accepted_at_after_password_written() assert pbkdf2_on_event_loop == [False] diff --git a/tests/unit/proxy/auth/test_password_policy.py b/tests/unit/proxy/auth/test_password_policy.py index 39e81793f1e..ea60c08a4ea 100644 --- a/tests/unit/proxy/auth/test_password_policy.py +++ b/tests/unit/proxy/auth/test_password_policy.py @@ -360,7 +360,7 @@ async def test_validate_passwords_bulk_empty_batch_makes_no_lookups(): @pytest.mark.asyncio -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") async def test_fips_mode_keeps_the_breach_check() -> None: client_says_breached: Final = _client_returning(f"{_sha1_upper(STRONG_PASSWORD)[5:]}:1387") assert await is_password_breached(STRONG_PASSWORD, {}, client_says_breached) is True diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 0884d0bbbed..d22684ea051 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,5 +1,5 @@ import _hashlib -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from contextlib import nullcontext from unittest.mock import Mock @@ -12,10 +12,17 @@ from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, enforce_fips_mode, + fips_mode_active, openssl_enforces_fips, ) +@pytest.fixture(autouse=True) +def _fips_mode_off_after_each_test() -> Iterator[None]: + yield + enforce_fips_mode({}) + + def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, scrypt_refused: bool) -> None: monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) if scrypt_refused: @@ -119,3 +126,29 @@ def test_enforce_fips_mode( with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode(general_settings) + + +@pytest.mark.parametrize( + "general_settings, active", + [({"fips_mode": True}, True), ({"fips_mode": "true"}, True), ({"fips_mode": False}, False), ({}, False)], + ids=["on", "on-from-a-string", "off", "unset"], +) +def test_fips_mode_active_follows_the_setting_the_gate_accepted( + monkeypatch: pytest.MonkeyPatch, general_settings: Mapping[str, object], active: bool +) -> None: + _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) + enforce_fips_mode(general_settings) + assert fips_mode_active() is active + + +def test_fips_mode_stays_inactive_on_a_refused_boot(monkeypatch: pytest.MonkeyPatch) -> None: + _fake_openssl(monkeypatch, fips_flag=0, scrypt_refused=False) + with pytest.raises(FipsModeError): + enforce_fips_mode({"fips_mode": True}) + assert fips_mode_active() is False + + +def test_a_fips_openssl_alone_does_not_activate_fips_mode(monkeypatch: pytest.MonkeyPatch) -> None: + _fake_openssl(monkeypatch, fips_flag=1, scrypt_refused=True) + assert fips_mode_active() is False diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 7f89bb2e459..e45ae2dd8fd 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -23,6 +23,7 @@ from prisma.errors import ClientNotConnectedError import litellm import litellm.proxy.proxy_server from litellm._logging import verbose_proxy_logger +from litellm.proxy.common_utils.fips import enforce_fips_mode from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( InitPassThroughEndpointHelpers, _registered_pass_through_routes, @@ -531,6 +532,14 @@ def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(_hashlib, "new", new) +@pytest.fixture +def fips_mode(fips_openssl: None, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True) + enforce_fips_mode({"fips_mode": True}) + yield + enforce_fips_mode({}) + + @pytest.fixture def pbkdf2_on_event_loop(monkeypatch: pytest.MonkeyPatch) -> list[bool]: """One entry per PBKDF2 call: whether it ran on the event loop's thread.""" diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index b1d4e89bde6..120cf8a2b84 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -5486,7 +5486,7 @@ async def test_user_update_reports_a_bad_budget_duration_as_a_request_error_not_ @pytest.mark.asyncio -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") async def test_fips_user_update_hashes_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None: from litellm.proxy.management_endpoints.internal_user_endpoints import _hash_password_in_dict diff --git a/tests/unit/proxy/management_endpoints/test_password_endpoints.py b/tests/unit/proxy/management_endpoints/test_password_endpoints.py index 30c299a11ee..d1e9f80129b 100644 --- a/tests/unit/proxy/management_endpoints/test_password_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_password_endpoints.py @@ -493,7 +493,7 @@ async def test_change_password_requires_db(): @pytest.mark.asyncio -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") async def test_fips_password_change_runs_both_pbkdf2_off_the_event_loop(pbkdf2_on_event_loop: list[bool]) -> None: from litellm.proxy._types import ChangePasswordRequest diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py b/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py index cf79e30b3b3..2df009a72a9 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_password_helpers.py @@ -232,19 +232,24 @@ async def test_migrate_passwords_raises_on_db_failure() -> None: @pytest.mark.usefixtures("fips_openssl") +def test_a_fips_openssl_without_fips_mode_stays_on_scrypt() -> None: + assert hash_password(PASSWORD).startswith("scrypt:") + + +@pytest.mark.usefixtures("fips_mode") def test_fips_hash_is_pbkdf2_sha256_at_600k_salted_by_openssl(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(ssl, "RAND_bytes", lambda size: bytes(range(size))) assert hash_password(PASSWORD) == PBKDF2_ROW -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") def test_fips_hash_salts_every_row_and_verifies() -> None: first: Final = hash_password(PASSWORD) assert first != hash_password(PASSWORD) assert verify_password(PASSWORD, first) -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") def test_fips_verify_needs_the_right_password_of_at_least_eight_characters() -> None: outcome: Final = { "right": verify_password(PASSWORD, PBKDF2_ROW), @@ -262,7 +267,7 @@ def test_fips_verify_needs_the_right_password_of_at_least_eight_characters() -> } -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") @pytest.mark.parametrize( "row", [ @@ -305,7 +310,7 @@ def test_standard_mode_verifies_the_rows_it_always_did(row: str, verifies: bool) @pytest.mark.asyncio -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") async def test_fips_boot_migration_writes_no_row() -> None: pc: Final = MagicMock() pc.db.litellm_usertable.find_many = AsyncMock( @@ -342,7 +347,7 @@ async def test_password_kdf_runs_on_the_event_loop_in_standard_mode() -> None: @pytest.mark.asyncio -@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.usefixtures("fips_mode") async def test_password_kdfs_leave_the_event_loop_on_a_single_thread_in_fips_mode() -> None: release_first: Final = threading.Event() first_kdf: Final = asyncio.ensure_future(run_off_loop_if_fips(lambda: release_first.wait(timeout=5)))