refactor(proxy): return FIPS boot refusals as values and harden the FIPS tests

litellm/llms/custom_httpx/http_handler.py

@@ -336,6 +336,21 @@ def _create_ssl_context(
 
 def get_ssl_verify(
     ssl_verify: bool | str | None = None,
+) -> bool | str:
+    """resolve_ssl_verify, refusing disabled TLS verification under LITELLM_FIPS_MODE."""
+    resolved: Final = resolve_ssl_verify(ssl_verify)
+    if resolved is not False:
+        return resolved
+
+    from litellm.proxy.common_utils.fips import TLS_VERIFICATION_DISABLED, FipsModeError, is_fips_mode
+
+    if is_fips_mode():
+        raise FipsModeError(TLS_VERIFICATION_DISABLED)
+    return resolved
+
+
+def resolve_ssl_verify(
+    ssl_verify: bool | str | None = None,
 ) -> bool | str:
     """
     Common utility to resolve the SSL verification setting.
@@ -370,16 +385,6 @@ def get_ssl_verify(
         if ssl_cert_file and os.path.exists(ssl_cert_file):
             return ssl_cert_file
 
-    if ssl_verify is False:
-        from litellm.proxy.common_utils.fips import FIPS_MODE_ENV_VAR, FipsModeError, is_fips_mode
-
-        if is_fips_mode():
-            raise FipsModeError(
-                f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, "
-                "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a "
-                "CA bundle."
-            )
-
     return ssl_verify if ssl_verify is not None else True
 
 

litellm/proxy/common_utils/fips.py

@@ -4,6 +4,10 @@ from typing import Final
 from litellm.secret_managers.main import str_to_bool
 
 FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE"
+TLS_VERIFICATION_DISABLED: Final = (
+    f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, "
+    "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a CA bundle."
+)
 
 
 class FipsModeError(Exception):
@@ -47,26 +51,35 @@ def cryptography_enforces_fips() -> bool:
     return True
 
 
-def enforce_fips_mode() -> None:
-    """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off
-    or Python's or the cryptography library's OpenSSL does not enforce FIPS."""
+def fips_mode_refusal() -> str | None:
+    """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off
+    or Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start."""
     value: Final = os.getenv(FIPS_MODE_ENV_VAR)
     if value and str_to_bool(value) is None:
-        raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.")
+        return f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it."
     if not is_fips_mode():
-        return
+        return None
 
-    from litellm.llms.custom_httpx.http_handler import get_ssl_verify
+    from litellm.llms.custom_httpx.http_handler import resolve_ssl_verify
 
-    get_ssl_verify()  # raises FipsModeError when SSL_VERIFY or litellm_settings.ssl_verify turns verification off
+    if resolve_ssl_verify() is False:
+        return TLS_VERIFICATION_DISABLED
     if not openssl_enforces_fips():
-        raise FipsModeError(
+        return (
             f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with "
             f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}."
         )
     if not cryptography_enforces_fips():
-        raise FipsModeError(
+        return (
             f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it "
             "against the system OpenSSL (pip install --no-binary cryptography cryptography), or unset "
             f"{FIPS_MODE_ENV_VAR}."
         )
+    return None
+
+
+def enforce_fips_mode() -> None:
+    """Raise FipsModeError with the fips_mode_refusal reason, if there is one."""
+    refusal: Final = fips_mode_refusal()
+    if refusal is not None:
+        raise FipsModeError(refusal)

tests/integration/configuration/test_fips_mode_boot.py

@@ -58,5 +58,7 @@ def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Ga
 
 def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None:
     config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False)
-    with owned_proxy(gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config) as proxy:
+    with owned_proxy(
+        gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config, remove_environment=("LITELLM_FIPS_MODE",)
+    ) as proxy:
         assert proxy.client.get("/health/readiness").status_code == 200

tests/unit/proxy/common_utils/test_fips.py

@@ -1,5 +1,6 @@
 import _hashlib
 from contextlib import nullcontext
+from typing import Final
 from unittest.mock import Mock
 
 import pytest
@@ -18,20 +19,20 @@ from litellm.proxy.common_utils.fips import (
 )
 
 
-def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None:
-    real_new = _hashlib.new
+def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None:
+    real_new: Final = _hashlib.new
 
-    def new(name, data=b"", **kwargs):
+    def new(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH:
         if name == "md5" and md5_refused:
             raise _hashlib.UnsupportedDigestmodError("unsupported hash type")
-        return real_new(name, data, **kwargs)
+        return real_new(name, data, usedforsecurity=usedforsecurity)
 
     monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag)
     monkeypatch.setattr(_hashlib, "new", new)
 
 
 @pytest.mark.parametrize("value, expected", [("true", True), (" TRUE ", True), ("false", False), (None, False)])
-def test_is_fips_mode(monkeypatch, value, expected):
+def test_is_fips_mode(monkeypatch: pytest.MonkeyPatch, value: str | None, expected: bool) -> None:
     if value is None:
         monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False)
     else:
@@ -44,12 +45,14 @@ def test_is_fips_mode(monkeypatch, value, expected):
     [(1, True, True), (0, True, False), (1, False, False)],
     ids=["fips", "fips-flag-off", "md5-allowed"],
 )
-def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5(monkeypatch, fips_flag, md5_refused, enforces):
+def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5(
+    monkeypatch: pytest.MonkeyPatch, fips_flag: int, md5_refused: bool, enforces: bool
+) -> None:
     _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused)
     assert openssl_enforces_fips() is enforces
 
 
-def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch):
+def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.delattr(_hashlib, "get_fips_mode")
     assert openssl_enforces_fips() is False
 
@@ -60,23 +63,28 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch):
     ids=["fips", "fips-flag-off", "sha256-broken"],
 )
 def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256(
-    monkeypatch, fips_enabled, sha256_broken, enforces
-):
+    monkeypatch: pytest.MonkeyPatch, fips_enabled: bool, sha256_broken: bool, enforces: bool
+) -> None:
     monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled)
     if sha256_broken:
         monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", [])))
     assert cryptography_enforces_fips() is enforces
 
 
-def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch):
+def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.delattr(rust_openssl, "is_fips_enabled")
     assert cryptography_enforces_fips() is False
 
 
-def test_this_runners_openssl_does_not_enforce_fips():
+def test_this_runners_python_openssl_does_not_enforce_fips() -> None:
     if _hashlib.get_fips_mode() == 1:
-        pytest.skip("this runner's OpenSSL is in FIPS mode")
+        pytest.skip("this runner's Python OpenSSL is in FIPS mode")
     assert openssl_enforces_fips() is False
+
+
+def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None:
+    if rust_openssl.is_fips_enabled():
+        pytest.skip("this runner's cryptography OpenSSL is in FIPS mode")
     assert cryptography_enforces_fips() is False
 
 
@@ -106,8 +114,14 @@ def test_this_runners_openssl_does_not_enforce_fips():
     ],
 )
 def test_enforce_fips_mode(
-    monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal
-):
+    monkeypatch: pytest.MonkeyPatch,
+    fips_mode: str | None,
+    ssl_verify_env: str | None,
+    ssl_verify_setting: bool,
+    openssl_fips: bool,
+    cryptography_fips: bool,
+    refusal: str | None,
+) -> None:
     for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)):
         if value is None:
             monkeypatch.delenv(name, raising=False)
@@ -122,7 +136,9 @@ def test_enforce_fips_mode(
 
 
 @pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"])
-def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode(monkeypatch, fips_mode, refused):
+def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode(
+    monkeypatch: pytest.MonkeyPatch, fips_mode: str, refused: bool
+) -> None:
     monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode)
     with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext():
         assert get_ssl_verify(ssl_verify=False) is False

tests/unit/proxy/test_proxy_server_endpoints_and_startup.py

@@ -1776,7 +1776,9 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database
 
 
 @pytest.mark.asyncio
-async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path):
+async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
     from litellm.proxy.common_utils.fips import FipsModeError
     from litellm.proxy.proxy_server import proxy_startup_event
 
@@ -1791,22 +1793,48 @@ async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_veri
             pass
 
 
+class _UsersWithOnePlaintextPassword:
+    async def find_many(self, where: object) -> tuple[types.SimpleNamespace, ...]:
+        return (types.SimpleNamespace(user_id="user-1", password="a-plaintext-password"),)
+
+    async def update(self, where: object, data: object) -> None:
+        raise AssertionError("a refused boot must not write to the database")
+
+
+class _DatabaseWithOnePlaintextPassword:
+    def __init__(self) -> None:
+        self.db: Final = types.SimpleNamespace(litellm_usertable=_UsersWithOnePlaintextPassword())
+
+
 @pytest.mark.asyncio
-async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails(monkeypatch, tmp_path):
+async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    import _hashlib
+    import hashlib
+
+    from cryptography.hazmat.bindings._rust import openssl as rust_openssl
+
     from litellm.proxy.common_utils.fips import FipsModeError
-    from litellm.proxy.proxy_server import ProxyStartupEvent, proxy_startup_event
+    from litellm.proxy.proxy_server import proxy_startup_event
+
+    real_new: Final = _hashlib.new
+
+    def new_refusing_md5(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH:
+        if name == "md5":
+            raise _hashlib.UnsupportedDigestmodError("unsupported hash type")
+        return real_new(name, data, usedforsecurity=usedforsecurity)
 
     _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"})
     monkeypatch.setattr(litellm, "ssl_verify", True)
     monkeypatch.delenv("SSL_VERIFY", raising=False)
     monkeypatch.setenv("LITELLM_FIPS_MODE", "true")
-    monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True)
-    monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True)
-    monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma()))
-    scrypt_refused = ValueError("[digital envelope routines] unsupported")
-    monkeypatch.setattr(
-        "litellm.proxy.proxy_server.migrate_passwords_to_scrypt_async", AsyncMock(side_effect=scrypt_refused)
-    )
+    monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1)
+    monkeypatch.setattr(_hashlib, "new", new_refusing_md5)
+    monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True)
+    scrypt_refused: Final = ValueError("[digital envelope routines] unsupported")
+    monkeypatch.setattr(hashlib, "scrypt", mock.Mock(side_effect=scrypt_refused))
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _DatabaseWithOnePlaintextPassword())
 
     with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused:
         async with proxy_startup_event(FastAPI()):