feat(proxy): add LITELLM_FIPS_MODE startup gate with provider assertion and loud password migration failure (#42700)

* feat(proxy): LITELLM_FIPS_MODE startup gate with provider assertion, TLS verify guard and loud password migration

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(proxy): match ssl_verify off detection to runtime str_to_bool semantics

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(proxy): drop tautological fips probe test and satisfy CodeQL return checks

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): inject the fake Prisma client through the module boundary instead of patching _setup_prisma_client

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yucheng <yucheng@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/proxy/common_utils/fips.py

@@ -0,0 +1,150 @@
+import hashlib
+import os
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Final
+
+from typing_extensions import assert_never
+
+from litellm.secret_managers.main import str_to_bool
+
+FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE"
+SSL_VERIFY_ENV_VAR: Final = "SSL_VERIFY"
+SSL_VERIFY_SETTING: Final = "litellm_settings.ssl_verify"
+REFUSAL_PREFIX: Final = "LiteLLM proxy refused to start"
+
+_TRUE_VALUES: Final = frozenset({"true", "1", "yes", "on"})
+_FALSE_VALUES: Final = frozenset({"false", "0", "no", "off", ""})
+
+
+@dataclass(frozen=True, slots=True)
+class FipsModeOff:
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class FipsModeOn:
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class MalformedFipsMode:
+    value: str
+
+
+FipsModeSetting = FipsModeOff | FipsModeOn | MalformedFipsMode
+
+
+@dataclass(frozen=True, slots=True)
+class ProviderDoesNotEnforceFips:
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class TlsVerificationDisabled:
+    sources: tuple[str, ...]
+
+
+FipsBootRefusal = MalformedFipsMode | ProviderDoesNotEnforceFips | TlsVerificationDisabled
+FipsBootVerdict = FipsModeOff | FipsModeOn | FipsBootRefusal
+
+
+class FipsModeError(Exception):
+    pass
+
+
+def parse_fips_mode(raw: str | None) -> FipsModeSetting:
+    if raw is None:
+        return FipsModeOff()
+    normalized: Final = raw.strip().lower()
+    if normalized in _TRUE_VALUES:
+        return FipsModeOn()
+    if normalized in _FALSE_VALUES:
+        return FipsModeOff()
+    return MalformedFipsMode(value=raw)
+
+
+def is_fips_mode(environ: Callable[[str], str | None] = os.environ.get) -> bool:
+    return isinstance(parse_fips_mode(environ(FIPS_MODE_ENV_VAR)), FipsModeOn)
+
+
+def openssl_enforces_fips() -> bool:
+    """MD5 is not an approved digest, so an enforcing FIPS provider refuses it even when asked for security use."""
+    try:
+        hashlib.md5(b"", usedforsecurity=True)
+    except ValueError:
+        return True
+    return False
+
+
+def fips_boot_verdict(
+    *,
+    raw_fips_mode: str | None,
+    provider_enforces_fips: Callable[[], bool],
+    ssl_verify_environment: str | None,
+    ssl_verify_setting: object,
+) -> FipsBootVerdict:
+    setting: Final = parse_fips_mode(raw_fips_mode)
+    match setting:
+        case FipsModeOff() | MalformedFipsMode():
+            return setting
+        case FipsModeOn():
+            pass
+        case _:
+            assert_never(setting)
+    disabled: Final = tuple(
+        source
+        for source, off in (
+            (SSL_VERIFY_ENV_VAR, _is_off(ssl_verify_environment)),
+            (SSL_VERIFY_SETTING, _is_off(ssl_verify_setting)),
+        )
+        if off
+    )
+    if disabled:
+        return TlsVerificationDisabled(sources=disabled)
+    if not provider_enforces_fips():
+        return ProviderDoesNotEnforceFips()
+    return setting
+
+
+def enforce_fips_boot_verdict(verdict: FipsBootVerdict, announce: Callable[[str], object]) -> None:
+    match verdict:
+        case FipsModeOff() | FipsModeOn():
+            return
+        case MalformedFipsMode() | ProviderDoesNotEnforceFips() | TlsVerificationDisabled():
+            message: Final = render_refusal(verdict)
+            announce(f"\n{message}\n\n")
+            raise FipsModeError(message)
+        case _:
+            assert_never(verdict)
+
+
+def render_refusal(refusal: FipsBootRefusal) -> str:
+    match refusal:
+        case MalformedFipsMode():
+            return (
+                f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR}={refusal.value} is not a boolean.\n"
+                f"Set {FIPS_MODE_ENV_VAR} to true or false, or unset it."
+            )
+        case ProviderDoesNotEnforceFips():
+            return (
+                f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but this Python does not enforce FIPS.\n"
+                "Its OpenSSL still allows non-approved algorithms (MD5 succeeded), so passwords and keys would be\n"
+                "protected with algorithms the FIPS 140-3 policy forbids. Run the proxy from a FIPS image whose\n"
+                f"OpenSSL FIPS provider is enabled, or unset {FIPS_MODE_ENV_VAR} on a non-FIPS runtime."
+            )
+        case TlsVerificationDisabled():
+            return (
+                f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by "
+                f"{' and '.join(refusal.sources)}.\nFIPS deployments must verify upstream certificates, so remove the "
+                "override or point ssl_verify at a CA bundle instead."
+            )
+    return assert_never(refusal)
+
+
+def _is_off(value: object) -> bool:
+    if isinstance(value, bool):
+        return value is False
+    if isinstance(value, str):
+        return str_to_bool(value) is False
+    return False

litellm/proxy/proxy_server.py

@@ -422,6 +422,14 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     encrypt_value_helper,
 )
 from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id
+from litellm.proxy.common_utils.fips import (
+    FIPS_MODE_ENV_VAR,
+    SSL_VERIFY_ENV_VAR,
+    enforce_fips_boot_verdict,
+    fips_boot_verdict,
+    is_fips_mode,
+    openssl_enforces_fips,
+)
 from litellm.proxy.common_utils.healthy_model_filter import (
     get_hidden_unhealthy_model_names,
     is_healthy_only_listing_default,
@@ -1329,6 +1337,16 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState
             if isinstance(worker_config, dict):
                 await initialize_from_worker_config(worker_config)
 
+    enforce_fips_boot_verdict(
+        fips_boot_verdict(
+            raw_fips_mode=os.getenv(FIPS_MODE_ENV_VAR),
+            provider_enforces_fips=openssl_enforces_fips,
+            ssl_verify_environment=os.getenv(SSL_VERIFY_ENV_VAR),
+            ssl_verify_setting=litellm.ssl_verify,
+        ),
+        announce=announce_on_stderr_at_exit,
+    )
+
     enforce_master_key_boot_verdict(
         await with_stored_secrets_counted(
             master_key_boot_verdict(
@@ -1368,10 +1386,21 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState
             try:
                 result: Final = await migrate_passwords_to_scrypt_async(prisma_client)
                 verbose_proxy_logger.info("Password migration: %s", result)
+            except ValueError as e:
+                verbose_proxy_logger.error(
+                    "Password migration failed, so plaintext passwords stay unhashed in the database: %s. "
+                    "This is what an OpenSSL FIPS provider reports when the hashing algorithm is not approved.",
+                    e,
+                )
+                if is_fips_mode():
+                    raise
             except Exception as e:
                 verbose_proxy_logger.warning("Password migration skipped: %s", e)
 
-        asyncio.create_task(_run_pw_migration())
+        if is_fips_mode():
+            await _run_pw_migration()
+        else:
+            asyncio.create_task(_run_pw_migration())
 
         async def _run_agent_grant_id_migration() -> None:
             from litellm.proxy.agent_endpoints.agent_registry import (

tests/integration/_support/process.py

@@ -230,6 +230,60 @@ def owned_proxy_process(
         _stop(process)
 
 
+def _is_ready(client: httpx.Client) -> bool:
+    try:
+        return client.get("/health/readiness", timeout=2).status_code == 200
+    except httpx.TransportError:
+        return False
+
+
+def refused_boot_log(
+    gateway: Gateway,
+    directory: Path,
+    overrides: Mapping[str, str],
+    *,
+    config: Path | None = None,
+) -> str:
+    """Start the proxy and return its log once it exits non-zero instead of becoming ready."""
+    root: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3])
+    environment: Final = {
+        **os.environ,
+        **proxy_database_environment(),
+        "LITELLM_MASTER_KEY": gateway.key,
+        "LITELLM_SALT_KEY": os.environ.get("LITELLM_SALT_KEY", "sk-integration-salt"),
+        "STORE_MODEL_IN_DB": "True",
+        **overrides,
+    }
+    output: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(directory)))
+    output.mkdir(parents=True, exist_ok=True)
+    command: Final = (
+        sys.executable,
+        "-m",
+        "integration._support.proxy",
+        "--config",
+        str(config or "tests/integration/proxy_config.yaml"),
+        "--host",
+        "127.0.0.1",
+        "--num_workers",
+        "1",
+        *DB_PUSH,
+    )
+    launch: Final = _launch(command, root, environment, output)
+    try:
+        with httpx.Client(base_url=f"http://127.0.0.1:{launch.port}", timeout=15, trust_env=False) as client:
+            deadline: Final = time.monotonic() + 70
+            while launch.process.poll() is None:
+                assert not _is_ready(client), (
+                    f"Proxy became ready instead of refusing to boot:\n{launch.log.read_text()}"
+                )
+                assert time.monotonic() < deadline, "Proxy neither exited nor became ready within the deadline"
+                time.sleep(0.1)
+        assert launch.process.returncode != 0, f"Proxy exited 0 instead of refusing to boot:\n{launch.log.read_text()}"
+        return launch.log.read_text()
+    finally:
+        _stop(launch.process)
+
+
 _UPSTREAM_READY_SECONDS: Final = 60
 
 

tests/integration/configuration/test_fips_mode_boot.py

@@ -0,0 +1,62 @@
+"""LITELLM_FIPS_MODE is a boot gate: the proxy refuses to serve unless the process really enforces FIPS.
+
+Every leg launches the real proxy binary against the suite's Postgres and asserts on what an operator sees:
+exit status and the refusal text in the log. Nothing is patched inside the proxy.
+"""
+
+import hashlib
+from pathlib import Path
+from typing import Final
+
+import pytest
+import yaml
+
+from tests.integration._support.client import Gateway
+from tests.integration._support.process import owned_proxy, refused_boot_log
+
+REFUSAL: Final = "LiteLLM proxy refused to start"
+
+
+def _this_python_enforces_fips() -> bool:
+    try:
+        hashlib.md5(b"probe", usedforsecurity=True)
+    except ValueError:
+        return True
+    return False
+
+
+def test_fips_mode_refuses_to_serve_when_this_python_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None:
+    if _this_python_enforces_fips():
+        pytest.skip("Runner OpenSSL enforces FIPS, so this leg cannot observe the non-enforcing refusal")
+    log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"})
+    assert REFUSAL in log, log
+    assert "LITELLM_FIPS_MODE" in log and "does not enforce FIPS" in log, log
+
+
+@pytest.mark.parametrize("source", ("environment", "config"))
+def test_fips_mode_refuses_to_serve_with_tls_verification_disabled(
+    gateway: Gateway, tmp_path: Path, source: str
+) -> None:
+    config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
+    path: Final = tmp_path / "ssl_verify_off.yaml"
+    settings: Final = {**config.get("litellm_settings", {}), "ssl_verify": False}
+    path.write_text(yaml.safe_dump({**config, "litellm_settings": settings}))
+    log: Final = (
+        refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true", "SSL_VERIFY": "false"})
+        if source == "environment"
+        else refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}, config=path)
+    )
+    assert REFUSAL in log, log
+    assert "TLS certificate verification is disabled" in log, log
+    assert ("SSL_VERIFY" if source == "environment" else "litellm_settings.ssl_verify") in log, log
+
+
+def test_fips_mode_refuses_to_serve_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None:
+    log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"})
+    assert REFUSAL in log, log
+    assert "LITELLM_FIPS_MODE=enforced" in log and "true or false" in log, log
+
+
+def test_fips_mode_off_serves_even_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None:
+    with owned_proxy(gateway, tmp_path, {"LITELLM_FIPS_MODE": "false", "SSL_VERIFY": "false"}) as candidate:
+        assert candidate.client.get("/health/readiness").status_code == 200

tests/unit/proxy/common_utils/test_fips.py

@@ -0,0 +1,106 @@
+import pytest
+
+from litellm.proxy.common_utils.fips import (
+    FipsModeError,
+    FipsModeOff,
+    FipsModeOn,
+    MalformedFipsMode,
+    ProviderDoesNotEnforceFips,
+    TlsVerificationDisabled,
+    enforce_fips_boot_verdict,
+    fips_boot_verdict,
+    is_fips_mode,
+    parse_fips_mode,
+)
+
+
+def _verdict(
+    raw: str | None,
+    *,
+    enforcing: bool = True,
+    ssl_env: str | None = None,
+    ssl_setting: object = True,
+):
+    return fips_boot_verdict(
+        raw_fips_mode=raw,
+        provider_enforces_fips=lambda: enforcing,
+        ssl_verify_environment=ssl_env,
+        ssl_verify_setting=ssl_setting,
+    )
+
+
+@pytest.mark.parametrize("raw", [None, "false", "0", "no", "off", "", " False "])
+def test_unset_and_false_spellings_leave_fips_mode_off(raw):
+    assert parse_fips_mode(raw) == FipsModeOff()
+    assert is_fips_mode({"LITELLM_FIPS_MODE": raw}.get) is False
+
+
+@pytest.mark.parametrize("raw", ["true", "1", "yes", "on", " TRUE "])
+def test_true_spellings_turn_fips_mode_on(raw):
+    assert parse_fips_mode(raw) == FipsModeOn()
+    assert is_fips_mode({"LITELLM_FIPS_MODE": raw}.get) is True
+
+
+@pytest.mark.parametrize("raw", ["enforced", "2", "strict", "yes please"])
+def test_anything_else_is_malformed_and_refused_with_the_offending_value(raw):
+    assert parse_fips_mode(raw) == MalformedFipsMode(value=raw)
+    with pytest.raises(FipsModeError) as refused:
+        enforce_fips_boot_verdict(_verdict(raw, enforcing=False), announce=lambda _: None)
+    assert f"LITELLM_FIPS_MODE={raw} is not a boolean" in str(refused.value)
+    assert "true or false" in str(refused.value)
+
+
+def test_off_never_consults_the_provider_or_tls_settings():
+    def explode() -> bool:
+        raise AssertionError("provider probe must not run while FIPS mode is off")
+
+    verdict = fips_boot_verdict(
+        raw_fips_mode=None, provider_enforces_fips=explode, ssl_verify_environment="false", ssl_verify_setting=False
+    )
+    assert verdict == FipsModeOff()
+    enforce_fips_boot_verdict(verdict, announce=lambda _: pytest.fail("nothing to announce when off"))
+
+
+def test_on_with_an_enforcing_provider_and_verified_tls_boots():
+    verdict = _verdict("true", enforcing=True, ssl_env="true", ssl_setting="/etc/ssl/certs/ca.pem")
+    assert verdict == FipsModeOn()
+    enforce_fips_boot_verdict(verdict, announce=lambda _: pytest.fail("nothing to announce when on"))
+
+
+def test_on_with_a_non_enforcing_provider_is_refused_and_names_the_fix():
+    announced = []
+    with pytest.raises(FipsModeError) as refused:
+        enforce_fips_boot_verdict(_verdict("true", enforcing=False), announce=announced.append)
+    message = str(refused.value)
+    assert message.startswith("LiteLLM proxy refused to start")
+    assert "LITELLM_FIPS_MODE is on but this Python does not enforce FIPS" in message
+    assert "FIPS image" in message
+    assert announced == [f"\n{message}\n\n"]
+
+
+@pytest.mark.parametrize(
+    "ssl_env, ssl_setting, sources",
+    [
+        ("false", True, ("SSL_VERIFY",)),
+        (" FALSE ", True, ("SSL_VERIFY",)),
+        (None, False, ("litellm_settings.ssl_verify",)),
+        (None, "False", ("litellm_settings.ssl_verify",)),
+        ("false", False, ("SSL_VERIFY", "litellm_settings.ssl_verify")),
+    ],
+)
+def test_disabled_tls_verification_is_refused_naming_every_source(ssl_env, ssl_setting, sources):
+    verdict = _verdict("true", enforcing=True, ssl_env=ssl_env, ssl_setting=ssl_setting)
+    assert verdict == TlsVerificationDisabled(sources=sources)
+    with pytest.raises(FipsModeError) as refused:
+        enforce_fips_boot_verdict(verdict, announce=lambda _: None)
+    assert "TLS certificate verification is disabled by " + " and ".join(sources) in str(refused.value)
+
+
+@pytest.mark.parametrize("ssl_setting", [True, "true", "/etc/ssl/certs/ca.pem", None, "", "0", "no"])
+def test_verified_or_custom_bundle_tls_settings_are_not_treated_as_disabled(ssl_setting):
+    assert _verdict("true", enforcing=True, ssl_setting=ssl_setting) == FipsModeOn()
+
+
+def test_disabled_tls_is_reported_before_the_provider_so_operators_see_config_mistakes_first():
+    assert _verdict("true", enforcing=False, ssl_env="false") == TlsVerificationDisabled(sources=("SSL_VERIFY",))
+    assert _verdict("true", enforcing=False) == ProviderDoesNotEnforceFips()

tests/unit/proxy/test_proxy_server_endpoints_and_startup.py

@@ -1771,6 +1771,108 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database
     assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database
 
 
+@pytest.mark.asyncio
+async def test_proxy_startup_refuses_fips_mode_when_this_python_does_not_enforce_fips(monkeypatch, tmp_path):
+    from fastapi import FastAPI
+
+    from litellm.proxy.common_utils.fips import FipsModeError
+    from litellm.proxy.proxy_server import proxy_startup_event
+
+    _, announced = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"})
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
+    monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: False)
+    monkeypatch.setenv("LITELLM_FIPS_MODE", "true")
+
+    with pytest.raises(FipsModeError):
+        async with proxy_startup_event(FastAPI()):
+            pass
+
+    assert len(announced) == 1
+    assert "does not enforce FIPS" in announced[0]
+
+
+@pytest.mark.asyncio
+async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path):
+    import yaml
+    from fastapi import FastAPI
+
+    from litellm.proxy.common_utils.fips import FipsModeError
+    from litellm.proxy.proxy_server import proxy_startup_event
+
+    config_path, announced = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"})
+    config_path.write_text(
+        yaml.dump({"general_settings": {"master_key": "sk-a-safe-master-key"}, "litellm_settings": {"ssl_verify": False}})
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
+    monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True)
+    monkeypatch.setattr(litellm, "ssl_verify", True)
+    monkeypatch.setenv("LITELLM_FIPS_MODE", "true")
+
+    with pytest.raises(FipsModeError):
+        async with proxy_startup_event(FastAPI()):
+            pass
+
+    assert "TLS certificate verification is disabled by litellm_settings.ssl_verify" in announced[0]
+
+
+class _PrismaClientWhoseUserTableCannotHash:
+    class _Table:
+        async def find_many(self, where):
+            raise ValueError("[digital envelope routines] unsupported")
+
+    class _Db:
+        litellm_usertable = None
+
+    def __init__(self, database_url, proxy_logging_obj):
+        self.db = self._Db()
+        self.db.litellm_usertable = self._Table()
+        self.writer_db = self.db
+
+    async def connect(self):
+        pass
+
+    async def disconnect(self):
+        pass
+
+    def start_view_setup_task(self):
+        pass
+
+    async def check_view_exists(self):
+        pass
+
+    async def health_check(self):
+        pass
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fips_mode", ["true", "false"])
+async def test_proxy_startup_surfaces_a_password_migration_crypto_failure(monkeypatch, tmp_path, caplog, fips_mode):
+    from fastapi import FastAPI
+
+    from litellm.proxy.proxy_server import proxy_startup_event
+
+    _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"})
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
+    monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable")
+    monkeypatch.setattr("litellm.proxy.proxy_server.PrismaClient", _PrismaClientWhoseUserTableCannotHash)
+    monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True)
+    monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode)
+
+    with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
+        if fips_mode == "true":
+            with pytest.raises(ValueError, match="digital envelope routines"):
+                async with proxy_startup_event(FastAPI()):
+                    pass
+        else:
+            async with proxy_startup_event(FastAPI()):
+                await asyncio.sleep(0)
+
+    failures = [r.getMessage() for r in caplog.records if "Password migration failed" in r.getMessage()]
+    assert len(failures) == 1
+    assert "plaintext passwords stay unhashed" in failures[0]
+    assert "digital envelope routines" in failures[0]
+
+
 class _DatabaseWithOneStoredCredential:
     def __init__(self, ciphertext):
         self._ciphertext = ciphertext