diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py
index 0b64885df45..ae9aa8f19d2 100644
--- a/litellm/llms/custom_httpx/http_handler.py
+++ b/litellm/llms/custom_httpx/http_handler.py
@@ -370,6 +370,16 @@ def get_ssl_verify(
         if ssl_cert_file and os.path.exists(ssl_cert_file):
             return ssl_cert_file
 
+    if ssl_verify is False:
+        from litellm.proxy.common_utils.fips import FIPS_MODE_ENV_VAR, FipsModeError, is_fips_mode
+
+        if is_fips_mode():
+            raise FipsModeError(
+                f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, "
+                "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a "
+                "CA bundle."
+            )
+
     return ssl_verify if ssl_verify is not None else True
 
 
diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py
index 671c023bbdb..2ffb17852eb 100644
--- a/litellm/proxy/common_utils/fips.py
+++ b/litellm/proxy/common_utils/fips.py
@@ -58,11 +58,7 @@ def enforce_fips_mode() -> None:
 
     from litellm.llms.custom_httpx.http_handler import get_ssl_verify
 
-    if get_ssl_verify() is False:
-        raise FipsModeError(
-            f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY or "
-            "litellm_settings.ssl_verify. Remove the override, or point ssl_verify at a CA bundle."
-        )
+    get_ssl_verify()  # raises FipsModeError when SSL_VERIFY or litellm_settings.ssl_verify turns verification off
     if not openssl_enforces_fips():
         raise FipsModeError(
             f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with "
diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py
index bd6834af536..e70fd2e3b46 100644
--- a/tests/unit/proxy/common_utils/test_fips.py
+++ b/tests/unit/proxy/common_utils/test_fips.py
@@ -8,6 +8,7 @@ from cryptography.hazmat.bindings._rust import openssl as rust_openssl
 from cryptography.hazmat.primitives import hashes
 
 import litellm
+from litellm.llms.custom_httpx.http_handler import get_ssl_verify
 from litellm.proxy.common_utils.fips import (
     FipsModeError,
     cryptography_enforces_fips,
@@ -118,3 +119,10 @@ def test_enforce_fips_mode(
 
     with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext():
         enforce_fips_mode()
+
+
+@pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"])
+def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode(monkeypatch, fips_mode, refused):
+    monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode)
+    with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext():
+        assert get_ssl_verify(ssl_verify=False) is False