From 67895955f6e9f40e78a96c89a642adc03b5d6a3c Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:20:08 +0000 Subject: [PATCH 01/11] feat(proxy): LITELLM_FIPS_MODE startup gate with provider assertion, TLS verify guard and loud password migration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_utils/fips.py | 151 ++++++++++++++++++ litellm/proxy/proxy_server.py | 31 +++- tests/integration/_support/process.py | 76 +++++++-- .../configuration/test_fips_mode_boot.py | 66 ++++++++ tests/integration/contracts.json | 15 ++ .../proxy/common_utils/test_fips.py | 118 ++++++++++++++ tests/test_litellm/proxy/test_proxy_server.py | 104 ++++++++++++ 7 files changed, 546 insertions(+), 15 deletions(-) create mode 100644 litellm/proxy/common_utils/fips.py create mode 100644 tests/integration/configuration/test_fips_mode_boot.py create mode 100644 tests/test_litellm/proxy/common_utils/test_fips.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py new file mode 100644 index 00000000000..33f35388567 --- /dev/null +++ b/litellm/proxy/common_utils/fips.py @@ -0,0 +1,151 @@ +import hashlib +import os +from collections.abc import Callable +from dataclasses import dataclass +from typing import Final + +from typing_extensions import assert_never + +FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" +SSL_VERIFY_ENV_VAR: Final = "SSL_VERIFY" +SSL_VERIFY_SETTING: Final = "litellm_settings.ssl_verify" +REFUSAL_PREFIX: Final = "LiteLLM proxy refused to start" + +_TRUE_VALUES: Final = frozenset({"true", "1", "yes", "on"}) +_FALSE_VALUES: Final = frozenset({"false", "0", "no", "off", ""}) + + +@dataclass(frozen=True, slots=True) +class FipsModeOff: + pass + + +@dataclass(frozen=True, slots=True) +class FipsModeOn: + pass + + +@dataclass(frozen=True, slots=True) +class MalformedFipsMode: + value: str + + +FipsModeSetting = FipsModeOff | FipsModeOn | MalformedFipsMode + + +@dataclass(frozen=True, slots=True) +class ProviderDoesNotEnforceFips: + pass + + +@dataclass(frozen=True, slots=True) +class TlsVerificationDisabled: + sources: tuple[str, ...] + + +FipsBootRefusal = MalformedFipsMode | ProviderDoesNotEnforceFips | TlsVerificationDisabled +FipsBootVerdict = FipsModeOff | FipsModeOn | FipsBootRefusal + + +class FipsModeError(Exception): + pass + + +def parse_fips_mode(raw: str | None) -> FipsModeSetting: + if raw is None: + return FipsModeOff() + normalized: Final = raw.strip().lower() + if normalized in _TRUE_VALUES: + return FipsModeOn() + if normalized in _FALSE_VALUES: + return FipsModeOff() + return MalformedFipsMode(value=raw) + + +def is_fips_mode(environ: Callable[[str], str | None] = os.environ.get) -> bool: + return isinstance(parse_fips_mode(environ(FIPS_MODE_ENV_VAR)), FipsModeOn) + + +def openssl_enforces_fips() -> bool: + """MD5 is not an approved digest, so an enforcing FIPS provider refuses it even when asked for security use.""" + try: + hashlib.md5(b"", usedforsecurity=True) + except ValueError: + return True + return False + + +def fips_boot_verdict( + *, + raw_fips_mode: str | None, + provider_enforces_fips: Callable[[], bool], + ssl_verify_environment: str | None, + ssl_verify_setting: object, +) -> FipsBootVerdict: + setting: Final = parse_fips_mode(raw_fips_mode) + match setting: + case FipsModeOff() | MalformedFipsMode(): + return setting + case FipsModeOn(): + pass + case _: + assert_never(setting) + disabled: Final = tuple( + source + for source, off in ( + (SSL_VERIFY_ENV_VAR, _is_off(ssl_verify_environment)), + (SSL_VERIFY_SETTING, _is_off(ssl_verify_setting)), + ) + if off + ) + if disabled: + return TlsVerificationDisabled(sources=disabled) + if not provider_enforces_fips(): + return ProviderDoesNotEnforceFips() + return setting + + +def enforce_fips_boot_verdict(verdict: FipsBootVerdict, announce: Callable[[str], object]) -> None: + match verdict: + case FipsModeOff() | FipsModeOn(): + return + case MalformedFipsMode() | ProviderDoesNotEnforceFips() | TlsVerificationDisabled(): + message: Final = render_refusal(verdict) + announce(f"\n{message}\n\n") + raise FipsModeError(message) + case _: + assert_never(verdict) + + +def render_refusal(refusal: FipsBootRefusal) -> str: + match refusal: + case MalformedFipsMode(value=value): + return ( + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR}={value} is not a boolean.\n" + f"Set {FIPS_MODE_ENV_VAR} to true or false, or unset it." + ) + case ProviderDoesNotEnforceFips(): + return ( + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but this Python does not enforce FIPS.\n" + "Its OpenSSL still allows non-approved algorithms (MD5 succeeded), so passwords and keys would be\n" + "protected with algorithms the FIPS 140-3 policy forbids. Run the proxy from a FIPS image whose\n" + f"OpenSSL FIPS provider is enabled, or unset {FIPS_MODE_ENV_VAR} on a non-FIPS runtime." + ) + case TlsVerificationDisabled(sources=sources): + return ( + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by " + f"{' and '.join(sources)}.\nFIPS deployments must verify upstream certificates, so remove the " + "override or point ssl_verify at a CA bundle instead." + ) + case _: + assert_never(refusal) + + +def _is_off(value: object) -> bool: + match value: + case bool(): + return value is False + case str(): + return value.strip().lower() in _FALSE_VALUES - {""} + case _: + return False diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dab7decd4dc..7e2e063ae88 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -404,6 +404,14 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id +from litellm.proxy.common_utils.fips import ( + FIPS_MODE_ENV_VAR, + SSL_VERIFY_ENV_VAR, + enforce_fips_boot_verdict, + fips_boot_verdict, + is_fips_mode, + openssl_enforces_fips, +) from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -1277,6 +1285,16 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: if isinstance(worker_config, dict): await initialize_from_worker_config(worker_config) + enforce_fips_boot_verdict( + fips_boot_verdict( + raw_fips_mode=os.getenv(FIPS_MODE_ENV_VAR), + provider_enforces_fips=openssl_enforces_fips, + ssl_verify_environment=os.getenv(SSL_VERIFY_ENV_VAR), + ssl_verify_setting=litellm.ssl_verify, + ), + announce=announce_on_stderr_at_exit, + ) + enforce_master_key_boot_verdict( await with_stored_secrets_counted( master_key_boot_verdict( @@ -1316,10 +1334,21 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: try: result: Final = await migrate_passwords_to_scrypt_async(prisma_client) verbose_proxy_logger.info("Password migration: %s", result) + except ValueError as e: + verbose_proxy_logger.error( + "Password migration failed, so plaintext passwords stay unhashed in the database: %s. " + "This is what an OpenSSL FIPS provider reports when the hashing algorithm is not approved.", + e, + ) + if is_fips_mode(): + raise except Exception as e: verbose_proxy_logger.warning("Password migration skipped: %s", e) - asyncio.create_task(_run_pw_migration()) + if is_fips_mode(): + await _run_pw_migration() + else: + asyncio.create_task(_run_pw_migration()) async def _run_agent_grant_id_migration() -> None: from litellm.proxy.agent_endpoints.agent_registry import ( diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 5c44beaa570..6dec960097b 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -68,8 +68,15 @@ def owned_proxy( yield owned.gateway +@dataclass(frozen=True, slots=True) +class LaunchedProxy: + port: int + process: subprocess.Popen[bytes] + log: Path + + @contextmanager -def owned_proxy_process( +def launched_proxy( gateway: Gateway, directory: Path, overrides: Mapping[str, str], @@ -77,7 +84,7 @@ def owned_proxy_process( config: Path | None = None, remove_environment: tuple[str, ...] = (), workers: int = 1, -) -> Iterator[OwnedProxy]: +) -> Iterator[LaunchedProxy]: with socket.socket() as reserve: reserve.bind(("127.0.0.1", 0)) port: Final = reserve.getsockname()[1] @@ -116,18 +123,7 @@ def owned_proxy_process( start_new_session=True, ) try: - with httpx.Client(base_url=f"http://127.0.0.1:{port}", timeout=15, trust_env=False) as client: - deadline: Final = time.monotonic() + 70 - while True: - assert process.poll() is None, "Owned proxy exited before readiness" - try: - if client.get("/health/readiness", timeout=2).status_code == 200: - break - except httpx.TransportError: - pass - assert time.monotonic() < deadline, "Owned proxy readiness deadline exceeded" - time.sleep(0.1) - yield OwnedProxy(Gateway(client, gateway.key, gateway.upstream_url), process, log_path) + yield LaunchedProxy(port, process, log_path) finally: root_stopped: Final = stop_root_process(process) residual: Final = group_members(process.pid) @@ -142,3 +138,55 @@ def owned_proxy_process( survivors: Final = group_members(process.pid) assert not survivors, "Owned proxy child survived cleanup" assert root_stopped and not remaining, "Owned proxy required forced cleanup" + + +def refused_boot_log( + gateway: Gateway, + directory: Path, + overrides: Mapping[str, str], + *, + config: Path | None = None, +) -> str: + """Start the proxy and return its log once it exits non-zero instead of becoming ready.""" + with launched_proxy(gateway, directory, overrides, config=config) as launched: + with httpx.Client(base_url=f"http://127.0.0.1:{launched.port}", timeout=15, trust_env=False) as client: + deadline: Final = time.monotonic() + 70 + while launched.process.poll() is None: + try: + ready: Final = client.get("/health/readiness", timeout=2).status_code == 200 + except httpx.TransportError: + ready = False + assert not ready, f"Proxy became ready instead of refusing to boot:\n{launched.log.read_text()}" + assert time.monotonic() < deadline, "Proxy neither exited nor became ready within the deadline" + time.sleep(0.1) + assert launched.process.returncode != 0, ( + f"Proxy exited 0 instead of refusing to boot:\n{launched.log.read_text()}" + ) + return launched.log.read_text() + + +@contextmanager +def owned_proxy_process( + gateway: Gateway, + directory: Path, + overrides: Mapping[str, str], + *, + config: Path | None = None, + remove_environment: tuple[str, ...] = (), + workers: int = 1, +) -> Iterator[OwnedProxy]: + with launched_proxy( + gateway, directory, overrides, config=config, remove_environment=remove_environment, workers=workers + ) as launched: + with httpx.Client(base_url=f"http://127.0.0.1:{launched.port}", timeout=15, trust_env=False) as client: + deadline: Final = time.monotonic() + 70 + while True: + assert launched.process.poll() is None, "Owned proxy exited before readiness" + try: + if client.get("/health/readiness", timeout=2).status_code == 200: + break + except httpx.TransportError: + pass + assert time.monotonic() < deadline, "Owned proxy readiness deadline exceeded" + time.sleep(0.1) + yield OwnedProxy(Gateway(client, gateway.key, gateway.upstream_url), launched.process, launched.log) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py new file mode 100644 index 00000000000..5cf69f21ab3 --- /dev/null +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -0,0 +1,66 @@ +"""LITELLM_FIPS_MODE is a boot gate: the proxy refuses to serve unless the process really enforces FIPS. + +Every leg launches the real proxy binary against the suite's Postgres and asserts on what an operator sees: +exit status and the refusal text in the log. Nothing is patched inside the proxy. +""" + +import hashlib +from pathlib import Path +from typing import Final + +import pytest +import yaml + +from tests.integration._support.client import Gateway +from tests.integration._support.process import owned_proxy, refused_boot_log + +REFUSAL: Final = "LiteLLM proxy refused to start" + + +def _this_python_enforces_fips() -> bool: + try: + hashlib.md5(b"probe", usedforsecurity=True) + except ValueError: + return True + return False + + +@pytest.mark.covers("other.configuration.fips_mode.refuses_boot_when_provider_does_not_enforce_fips") +def test_fips_mode_refuses_to_serve_when_this_python_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None: + if _this_python_enforces_fips(): + pytest.skip("Runner OpenSSL enforces FIPS, so this leg cannot observe the non-enforcing refusal") + log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}) + assert REFUSAL in log, log + assert "LITELLM_FIPS_MODE" in log and "does not enforce FIPS" in log, log + + +@pytest.mark.covers("other.configuration.fips_mode.refuses_boot_with_tls_verification_disabled") +@pytest.mark.parametrize("source", ("environment", "config")) +def test_fips_mode_refuses_to_serve_with_tls_verification_disabled( + gateway: Gateway, tmp_path: Path, source: str +) -> None: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + path: Final = tmp_path / "ssl_verify_off.yaml" + settings: Final = {**config.get("litellm_settings", {}), "ssl_verify": False} + path.write_text(yaml.safe_dump({**config, "litellm_settings": settings})) + log: Final = ( + refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true", "SSL_VERIFY": "false"}) + if source == "environment" + else refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}, config=path) + ) + assert REFUSAL in log, log + assert "TLS certificate verification is disabled" in log, log + assert ("SSL_VERIFY" if source == "environment" else "litellm_settings.ssl_verify") in log, log + + +@pytest.mark.covers("other.configuration.fips_mode.refuses_boot_on_malformed_value") +def test_fips_mode_refuses_to_serve_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: + log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}) + assert REFUSAL in log, log + assert "LITELLM_FIPS_MODE=enforced" in log and "true or false" in log, log + + +@pytest.mark.covers("other.configuration.fips_mode.off_leaves_boot_unchanged") +def test_fips_mode_off_serves_even_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: + with owned_proxy(gateway, tmp_path, {"LITELLM_FIPS_MODE": "false", "SSL_VERIFY": "false"}) as candidate: + assert candidate.client.get("/health/readiness").status_code == 200 diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index a8d9cf1df8b..673f62fd731 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -76,6 +76,21 @@ "tests/integration/configuration/test_effective_settings.py::test_credential_value_update_and_model_reload_reach_provider": [ "mgmt.credential.update.saved_value_reaches_wire" ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_when_this_python_does_not_enforce_fips": [ + "other.configuration.fips_mode.refuses_boot_when_provider_does_not_enforce_fips" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_with_tls_verification_disabled[environment]": [ + "other.configuration.fips_mode.refuses_boot_with_tls_verification_disabled" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_with_tls_verification_disabled[config]": [ + "other.configuration.fips_mode.refuses_boot_with_tls_verification_disabled" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_on_a_value_that_is_not_a_boolean": [ + "other.configuration.fips_mode.refuses_boot_on_malformed_value" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_off_serves_even_with_tls_verification_disabled": [ + "other.configuration.fips_mode.off_leaves_boot_unchanged" + ], "tests/integration/management/test_partial_update_sequences.py::test_denied_key_update_preserves_saved_grants_and_serving": [ "mgmt.key.update.denied_request_preserves_effective_state" ], diff --git a/tests/test_litellm/proxy/common_utils/test_fips.py b/tests/test_litellm/proxy/common_utils/test_fips.py new file mode 100644 index 00000000000..e934cafbdda --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_fips.py @@ -0,0 +1,118 @@ +import hashlib + +import pytest + +from litellm.proxy.common_utils.fips import ( + FipsModeError, + FipsModeOff, + FipsModeOn, + MalformedFipsMode, + ProviderDoesNotEnforceFips, + TlsVerificationDisabled, + enforce_fips_boot_verdict, + fips_boot_verdict, + is_fips_mode, + openssl_enforces_fips, + parse_fips_mode, +) + + +def _verdict( + raw: str | None, + *, + enforcing: bool = True, + ssl_env: str | None = None, + ssl_setting: object = True, +): + return fips_boot_verdict( + raw_fips_mode=raw, + provider_enforces_fips=lambda: enforcing, + ssl_verify_environment=ssl_env, + ssl_verify_setting=ssl_setting, + ) + + +@pytest.mark.parametrize("raw", [None, "false", "0", "no", "off", "", " False "]) +def test_unset_and_false_spellings_leave_fips_mode_off(raw): + assert parse_fips_mode(raw) == FipsModeOff() + assert is_fips_mode({"LITELLM_FIPS_MODE": raw}.get) is False + + +@pytest.mark.parametrize("raw", ["true", "1", "yes", "on", " TRUE "]) +def test_true_spellings_turn_fips_mode_on(raw): + assert parse_fips_mode(raw) == FipsModeOn() + assert is_fips_mode({"LITELLM_FIPS_MODE": raw}.get) is True + + +@pytest.mark.parametrize("raw", ["enforced", "2", "strict", "yes please"]) +def test_anything_else_is_malformed_and_refused_with_the_offending_value(raw): + assert parse_fips_mode(raw) == MalformedFipsMode(value=raw) + with pytest.raises(FipsModeError) as refused: + enforce_fips_boot_verdict(_verdict(raw, enforcing=False), announce=lambda _: None) + assert f"LITELLM_FIPS_MODE={raw} is not a boolean" in str(refused.value) + assert "true or false" in str(refused.value) + + +def test_off_never_consults_the_provider_or_tls_settings(): + def explode() -> bool: + raise AssertionError("provider probe must not run while FIPS mode is off") + + verdict = fips_boot_verdict( + raw_fips_mode=None, provider_enforces_fips=explode, ssl_verify_environment="false", ssl_verify_setting=False + ) + assert verdict == FipsModeOff() + enforce_fips_boot_verdict(verdict, announce=lambda _: pytest.fail("nothing to announce when off")) + + +def test_on_with_an_enforcing_provider_and_verified_tls_boots(): + verdict = _verdict("true", enforcing=True, ssl_env="true", ssl_setting="/etc/ssl/certs/ca.pem") + assert verdict == FipsModeOn() + enforce_fips_boot_verdict(verdict, announce=lambda _: pytest.fail("nothing to announce when on")) + + +def test_on_with_a_non_enforcing_provider_is_refused_and_names_the_fix(): + announced = [] + with pytest.raises(FipsModeError) as refused: + enforce_fips_boot_verdict(_verdict("true", enforcing=False), announce=announced.append) + message = str(refused.value) + assert message.startswith("LiteLLM proxy refused to start") + assert "LITELLM_FIPS_MODE is on but this Python does not enforce FIPS" in message + assert "FIPS image" in message + assert announced == [f"\n{message}\n\n"] + + +@pytest.mark.parametrize( + "ssl_env, ssl_setting, sources", + [ + ("false", True, ("SSL_VERIFY",)), + ("0", True, ("SSL_VERIFY",)), + (None, False, ("litellm_settings.ssl_verify",)), + (None, "False", ("litellm_settings.ssl_verify",)), + ("no", False, ("SSL_VERIFY", "litellm_settings.ssl_verify")), + ], +) +def test_disabled_tls_verification_is_refused_naming_every_source(ssl_env, ssl_setting, sources): + verdict = _verdict("true", enforcing=True, ssl_env=ssl_env, ssl_setting=ssl_setting) + assert verdict == TlsVerificationDisabled(sources=sources) + with pytest.raises(FipsModeError) as refused: + enforce_fips_boot_verdict(verdict, announce=lambda _: None) + assert "TLS certificate verification is disabled by " + " and ".join(sources) in str(refused.value) + + +@pytest.mark.parametrize("ssl_setting", [True, "true", "/etc/ssl/certs/ca.pem", None, ""]) +def test_verified_or_custom_bundle_tls_settings_are_not_treated_as_disabled(ssl_setting): + assert _verdict("true", enforcing=True, ssl_setting=ssl_setting) == FipsModeOn() + + +def test_disabled_tls_is_reported_before_the_provider_so_operators_see_config_mistakes_first(): + assert _verdict("true", enforcing=False, ssl_env="false") == TlsVerificationDisabled(sources=("SSL_VERIFY",)) + assert _verdict("true", enforcing=False) == ProviderDoesNotEnforceFips() + + +def test_provider_probe_agrees_with_whether_md5_is_usable_for_security_here(): + try: + hashlib.md5(b"", usedforsecurity=True) + except ValueError: + assert openssl_enforces_fips() is True + else: + assert openssl_enforces_fips() is False diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 89fd9c5c9d4..42ad00d6b13 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1681,6 +1681,110 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_this_python_does_not_enforce_fips(monkeypatch, tmp_path): + from fastapi import FastAPI + + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + _, announced = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError): + async with proxy_startup_event(FastAPI()): + pass + + assert len(announced) == 1 + assert "does not enforce FIPS" in announced[0] + + +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path): + import yaml + from fastapi import FastAPI + + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + config_path, announced = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + config_path.write_text( + yaml.dump({"general_settings": {"master_key": "sk-a-safe-master-key"}, "litellm_settings": {"ssl_verify": False}}) + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError): + async with proxy_startup_event(FastAPI()): + pass + + assert "TLS certificate verification is disabled by litellm_settings.ssl_verify" in announced[0] + + +class _PrismaClientWhoseUserTableCannotHash: + class _Table: + async def find_many(self, where): + raise ValueError("[digital envelope routines] unsupported") + + class _Db: + litellm_usertable = None + + def __init__(self): + self.db = self._Db() + self.db.litellm_usertable = self._Table() + self.writer_db = self.db + + async def connect(self): + pass + + async def disconnect(self): + pass + + async def check_view_exists(self): + pass + + async def health_check(self): + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fips_mode", ["true", "false"]) +async def test_proxy_startup_surfaces_a_password_migration_crypto_failure(monkeypatch, tmp_path, caplog, fips_mode): + from fastapi import FastAPI + + from litellm.proxy.proxy_server import proxy_startup_event + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable") + cannot_hash = _PrismaClientWhoseUserTableCannotHash() + + async def connected(**kwargs): + return cannot_hash + + monkeypatch.setattr("litellm.proxy.proxy_server.ProxyStartupEvent._setup_prisma_client", connected) + monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True) + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + if fips_mode == "true": + with pytest.raises(ValueError, match="digital envelope routines"): + async with proxy_startup_event(FastAPI()): + pass + else: + async with proxy_startup_event(FastAPI()): + await asyncio.sleep(0) + + failures = [r.getMessage() for r in caplog.records if "Password migration failed" in r.getMessage()] + assert len(failures) == 1 + assert "plaintext passwords stay unhashed" in failures[0] + assert "digital envelope routines" in failures[0] + + class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): self._ciphertext = ciphertext From 5c237840f4055a5a0058d09598ec51a66fae8e27 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:31:01 +0000 Subject: [PATCH 02/11] refactor(proxy): match ssl_verify off detection to runtime str_to_bool semantics Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_utils/fips.py | 4 +++- tests/integration/_support/process.py | 15 ++++++++++----- .../test_litellm/proxy/common_utils/test_fips.py | 6 +++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 33f35388567..8a3c26b3220 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -6,6 +6,8 @@ from typing import Final from typing_extensions import assert_never +from litellm.secret_managers.main import str_to_bool + FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" SSL_VERIFY_ENV_VAR: Final = "SSL_VERIFY" SSL_VERIFY_SETTING: Final = "litellm_settings.ssl_verify" @@ -146,6 +148,6 @@ def _is_off(value: object) -> bool: case bool(): return value is False case str(): - return value.strip().lower() in _FALSE_VALUES - {""} + return str_to_bool(value) is False case _: return False diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 6dec960097b..f65ab0e8093 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -140,6 +140,13 @@ def launched_proxy( assert root_stopped and not remaining, "Owned proxy required forced cleanup" +def _is_ready(client: httpx.Client) -> bool: + try: + return client.get("/health/readiness", timeout=2).status_code == 200 + except httpx.TransportError: + return False + + def refused_boot_log( gateway: Gateway, directory: Path, @@ -152,11 +159,9 @@ def refused_boot_log( with httpx.Client(base_url=f"http://127.0.0.1:{launched.port}", timeout=15, trust_env=False) as client: deadline: Final = time.monotonic() + 70 while launched.process.poll() is None: - try: - ready: Final = client.get("/health/readiness", timeout=2).status_code == 200 - except httpx.TransportError: - ready = False - assert not ready, f"Proxy became ready instead of refusing to boot:\n{launched.log.read_text()}" + assert not _is_ready(client), ( + f"Proxy became ready instead of refusing to boot:\n{launched.log.read_text()}" + ) assert time.monotonic() < deadline, "Proxy neither exited nor became ready within the deadline" time.sleep(0.1) assert launched.process.returncode != 0, ( diff --git a/tests/test_litellm/proxy/common_utils/test_fips.py b/tests/test_litellm/proxy/common_utils/test_fips.py index e934cafbdda..4b50ae4ad20 100644 --- a/tests/test_litellm/proxy/common_utils/test_fips.py +++ b/tests/test_litellm/proxy/common_utils/test_fips.py @@ -85,10 +85,10 @@ def test_on_with_a_non_enforcing_provider_is_refused_and_names_the_fix(): "ssl_env, ssl_setting, sources", [ ("false", True, ("SSL_VERIFY",)), - ("0", True, ("SSL_VERIFY",)), + (" FALSE ", True, ("SSL_VERIFY",)), (None, False, ("litellm_settings.ssl_verify",)), (None, "False", ("litellm_settings.ssl_verify",)), - ("no", False, ("SSL_VERIFY", "litellm_settings.ssl_verify")), + ("false", False, ("SSL_VERIFY", "litellm_settings.ssl_verify")), ], ) def test_disabled_tls_verification_is_refused_naming_every_source(ssl_env, ssl_setting, sources): @@ -99,7 +99,7 @@ def test_disabled_tls_verification_is_refused_naming_every_source(ssl_env, ssl_s assert "TLS certificate verification is disabled by " + " and ".join(sources) in str(refused.value) -@pytest.mark.parametrize("ssl_setting", [True, "true", "/etc/ssl/certs/ca.pem", None, ""]) +@pytest.mark.parametrize("ssl_setting", [True, "true", "/etc/ssl/certs/ca.pem", None, "", "0", "no"]) def test_verified_or_custom_bundle_tls_settings_are_not_treated_as_disabled(ssl_setting): assert _verdict("true", enforcing=True, ssl_setting=ssl_setting) == FipsModeOn() From 6ae5df66c1d24ba5ac460c2338ea01585753b04a Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:45:39 +0000 Subject: [PATCH 03/11] refactor(proxy): drop tautological fips probe test and satisfy CodeQL return checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_utils/fips.py | 23 ++++++++----------- .../proxy/common_utils/test_fips.py | 12 ---------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 8a3c26b3220..07bd4d2462c 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -121,9 +121,9 @@ def enforce_fips_boot_verdict(verdict: FipsBootVerdict, announce: Callable[[str] def render_refusal(refusal: FipsBootRefusal) -> str: match refusal: - case MalformedFipsMode(value=value): + case MalformedFipsMode(): return ( - f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR}={value} is not a boolean.\n" + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR}={refusal.value} is not a boolean.\n" f"Set {FIPS_MODE_ENV_VAR} to true or false, or unset it." ) case ProviderDoesNotEnforceFips(): @@ -133,21 +133,18 @@ def render_refusal(refusal: FipsBootRefusal) -> str: "protected with algorithms the FIPS 140-3 policy forbids. Run the proxy from a FIPS image whose\n" f"OpenSSL FIPS provider is enabled, or unset {FIPS_MODE_ENV_VAR} on a non-FIPS runtime." ) - case TlsVerificationDisabled(sources=sources): + case TlsVerificationDisabled(): return ( f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by " - f"{' and '.join(sources)}.\nFIPS deployments must verify upstream certificates, so remove the " + f"{' and '.join(refusal.sources)}.\nFIPS deployments must verify upstream certificates, so remove the " "override or point ssl_verify at a CA bundle instead." ) - case _: - assert_never(refusal) + return assert_never(refusal) def _is_off(value: object) -> bool: - match value: - case bool(): - return value is False - case str(): - return str_to_bool(value) is False - case _: - return False + if isinstance(value, bool): + return value is False + if isinstance(value, str): + return str_to_bool(value) is False + return False diff --git a/tests/test_litellm/proxy/common_utils/test_fips.py b/tests/test_litellm/proxy/common_utils/test_fips.py index 4b50ae4ad20..0e8138856c0 100644 --- a/tests/test_litellm/proxy/common_utils/test_fips.py +++ b/tests/test_litellm/proxy/common_utils/test_fips.py @@ -1,5 +1,3 @@ -import hashlib - import pytest from litellm.proxy.common_utils.fips import ( @@ -12,7 +10,6 @@ from litellm.proxy.common_utils.fips import ( enforce_fips_boot_verdict, fips_boot_verdict, is_fips_mode, - openssl_enforces_fips, parse_fips_mode, ) @@ -107,12 +104,3 @@ def test_verified_or_custom_bundle_tls_settings_are_not_treated_as_disabled(ssl_ def test_disabled_tls_is_reported_before_the_provider_so_operators_see_config_mistakes_first(): assert _verdict("true", enforcing=False, ssl_env="false") == TlsVerificationDisabled(sources=("SSL_VERIFY",)) assert _verdict("true", enforcing=False) == ProviderDoesNotEnforceFips() - - -def test_provider_probe_agrees_with_whether_md5_is_usable_for_security_here(): - try: - hashlib.md5(b"", usedforsecurity=True) - except ValueError: - assert openssl_enforces_fips() is True - else: - assert openssl_enforces_fips() is False From e6a5d32cd9925caedbd1eb067b86772031dcf556 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 06:07:49 +0000 Subject: [PATCH 04/11] test(proxy): inject the fake Prisma client through the module boundary instead of patching _setup_prisma_client Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/proxy/test_proxy_server.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 42ad00d6b13..39e4a417d19 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1733,7 +1733,7 @@ class _PrismaClientWhoseUserTableCannotHash: class _Db: litellm_usertable = None - def __init__(self): + def __init__(self, database_url, proxy_logging_obj): self.db = self._Db() self.db.litellm_usertable = self._Table() self.writer_db = self.db @@ -1744,6 +1744,9 @@ class _PrismaClientWhoseUserTableCannotHash: async def disconnect(self): pass + def start_view_setup_task(self): + pass + async def check_view_exists(self): pass @@ -1761,12 +1764,7 @@ async def test_proxy_startup_surfaces_a_password_migration_crypto_failure(monkey _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable") - cannot_hash = _PrismaClientWhoseUserTableCannotHash() - - async def connected(**kwargs): - return cannot_hash - - monkeypatch.setattr("litellm.proxy.proxy_server.ProxyStartupEvent._setup_prisma_client", connected) + monkeypatch.setattr("litellm.proxy.proxy_server.PrismaClient", _PrismaClientWhoseUserTableCannotHash) monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True) monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) From 0660064397d967040010f1ad8b82a8be65e3eca5 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:09:05 +0000 Subject: [PATCH 05/11] feat(proxy): store password hashes as PBKDF2-HMAC-SHA256 and rehash legacy rows on login Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/login_utils.py | 5 +- litellm/proxy/proxy_server.py | 4 +- litellm/proxy/utils.py | 88 ++++--- .../authorization/test_password_hash_login.py | 215 ++++++++++++++++++ .../proxy/auth/test_login_utils.py | 47 ++++ .../proxy/auth/test_password_hashing.py | 112 +++++++-- .../test_password_endpoints.py | 3 +- .../prisma_and_spend/test_password_helpers.py | 39 ++-- .../test_prisma_client_health.py | 5 +- 9 files changed, 443 insertions(+), 75 deletions(-) create mode 100644 tests/integration/authorization/test_password_hash_login.py diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 629b31024e2..e93bc2a56f2 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -42,6 +42,7 @@ from litellm.proxy.utils import ( PrismaClient, get_server_root_path, hash_password, + needs_password_rehash, verify_password, ) from litellm.repositories.user_repository import UserRepository @@ -106,8 +107,8 @@ async def screen_login_password_for_breach( async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None: - """Rehash legacy password (SHA256) to scrypt on successful login.""" - if stored.startswith("scrypt:"): + """Rehash legacy scrypt or SHA256 password rows to pbkdf2 on successful login.""" + if not needs_password_rehash(stored): return from litellm.proxy.proxy_server import prisma_client diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a563b901fc4..dc8a62d488b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -809,7 +809,7 @@ from litellm.proxy.utils import ( hash_token, invalidate_config_param, litellm_config_cache, - migrate_passwords_to_scrypt_async, + migrate_plaintext_passwords_async, model_dump_with_preserved_fields, prefetch_config_params, update_spend, @@ -1336,7 +1336,7 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: async def _run_pw_migration(): try: - result: Final = await migrate_passwords_to_scrypt_async(prisma_client) + result: Final = await migrate_plaintext_passwords_async(prisma_client) verbose_proxy_logger.info("Password migration: %s", result) except ValueError as e: verbose_proxy_logger.error( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bc64293c9b3..bcd6e30a020 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1,4 +1,6 @@ import asyncio +import base64 +import binascii import contextlib import copy import hashlib @@ -6,6 +8,7 @@ import inspect import json import math import os +import secrets import smtplib import ssl import sys @@ -71,6 +74,7 @@ from litellm.proxy._types import ( SpendLogsPayload, ) from litellm.proxy.bug_report_config import build_proxy_bug_report +from litellm.proxy.common_utils.fips import is_fips_mode from litellm.proxy.common_utils.openai_error_payload import ( litellm_call_id_headers, openai_error_param, @@ -7043,54 +7047,86 @@ def hash_token(token: str): return hashed_token -def hash_password(password: str) -> str: - """Hash a password using scrypt with a random salt.""" - import base64 - import hashlib - import os +PBKDF2_ITERATIONS: Final = 600_000 +PBKDF2_PREFIX: Final = "pbkdf2:sha256:" +SCRYPT_PREFIX: Final = "scrypt:" + +def hash_password(password: str) -> str: + """Hash a password as ``pbkdf2:sha256:::``. + + Iteration count is the OWASP Password Storage Cheat Sheet floor for + PBKDF2-HMAC-SHA256; PBKDF2 is a FIPS-approved primitive so the same row + format is written on every image, FIPS mode or not. + """ salt: Final = os.urandom(16) - dk: Final = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) - return "scrypt:" + base64.b64encode(salt + dk).decode() + derived: Final = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, PBKDF2_ITERATIONS, dklen=32) + return f"{PBKDF2_PREFIX}{PBKDF2_ITERATIONS}:{base64.b64encode(salt).decode()}:{base64.b64encode(derived).decode()}" + + +def _is_sha256_hex(value: str) -> bool: + return len(value) == 64 and all(c in "0123456789abcdef" for c in value) + + +def _is_hashed_password(value: str) -> bool: + return value.startswith(PBKDF2_PREFIX) or value.startswith(SCRYPT_PREFIX) or _is_sha256_hex(value) + + +def needs_password_rehash(stored: str) -> bool: + return not stored.startswith(PBKDF2_PREFIX) + + +def _verify_pbkdf2(password: str, stored: str) -> bool: + try: + scheme, digest, iterations, salt, derived = stored.split(":") + if (scheme, digest) != ("pbkdf2", "sha256"): + return False + expected: Final = hashlib.pbkdf2_hmac( + "sha256", password.encode(), base64.b64decode(salt, validate=True), int(iterations) + ) + return secrets.compare_digest(base64.b64decode(derived, validate=True), expected) + except (ValueError, binascii.Error, TypeError): + return False def verify_password(password: str, stored: str) -> bool: - """Verify a password against a stored hash. Supports scrypt and SHA256.""" - import base64 - import hashlib - import secrets - - if stored.startswith("scrypt:"): + """Verify a password against a stored hash. Supports pbkdf2, scrypt and SHA256 rows.""" + if stored.startswith(PBKDF2_PREFIX): + return _verify_pbkdf2(password, stored) + if stored.startswith(SCRYPT_PREFIX): + if is_fips_mode(): + verbose_proxy_logger.error( + "LITELLM_FIPS_MODE is on and this account still has a scrypt password hash, " + "which is not a FIPS approved primitive. Reset the password " + "(POST /user/password/change, or an admin POST /user/update with a new password) " + "so it is stored as pbkdf2 and the account can sign in" + ) + return False try: raw: Final = base64.b64decode(stored[7:]) salt, dk = raw[:16], raw[16:] dk2: Final = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) return secrets.compare_digest(dk, dk2) - except Exception: + except (ValueError, binascii.Error, TypeError): return False # SHA256 fallback (not vulnerable to pass-the-hash: checks sha256(input) == stored) - if len(stored) == 64 and all(c in "0123456789abcdef" for c in stored): + if _is_sha256_hex(stored): return secrets.compare_digest(hashlib.sha256(password.encode()).hexdigest().encode(), stored.encode()) return False -async def migrate_passwords_to_scrypt_async(prisma_client) -> str: +async def migrate_plaintext_passwords_async(prisma_client) -> str: """ - Migrate plaintext passwords in the DB to scrypt. SHA256 passwords - are left alone (they migrate on next login via the SHA256 fallback). - Skips quickly if no plaintext passwords exist. + Migrate plaintext passwords in the DB to pbkdf2. Already-hashed rows + (pbkdf2, scrypt, sha256) are left alone; scrypt and sha256 rows rehash + on next successful login. Skips quickly if no plaintext passwords exist. """ all_with_pw: Final = await UserRepository(prisma_client).table.find_many( where={"password": {"not": None}}, ) - def _is_sha256_hex(s: str) -> bool: - return len(s) == 64 and all(c in "0123456789abcdef" for c in s) - plaintext_users: Final = [ - (u.user_id, u.password) - for u in all_with_pw - if u.password and not u.password.startswith("scrypt:") and not _is_sha256_hex(u.password) + (u.user_id, u.password) for u in all_with_pw if u.password and not _is_hashed_password(u.password) ] if not plaintext_users: return "No plaintext passwords found" @@ -7100,7 +7136,7 @@ async def migrate_passwords_to_scrypt_async(prisma_client) -> str: where={"user_id": user_id}, data={"password": hash_password(plaintext_password)}, ) - return f"Migrated {len(plaintext_users)} plaintext passwords to scrypt" + return f"Migrated {len(plaintext_users)} plaintext passwords to pbkdf2" def _hash_token_if_needed(token: str) -> str: diff --git a/tests/integration/authorization/test_password_hash_login.py b/tests/integration/authorization/test_password_hash_login.py new file mode 100644 index 00000000000..dc6c3d8a8fe --- /dev/null +++ b/tests/integration/authorization/test_password_hash_login.py @@ -0,0 +1,215 @@ +"""Stored passwords are PBKDF2-HMAC-SHA256 rows; scrypt and legacy SHA256 rows still sign in and get rehashed. + +Every cell drives the real /login form and the management endpoints of a proxy this module owns (two workers, +login throttle off, breach screening off so no request leaves the box) and reads the stored row back from +Postgres. The expected hash is recomputed here with hashlib, never with litellm code. +""" + +import base64 +import hashlib +import os +import uuid +from collections.abc import Callable, Iterator +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final + +import jwt +import psutil +import psycopg +import pytest +import yaml + +from tests.integration._support.client import Gateway, eventually, gateway_from_environment +from tests.integration._support.database import read_rows +from tests.integration._support.process import OwnedProxy, owned_proxy_process + +PBKDF2_ITERATIONS: Final = 600_000 +PASSWORD: Final = "Correct-Horse-9-Battery" +WRONG_PASSWORD: Final = "Wrong-Horse-9-Battery" +BURST_SIZE: Final = 30 + + +@pytest.fixture(scope="module") +def password_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]: + directory: Final = tmp_path_factory.mktemp("password-proxy") + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + settings: Final = {**config["general_settings"], "password_policy_check_breached_passwords": False} + path: Final = directory / "config.yaml" + path.write_text(yaml.safe_dump({**config, "general_settings": settings})) + with gateway_from_environment() as suite_gateway: + with owned_proxy_process( + suite_gateway, + directory, + {"LITELLM_DISABLE_LOGIN_RATE_LIMIT": "true"}, + config=path, + workers=2, + ) as owned: + yield owned + + +@pytest.fixture +def proxy(password_proxy: OwnedProxy) -> Gateway: + return password_proxy.gateway + + +def _stored_password(user_id: str) -> str: + rows: Final = read_rows('SELECT password FROM "LiteLLM_UserTable" WHERE user_id = %s', (user_id,)) + assert len(rows) == 1, rows + stored: Final = rows[0]["password"] + assert isinstance(stored, str), rows + return stored + + +def _write_stored_password(user_id: str, stored: str) -> None: + with psycopg.connect(os.environ["DATABASE_URL"]) as connection: + connection.execute('UPDATE "LiteLLM_UserTable" SET password = %s WHERE user_id = %s', (stored, user_id)) + connection.commit() + + +def _scrypt_row(password: str) -> str: + salt: Final = os.urandom(16) + derived: Final = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) + return "scrypt:" + base64.b64encode(salt + derived).decode() + + +def _sha256_row(password: str) -> str: + return hashlib.sha256(password.encode()).hexdigest() + + +def _pbkdf2_matches(stored: str, password: str) -> bool: + """Independent check of a ``pbkdf2:sha256:::`` row with the stdlib.""" + scheme, digest, iterations, salt, derived = stored.split(":") + assert (scheme, digest, int(iterations)) == ("pbkdf2", "sha256", PBKDF2_ITERATIONS), stored + expected: Final = hashlib.pbkdf2_hmac("sha256", password.encode(), base64.b64decode(salt), int(iterations)) + return base64.b64decode(derived) == expected + + +def _login(proxy: Gateway, email: str, password: str) -> int: + response: Final = proxy.client.post( + "/login", data={"username": email, "password": password}, follow_redirects=False + ) + if response.status_code == 303: + assert response.headers["location"].endswith("/ui?login=success"), response.headers + assert "token=" in response.headers.get("set-cookie", ""), response.headers + return response.status_code + + +def _user_with_password(proxy: Gateway, password: str | None) -> tuple[str, str]: + email: Final = f"integration-{uuid.uuid4().hex}@example.com" + created: Final = proxy.post( + "/user/new", + {"user_id": f"integration-{uuid.uuid4().hex}", "user_email": email, "auto_create_key": False}, + ) + user_id: Final = created["user_id"] + assert isinstance(user_id, str), created + if password is not None: + proxy.post("/user/update", {"user_id": user_id, "password": password}) + return user_id, email + + +def _delete_user(proxy: Gateway, user_id: str) -> None: + response: Final = proxy.request("POST", "/user/delete", {"user_ids": [user_id]}) + assert response.status_code == 200, response.text + + +def test_new_user_password_is_stored_as_pbkdf2_and_signs_in(proxy: Gateway) -> None: + user_id, email = _user_with_password(proxy, PASSWORD) + try: + stored: Final = _stored_password(user_id) + assert stored.startswith("pbkdf2:sha256:"), stored + assert _pbkdf2_matches(stored, PASSWORD), stored + assert _login(proxy, email, PASSWORD) == 303 + assert _stored_password(user_id) == stored, "a pbkdf2 row must not be rewritten on login" + finally: + _delete_user(proxy, user_id) + + +def test_wrong_password_is_rejected_and_row_is_untouched(proxy: Gateway) -> None: + user_id, email = _user_with_password(proxy, PASSWORD) + try: + before: Final = _stored_password(user_id) + assert _login(proxy, email, WRONG_PASSWORD) == 401 + assert _login(proxy, email, "") == 401 + assert _stored_password(user_id) == before + finally: + _delete_user(proxy, user_id) + + +@pytest.mark.parametrize("legacy_row", (_scrypt_row, _sha256_row), ids=("scrypt", "sha256")) +def test_legacy_hash_signs_in_and_is_rehashed_to_pbkdf2(proxy: Gateway, legacy_row: Callable[[str], str]) -> None: + user_id, email = _user_with_password(proxy, None) + try: + legacy: Final = legacy_row(PASSWORD) + _write_stored_password(user_id, legacy) + assert _login(proxy, email, WRONG_PASSWORD) == 401 + assert _stored_password(user_id) == legacy, "a rejected login must not rewrite the row" + assert _login(proxy, email, PASSWORD) == 303 + rehashed: Final = eventually(lambda: _stored_password(user_id), lambda row: row.startswith("pbkdf2:")) + assert _pbkdf2_matches(rehashed, PASSWORD), rehashed + assert _login(proxy, email, PASSWORD) == 303 + assert _login(proxy, email, WRONG_PASSWORD) == 401 + finally: + _delete_user(proxy, user_id) + + +@pytest.mark.parametrize("garbage", ("", "plaintext-password", "pbkdf2:sha256:1:not-base64:!!", "scrypt:%%%")) +def test_unreadable_stored_row_rejects_every_password(proxy: Gateway, garbage: str) -> None: + user_id, email = _user_with_password(proxy, None) + try: + _write_stored_password(user_id, garbage) + assert _login(proxy, email, garbage) == 401 + assert _login(proxy, email, PASSWORD) == 401 + assert _stored_password(user_id) == garbage + finally: + _delete_user(proxy, user_id) + + +def test_changed_password_is_stored_as_pbkdf2(proxy: Gateway) -> None: + user_id, email = _user_with_password(proxy, PASSWORD) + try: + signed_in: Final = proxy.client.post( + "/login", data={"username": email, "password": PASSWORD}, follow_redirects=False + ) + assert signed_in.status_code == 303, signed_in.text + session: Final = jwt.decode(signed_in.cookies["token"], options={"verify_signature": False})["key"] + assert isinstance(session, str) and session.startswith("sk-"), session + new_password: Final = "Fresh-Horse-7-Battery" + changed: Final = proxy.request( + "POST", + "/user/password/change", + {"current_password": PASSWORD, "new_password": new_password}, + key=session, + ) + assert changed.status_code == 200, changed.text + stored: Final = _stored_password(user_id) + assert stored.startswith("pbkdf2:sha256:"), stored + assert _pbkdf2_matches(stored, new_password), stored + assert _login(proxy, email, PASSWORD) == 401 + assert _login(proxy, email, new_password) == 303 + finally: + _delete_user(proxy, user_id) + + +def test_concurrent_logins_on_a_scrypt_row_rehash_once_while_one_worker_is_killed( + password_proxy: OwnedProxy, +) -> None: + proxy: Final = password_proxy.gateway + user_id, email = _user_with_password(proxy, None) + try: + _write_stored_password(user_id, _scrypt_row(PASSWORD)) + workers: Final = psutil.Process(password_proxy.process.pid).children(recursive=True) + assert len(workers) >= 2, workers + victim: Final = workers[0] + victim.kill() + eventually(lambda: victim.is_running() and victim.status() != psutil.STATUS_ZOMBIE, lambda alive: not alive) + attempts: Final = tuple(PASSWORD if index % 3 else WRONG_PASSWORD for index in range(BURST_SIZE)) + with ThreadPoolExecutor(max_workers=BURST_SIZE) as pool: + statuses: Final = tuple(pool.map(lambda password: _login(proxy, email, password), attempts)) + assert sorted(statuses) == sorted(303 if password == PASSWORD else 401 for password in attempts), statuses + rehashed: Final = eventually(lambda: _stored_password(user_id), lambda row: row.startswith("pbkdf2:")) + assert _pbkdf2_matches(rehashed, PASSWORD), rehashed + assert _login(proxy, email, PASSWORD) == 303 + assert _stored_password(user_id) == rehashed, "a settled pbkdf2 row must stay stable across logins" + finally: + _delete_user(proxy, user_id) diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index 1b15994e777..5ec37195c58 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -2343,3 +2343,50 @@ class TestScreenLoginPasswordForBreach: ) is True ) + + +class TestRehashPasswordIfNeeded: + """A successful login rewrites legacy hash rows to pbkdf2 in place.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("legacy_kind", ("scrypt", "sha256")) + async def test_legacy_row_is_rewritten_to_a_verifying_pbkdf2_row(self, legacy_kind): + import base64 + + from litellm.proxy.auth.login_utils import _rehash_password_if_needed + from litellm.proxy.utils import verify_password + + password = "rehash-me-1" + if legacy_kind == "scrypt": + salt = os.urandom(16) + derived = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) + stored = "scrypt:" + base64.b64encode(salt + derived).decode() + else: + stored = hashlib.sha256(password.encode()).hexdigest() + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.update = AsyncMock() + with patch( # test-quality-ok: the rehash writes to the database; faked so no DB is needed + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ): + await _rehash_password_if_needed("u-1", password, stored) + + update_kwargs = mock_prisma_client.db.litellm_usertable.update.await_args.kwargs + assert update_kwargs["where"] == {"user_id": "u-1"} + written = update_kwargs["data"]["password"] + assert written.startswith("pbkdf2:sha256:") + assert verify_password(password, written) + + @pytest.mark.asyncio + async def test_pbkdf2_row_triggers_no_update(self): + from litellm.proxy.auth.login_utils import _rehash_password_if_needed + from litellm.proxy.utils import hash_password + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.update = AsyncMock() + with patch( # test-quality-ok: the rehash writes to the database; faked so no DB is needed + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ): + await _rehash_password_if_needed("u-1", "rehash-me-1", hash_password("rehash-me-1")) + + mock_prisma_client.db.litellm_usertable.update.assert_not_called() diff --git a/tests/test_litellm/proxy/auth/test_password_hashing.py b/tests/test_litellm/proxy/auth/test_password_hashing.py index be4ae21264f..97ef252eeb2 100644 --- a/tests/test_litellm/proxy/auth/test_password_hashing.py +++ b/tests/test_litellm/proxy/auth/test_password_hashing.py @@ -1,31 +1,41 @@ """Tests for password hashing and verification utilities.""" +import base64 import hashlib +import logging +import os import pytest -from litellm.proxy.utils import hash_password, verify_password +from litellm._logging import verbose_proxy_logger +from litellm.proxy.utils import hash_password, needs_password_rehash, verify_password + + +def _scrypt_row(password: str) -> str: + salt = os.urandom(16) + derived = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) + return "scrypt:" + base64.b64encode(salt + derived).decode() + + +def _pbkdf2_row(password: str, iterations: int) -> str: + salt = os.urandom(16) + derived = hashlib.pbkdf2_hmac("sha256", password.encode(), salt, iterations) + return f"pbkdf2:sha256:{iterations}:{base64.b64encode(salt).decode()}:{base64.b64encode(derived).decode()}" class TestHashPassword: - def test_produces_scrypt_prefix(self): - assert hash_password("test").startswith("scrypt:") + def test_produces_pbkdf2_prefix_at_owasp_floor(self): + h = hash_password("test") + assert h.startswith("pbkdf2:sha256:600000:") + iterations = int(h.split(":")[2]) + assert iterations >= 600_000 def test_unique_salt_per_call(self): assert hash_password("same") != hash_password("same") - def test_output_length(self): - # "scrypt:" (7) + base64(48 bytes) (64) = 71 - assert len(hash_password("test")) == 71 - - -class TestVerifyPassword: - def test_correct_password(self): + def test_round_trip_and_wrong_password(self): h = hash_password("correct") assert verify_password("correct", h) is True - - def test_wrong_password(self): - h = hash_password("correct") assert verify_password("wrong", h) is False def test_empty_password(self): @@ -44,9 +54,65 @@ class TestVerifyPassword: assert verify_password(pw, h) is True +class TestVerifyPasswordFormats: + def test_row_with_higher_iteration_count_verifies(self): + stored = _pbkdf2_row("iterated", 700_000) + assert verify_password("iterated", stored) is True + assert verify_password("other", stored) is False + + def test_scrypt_row_verifies_when_fips_off(self, monkeypatch): + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + stored = _scrypt_row("legacy-scrypt-pass") + assert verify_password("legacy-scrypt-pass", stored) is True + assert verify_password("wrong", stored) is False + + def test_scrypt_row_rejected_and_logged_when_fips_on(self, monkeypatch, caplog): + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + stored = _scrypt_row("legacy-scrypt-pass") + with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): + assert verify_password("legacy-scrypt-pass", stored) is False + assert "scrypt" in caplog.text + assert "/user/password/change" in caplog.text + + def test_sha256_fallback_verifies_in_both_modes(self, monkeypatch): + stored = hashlib.sha256(b"oldpass").hexdigest() + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + assert verify_password("oldpass", stored) is True + assert verify_password("wrong", stored) is False + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + assert verify_password("oldpass", stored) is True + + @pytest.mark.parametrize( + "stored", + ( + "pbkdf2:sha256:600000", + "pbkdf2:sha256:600000:c2FsdA==", + "pbkdf2:sha256:600000:c2FsdA==:a2V5:extra", + "pbkdf2:sha256:not-an-int:c2FsdA==:a2V5", + "pbkdf2:sha256:600000:not-base64-!!:a2V5", + "pbkdf2:sha256:600000:c2FsdA==:not-base64-!!", + "pbkdf2:md5:600000:c2FsdA==:a2V5", + ), + ids=( + "missing_fields", + "four_fields", + "six_fields", + "non_int_iterations", + "bad_salt_base64", + "bad_key_base64", + "wrong_digest", + ), + ) + def test_malformed_pbkdf2_rows_return_false(self, stored): + assert verify_password("test", stored) is False + + def test_scrypt_invalid_base64_rejected(self): + assert verify_password("test", "scrypt:not-valid-base64!!!") is False + + class TestVerifyPasswordFallbacks: def test_sha256_fallback(self): - stored = hashlib.sha256("oldpass".encode()).hexdigest() + stored = hashlib.sha256(b"oldpass").hexdigest() assert verify_password("oldpass", stored) is True assert verify_password("wrong", stored) is False @@ -54,16 +120,18 @@ class TestVerifyPasswordFallbacks: # Plaintext fallback removed to prevent pass-the-hash attacks assert verify_password("plaintext", "plaintext") is False - def test_scrypt_preferred_over_fallbacks(self): - h = hash_password("test") - # Scrypt hash should not accidentally match as plaintext or SHA256 - assert verify_password("test", h) is True - assert h.startswith("scrypt:") - def test_sha256_not_confused_with_plaintext(self): # A 64-char hex string that isn't a valid SHA256 of the password fake_hex = "a" * 64 assert verify_password("test", fake_hex) is False - def test_scrypt_invalid_base64_rejected(self): - assert verify_password("test", "scrypt:not-valid-base64!!!") is False + +class TestNeedsPasswordRehash: + def test_legacy_rows_need_rehash(self): + assert needs_password_rehash(_scrypt_row("x")) is True + assert needs_password_rehash(hashlib.sha256(b"x").hexdigest()) is True + assert needs_password_rehash("plaintext") is True + + def test_pbkdf2_rows_do_not_need_rehash(self): + assert needs_password_rehash(hash_password("x")) is False + assert needs_password_rehash(_pbkdf2_row("x", 700_000)) is False diff --git a/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py index adff4eda47c..e3bfb12e6ab 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py @@ -75,7 +75,7 @@ def _hibp_client_recording(calls: list[httpx.Request]) -> AsyncHTTPHandler: @pytest.mark.asyncio -async def test_change_password_success_writes_new_scrypt_hash(): +async def test_change_password_success_writes_new_pbkdf2_hash(): from litellm.proxy._types import ChangePasswordRequest prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) @@ -99,6 +99,7 @@ async def test_change_password_success_writes_new_scrypt_hash(): assert update_kwargs["where"] == {"user_id": "user-123"} stored = update_kwargs["data"]["password"] assert stored != NEW_PASSWORD + assert stored.startswith("pbkdf2:sha256:") assert verify_password(NEW_PASSWORD, stored) # A successful change lifts any pending forced reset and re-arms the # login-time breach screen for the new password. diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_password_helpers.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_password_helpers.py index 3c028473479..e330601ad98 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_password_helpers.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_password_helpers.py @@ -4,18 +4,17 @@ Symbols pinned here: - ``hash_token`` - ``hash_password`` - ``verify_password`` - - ``migrate_passwords_to_scrypt_async`` + - ``migrate_plaintext_passwords_async`` - ``_hash_token_if_needed`` - - ``PrismaClient._is_sha256_hex`` (a nested helper inside - ``migrate_passwords_to_scrypt_async``; the pin list labels it under the - PrismaClient health cluster as a documentation artifact) + - ``_is_sha256_hex`` (used by ``migrate_plaintext_passwords_async``; the + pin list labels it under the PrismaClient health cluster as a + documentation artifact) """ from __future__ import annotations import hashlib from types import SimpleNamespace -from typing import List from unittest.mock import AsyncMock, MagicMock import pytest @@ -24,7 +23,7 @@ from litellm.proxy.utils import ( _hash_token_if_needed, hash_password, hash_token, - migrate_passwords_to_scrypt_async, + migrate_plaintext_passwords_async, verify_password, ) @@ -57,16 +56,16 @@ def test_hash_token_raises_for_non_string() -> None: hash_token(None) # type: ignore[arg-type] -def test_hash_password_uses_scrypt_prefix() -> None: +def test_hash_password_uses_pbkdf2_prefix() -> None: h = hash_password("hunter2") fields = { - "prefix": h[:7], + "prefix": h[:14], "min_length": len(h) > 60, "verifies_self": verify_password("hunter2", h), "rejects_other": verify_password("hunter3", h), } assert fields == { - "prefix": "scrypt:", + "prefix": "pbkdf2:sha256:", "min_length": True, "verifies_self": True, "rejects_other": False, @@ -133,9 +132,9 @@ def test_hash_token_if_needed_error_on_non_string() -> None: # --------------------------------------------------------------------------- -# migrate_passwords_to_scrypt_async — pins behavior of the nested -# ``_is_sha256_hex`` helper too: scrypt-prefixed and sha256-hex rows are -# left alone, plaintext rows are upgraded in place. +# migrate_plaintext_passwords_async — pins behavior of the +# ``_is_hashed_password`` helper too: pbkdf2-prefixed, scrypt-prefixed and +# sha256-hex rows are left alone, plaintext rows are upgraded in place. # --------------------------------------------------------------------------- @@ -152,11 +151,12 @@ async def test_migrate_passwords_skips_when_no_plaintext() -> None: return_value=[ _make_user("a", "scrypt:abc"), _make_user("b", sha), + _make_user("c", "pbkdf2:sha256:600000:c2FsdA==:a2V5"), ] ) pc.db.litellm_usertable.update = AsyncMock() - result = await migrate_passwords_to_scrypt_async(pc) + result = await migrate_plaintext_passwords_async(pc) outcome = { "message": result, "updates": pc.db.litellm_usertable.update.await_count, @@ -175,10 +175,11 @@ async def test_migrate_passwords_skips_when_no_plaintext() -> None: async def test_migrate_passwords_upgrades_only_plaintext_rows() -> None: pc = MagicMock() pc.db = MagicMock() - users: List[SimpleNamespace] = [ + users: list[SimpleNamespace] = [ _make_user("plaintext-user-1", "plain-1"), _make_user("plaintext-user-2", "plain-2"), _make_user("scrypt-user", "scrypt:already"), + _make_user("pbkdf2-user", "pbkdf2:sha256:600000:c2FsdA==:a2V5"), _make_user( "sha-user", hashlib.sha256(b"alreadyhashed").hexdigest(), @@ -188,7 +189,7 @@ async def test_migrate_passwords_upgrades_only_plaintext_rows() -> None: pc.db.litellm_usertable.find_many = AsyncMock(return_value=users) pc.db.litellm_usertable.update = AsyncMock() - result = await migrate_passwords_to_scrypt_async(pc) + result = await migrate_plaintext_passwords_async(pc) updated_user_ids = sorted( call.kwargs["where"]["user_id"] @@ -202,13 +203,13 @@ async def test_migrate_passwords_upgrades_only_plaintext_rows() -> None: "message": result, "update_count": pc.db.litellm_usertable.update.await_count, "updated_ids": updated_user_ids, - "all_scrypt_prefixed": new_password_prefixes, + "all_pbkdf2_prefixed": new_password_prefixes, } assert outcome == { - "message": "Migrated 2 plaintext passwords to scrypt", + "message": "Migrated 2 plaintext passwords to pbkdf2", "update_count": 2, "updated_ids": ["plaintext-user-1", "plaintext-user-2"], - "all_scrypt_prefixed": ["scrypt:", "scrypt:"], + "all_pbkdf2_prefixed": ["pbkdf2:", "pbkdf2:"], } @@ -220,4 +221,4 @@ async def test_migrate_passwords_raises_on_db_failure() -> None: side_effect=RuntimeError("db unavailable") ) with pytest.raises(RuntimeError, match="db unavailable"): - await migrate_passwords_to_scrypt_async(pc) + await migrate_plaintext_passwords_async(pc) diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py index fbe9934f06c..433016cb63a 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_health.py @@ -10,9 +10,8 @@ Symbols pinned here: - ``PrismaClient.get_health_check_history`` - ``PrismaClient.get_all_latest_health_checks`` - ``PrismaClient.get_latest_health_checks_for_models`` - - ``PrismaClient._is_sha256_hex`` (a nested helper inside - ``migrate_passwords_to_scrypt_async``; the pin list assigns it to this - cluster as a documentation artifact) + - ``_is_sha256_hex`` (used by ``migrate_plaintext_passwords_async``; the + pin list assigns it to this cluster as a documentation artifact) """ from __future__ import annotations From d58958c57739ed2216dcebf1fca524e220234df7 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:20:45 +0000 Subject: [PATCH 06/11] fix(proxy): point the FIPS scrypt rejection log at the admin reset path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 5 ++--- tests/test_litellm/proxy/auth/test_password_hashing.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bcd6e30a020..42dfbf7e421 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7097,9 +7097,8 @@ def verify_password(password: str, stored: str) -> bool: if is_fips_mode(): verbose_proxy_logger.error( "LITELLM_FIPS_MODE is on and this account still has a scrypt password hash, " - "which is not a FIPS approved primitive. Reset the password " - "(POST /user/password/change, or an admin POST /user/update with a new password) " - "so it is stored as pbkdf2 and the account can sign in" + "which is not a FIPS approved primitive. An admin must set a new password with " + "POST /user/update so it is stored as pbkdf2 and the account can sign in again" ) return False try: diff --git a/tests/test_litellm/proxy/auth/test_password_hashing.py b/tests/test_litellm/proxy/auth/test_password_hashing.py index 97ef252eeb2..a02818546de 100644 --- a/tests/test_litellm/proxy/auth/test_password_hashing.py +++ b/tests/test_litellm/proxy/auth/test_password_hashing.py @@ -72,7 +72,7 @@ class TestVerifyPasswordFormats: with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): assert verify_password("legacy-scrypt-pass", stored) is False assert "scrypt" in caplog.text - assert "/user/password/change" in caplog.text + assert "/user/update" in caplog.text def test_sha256_fallback_verifies_in_both_modes(self, monkeypatch): stored = hashlib.sha256(b"oldpass").hexdigest() From cda3e61414c692772d957703ddaa7867036e071a Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:30:52 +0000 Subject: [PATCH 07/11] test(integration): cover pbkdf2 iteration pinning, /user/new contract and login burst liveness Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../authorization/test_password_hash_login.py | 80 ++++++++++++++++++- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/tests/integration/authorization/test_password_hash_login.py b/tests/integration/authorization/test_password_hash_login.py index dc6c3d8a8fe..5955c201196 100644 --- a/tests/integration/authorization/test_password_hash_login.py +++ b/tests/integration/authorization/test_password_hash_login.py @@ -14,6 +14,7 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Final +import httpx import jwt import psutil import psycopg @@ -95,6 +96,13 @@ def _login(proxy: Gateway, email: str, password: str) -> int: return response.status_code +def _login_retrying_dropped_connections(proxy: Gateway, email: str, password: str) -> int: + try: + return _login(proxy, email, password) + except httpx.TransportError: + return _login(proxy, email, password) + + def _user_with_password(proxy: Gateway, password: str | None) -> tuple[str, str]: email: Final = f"integration-{uuid.uuid4().hex}@example.com" created: Final = proxy.post( @@ -191,6 +199,69 @@ def test_changed_password_is_stored_as_pbkdf2(proxy: Gateway) -> None: _delete_user(proxy, user_id) +def test_pbkdf2_row_with_a_higher_iteration_count_signs_in_and_is_kept(proxy: Gateway) -> None: + user_id, email = _user_with_password(proxy, None) + try: + salt: Final = os.urandom(16) + derived: Final = hashlib.pbkdf2_hmac("sha256", PASSWORD.encode(), salt, 700_000) + row: Final = "pbkdf2:sha256:700000:{}:{}".format( + base64.b64encode(salt).decode(), base64.b64encode(derived).decode() + ) + _write_stored_password(user_id, row) + assert _login(proxy, email, WRONG_PASSWORD) == 401 + assert _login(proxy, email, PASSWORD) == 303 + assert _stored_password(user_id) == row, "a 700k-iteration row must not be downgraded to 600k on login" + finally: + _delete_user(proxy, user_id) + + +def test_user_new_refuses_a_password(proxy: Gateway) -> None: + refused: Final = proxy.request( + "POST", + "/user/new", + { + "user_id": f"integration-{uuid.uuid4().hex}", + "user_email": f"integration-{uuid.uuid4().hex}@example.com", + "auto_create_key": False, + "password": PASSWORD, + }, + ) + assert refused.status_code == 422, refused.text + assert "password cannot be set via /user/new" in refused.text, refused.text + + +def test_unrelated_routes_keep_serving_during_a_login_burst(proxy: Gateway) -> None: + user_id, email = _user_with_password(proxy, None) + try: + _write_stored_password(user_id, _scrypt_row(PASSWORD)) + with proxy.scenario() as scenario: + model: Final = scenario.model() + key: Final = scenario.key(models=[model]) + attempts: Final = tuple(PASSWORD if index % 3 else WRONG_PASSWORD for index in range(BURST_SIZE)) + with ThreadPoolExecutor(max_workers=BURST_SIZE + 2) as pool: + login_futures: Final = tuple( + pool.submit(_login, proxy, email, password) for password in attempts + ) + key_future: Final = pool.submit(proxy.request, "POST", "/key/generate", {}) + chat_future: Final = pool.submit( + proxy.request, + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "login burst liveness"}]}, + key=key, + ) + statuses: Final = tuple(future.result() for future in login_futures) + generated: Final = key_future.result() + chat: Final = chat_future.result() + assert sorted(statuses) == sorted(303 if password == PASSWORD else 401 for password in attempts), statuses + assert generated.status_code == 200, generated.text + assert chat.status_code == 200, chat.text + rehashed: Final = eventually(lambda: _stored_password(user_id), lambda row: row.startswith("pbkdf2:")) + assert _pbkdf2_matches(rehashed, PASSWORD), rehashed + finally: + _delete_user(proxy, user_id) + + def test_concurrent_logins_on_a_scrypt_row_rehash_once_while_one_worker_is_killed( password_proxy: OwnedProxy, ) -> None: @@ -201,11 +272,14 @@ def test_concurrent_logins_on_a_scrypt_row_rehash_once_while_one_worker_is_kille workers: Final = psutil.Process(password_proxy.process.pid).children(recursive=True) assert len(workers) >= 2, workers victim: Final = workers[0] - victim.kill() - eventually(lambda: victim.is_running() and victim.status() != psutil.STATUS_ZOMBIE, lambda alive: not alive) attempts: Final = tuple(PASSWORD if index % 3 else WRONG_PASSWORD for index in range(BURST_SIZE)) with ThreadPoolExecutor(max_workers=BURST_SIZE) as pool: - statuses: Final = tuple(pool.map(lambda password: _login(proxy, email, password), attempts)) + futures: Final = tuple( + pool.submit(_login_retrying_dropped_connections, proxy, email, password) for password in attempts + ) + eventually(lambda: any(future.done() for future in futures), lambda done: done) + victim.kill() + statuses: Final = tuple(future.result() for future in futures) assert sorted(statuses) == sorted(303 if password == PASSWORD else 401 for password in attempts), statuses rehashed: Final = eventually(lambda: _stored_password(user_id), lambda row: row.startswith("pbkdf2:")) assert _pbkdf2_matches(rehashed, PASSWORD), rehashed From 6e7247071c318cd79834641fb25658a81941ef2a Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:31:03 +0000 Subject: [PATCH 08/11] test(integration): format the new password-hash cells Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integration/authorization/test_password_hash_login.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/integration/authorization/test_password_hash_login.py b/tests/integration/authorization/test_password_hash_login.py index 5955c201196..b28cef5fb38 100644 --- a/tests/integration/authorization/test_password_hash_login.py +++ b/tests/integration/authorization/test_password_hash_login.py @@ -204,9 +204,7 @@ def test_pbkdf2_row_with_a_higher_iteration_count_signs_in_and_is_kept(proxy: Ga try: salt: Final = os.urandom(16) derived: Final = hashlib.pbkdf2_hmac("sha256", PASSWORD.encode(), salt, 700_000) - row: Final = "pbkdf2:sha256:700000:{}:{}".format( - base64.b64encode(salt).decode(), base64.b64encode(derived).decode() - ) + row: Final = f"pbkdf2:sha256:700000:{base64.b64encode(salt).decode()}:{base64.b64encode(derived).decode()}" _write_stored_password(user_id, row) assert _login(proxy, email, WRONG_PASSWORD) == 401 assert _login(proxy, email, PASSWORD) == 303 @@ -239,9 +237,7 @@ def test_unrelated_routes_keep_serving_during_a_login_burst(proxy: Gateway) -> N key: Final = scenario.key(models=[model]) attempts: Final = tuple(PASSWORD if index % 3 else WRONG_PASSWORD for index in range(BURST_SIZE)) with ThreadPoolExecutor(max_workers=BURST_SIZE + 2) as pool: - login_futures: Final = tuple( - pool.submit(_login, proxy, email, password) for password in attempts - ) + login_futures: Final = tuple(pool.submit(_login, proxy, email, password) for password in attempts) key_future: Final = pool.submit(proxy.request, "POST", "/key/generate", {}) chat_future: Final = pool.submit( proxy.request, From a5bf805d4d72df87e6858ca470aa2dba9e7de821 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:42:06 +0000 Subject: [PATCH 09/11] ci(codeql): filter the legacy sha256 password-row verifier alert Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 9a85ced57f6..d767c6a8951 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -72,13 +72,19 @@ jobs: # a lookup key into the haveibeenpwned range API (the protocol mandates # SHA-1) and the digest itself never leaves the proxy beyond its first 5 # characters. - - name: Filter SARIF (OCI sha256, HIBP sha1) + # The same query fires on the legacy-row verifier in + # litellm/proxy/utils.py, where a stored 64-hex SHA256 row is matched by + # hashing the submitted password to compare. No new SHA256 rows are + # written (new passwords are PBKDF2-HMAC-SHA256); the call exists only so + # pre-migration rows can still sign in and be rehashed. + - name: Filter SARIF (OCI sha256, HIBP sha1, legacy password row verify) if: matrix.language == 'python' uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1 with: patterns: | -litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing -litellm/proxy/auth/password_policy.py:py/weak-sensitive-data-hashing + -litellm/proxy/utils.py:py/weak-sensitive-data-hashing input: sarif-results/python.sarif output: sarif-results/python.sarif From 92d3fa6c418fddcf018f81dcfa5c78a9441c240d Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 09:48:03 +0000 Subject: [PATCH 10/11] fix(proxy): make login rehash conditional and non fatal, bound pbkdf2 iterations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/login_utils.py | 11 +++++--- litellm/proxy/utils.py | 10 ++++--- .../proxy/auth/test_login_utils.py | 27 +++++++++++++++---- .../proxy/auth/test_password_hashing.py | 9 +++++++ 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index e93bc2a56f2..ef0732221df 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -113,10 +113,13 @@ async def _rehash_password_if_needed(user_id: str, password: str, stored: str) - from litellm.proxy.proxy_server import prisma_client if prisma_client is not None: - await UserRepository(prisma_client).table.update( - where={"user_id": user_id}, - data={"password": hash_password(password)}, - ) + try: + await UserRepository(prisma_client).table.update_many( + where={"user_id": user_id, "password": stored}, + data={"password": hash_password(password)}, + ) + except Exception as e: # noqa: BLE001 # a failed rehash must never surface into the login + verbose_proxy_logger.warning("Login-time password rehash could not update user %s: %s", user_id, e) def get_ui_credentials(master_key: str | None) -> tuple[str, str]: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 42dfbf7e421..4330cf90d6d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7048,6 +7048,7 @@ def hash_token(token: str): PBKDF2_ITERATIONS: Final = 600_000 +PBKDF2_MAX_ITERATIONS: Final = 10_000_000 PBKDF2_PREFIX: Final = "pbkdf2:sha256:" SCRYPT_PREFIX: Final = "scrypt:" @@ -7081,11 +7082,12 @@ def _verify_pbkdf2(password: str, stored: str) -> bool: scheme, digest, iterations, salt, derived = stored.split(":") if (scheme, digest) != ("pbkdf2", "sha256"): return False - expected: Final = hashlib.pbkdf2_hmac( - "sha256", password.encode(), base64.b64decode(salt, validate=True), int(iterations) - ) + count: Final = int(iterations) + if not 1 <= count <= PBKDF2_MAX_ITERATIONS: + return False + expected: Final = hashlib.pbkdf2_hmac("sha256", password.encode(), base64.b64decode(salt, validate=True), count) return secrets.compare_digest(base64.b64decode(derived, validate=True), expected) - except (ValueError, binascii.Error, TypeError): + except (ValueError, binascii.Error, TypeError, OverflowError): return False diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index 5ec37195c58..83e8ee25e95 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -2365,14 +2365,14 @@ class TestRehashPasswordIfNeeded: stored = hashlib.sha256(password.encode()).hexdigest() mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_usertable.update = AsyncMock() + mock_prisma_client.db.litellm_usertable.update_many = AsyncMock() with patch( # test-quality-ok: the rehash writes to the database; faked so no DB is needed "litellm.proxy.proxy_server.prisma_client", mock_prisma_client ): await _rehash_password_if_needed("u-1", password, stored) - update_kwargs = mock_prisma_client.db.litellm_usertable.update.await_args.kwargs - assert update_kwargs["where"] == {"user_id": "u-1"} + update_kwargs = mock_prisma_client.db.litellm_usertable.update_many.await_args.kwargs + assert update_kwargs["where"] == {"user_id": "u-1", "password": stored} written = update_kwargs["data"]["password"] assert written.startswith("pbkdf2:sha256:") assert verify_password(password, written) @@ -2383,10 +2383,27 @@ class TestRehashPasswordIfNeeded: from litellm.proxy.utils import hash_password mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_usertable.update = AsyncMock() + mock_prisma_client.db.litellm_usertable.update_many = AsyncMock() with patch( # test-quality-ok: the rehash writes to the database; faked so no DB is needed "litellm.proxy.proxy_server.prisma_client", mock_prisma_client ): await _rehash_password_if_needed("u-1", "rehash-me-1", hash_password("rehash-me-1")) - mock_prisma_client.db.litellm_usertable.update.assert_not_called() + mock_prisma_client.db.litellm_usertable.update_many.assert_not_called() + + @pytest.mark.asyncio + async def test_a_failed_rehash_logs_a_warning_and_never_raises(self, caplog): + import logging + + from litellm._logging import verbose_proxy_logger + from litellm.proxy.auth.login_utils import _rehash_password_if_needed + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(side_effect=RuntimeError("db down")) + with patch( # test-quality-ok: the rehash writes to the database; faked so no DB is needed + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ): + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + await _rehash_password_if_needed("u-1", "rehash-me-1", "scrypt:stored") + + assert "Login-time password rehash could not update user u-1" in caplog.text diff --git a/tests/test_litellm/proxy/auth/test_password_hashing.py b/tests/test_litellm/proxy/auth/test_password_hashing.py index a02818546de..b2ac73e6e7d 100644 --- a/tests/test_litellm/proxy/auth/test_password_hashing.py +++ b/tests/test_litellm/proxy/auth/test_password_hashing.py @@ -106,6 +106,15 @@ class TestVerifyPasswordFormats: def test_malformed_pbkdf2_rows_return_false(self, stored): assert verify_password("test", stored) is False + @pytest.mark.parametrize( + "iterations", + (10_000_001, 10**30, 0), + ids=("above_max", "overflows_c_long", "zero"), + ) + def test_out_of_range_iteration_counts_return_false(self, iterations): + stored = _pbkdf2_row("test", 600_000).replace(":600000:", f":{iterations}:") + assert verify_password("test", stored) is False + def test_scrypt_invalid_base64_rejected(self): assert verify_password("test", "scrypt:not-valid-base64!!!") is False From be84b90aedac1eafa6f09b399c514e1116447e21 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 5 Oct 2026 08:48:21 +0000 Subject: [PATCH 11/11] test(proxy): assert pbkdf2 row still verifies when login skips the rehash Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/proxy/auth/test_login_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/proxy/auth/test_login_utils.py b/tests/unit/proxy/auth/test_login_utils.py index 7b05e5b0c6b..ffafaedbabf 100644 --- a/tests/unit/proxy/auth/test_login_utils.py +++ b/tests/unit/proxy/auth/test_login_utils.py @@ -2386,15 +2386,17 @@ class TestRehashPasswordIfNeeded: @pytest.mark.asyncio async def test_pbkdf2_row_triggers_no_update(self): from litellm.proxy.auth.login_utils import _rehash_password_if_needed - from litellm.proxy.utils import hash_password + from litellm.proxy.utils import hash_password, verify_password + stored = hash_password("rehash-me-1") mock_prisma_client = MagicMock() mock_prisma_client.db.litellm_usertable.update_many = AsyncMock() with patch( # test-quality-ok: the rehash writes to the database; faked so no DB is needed "litellm.proxy.proxy_server.prisma_client", mock_prisma_client ): - await _rehash_password_if_needed("u-1", "rehash-me-1", hash_password("rehash-me-1")) + await _rehash_password_if_needed("u-1", "rehash-me-1", stored) + assert verify_password("rehash-me-1", stored) mock_prisma_client.db.litellm_usertable.update_many.assert_not_called() @pytest.mark.asyncio