From 67895955f6e9f40e78a96c89a642adc03b5d6a3c Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:20:08 +0000 Subject: [PATCH 01/15] feat(proxy): LITELLM_FIPS_MODE startup gate with provider assertion, TLS verify guard and loud password migration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_utils/fips.py | 151 ++++++++++++++++++ litellm/proxy/proxy_server.py | 31 +++- tests/integration/_support/process.py | 76 +++++++-- .../configuration/test_fips_mode_boot.py | 66 ++++++++ tests/integration/contracts.json | 15 ++ .../proxy/common_utils/test_fips.py | 118 ++++++++++++++ tests/test_litellm/proxy/test_proxy_server.py | 104 ++++++++++++ 7 files changed, 546 insertions(+), 15 deletions(-) create mode 100644 litellm/proxy/common_utils/fips.py create mode 100644 tests/integration/configuration/test_fips_mode_boot.py create mode 100644 tests/test_litellm/proxy/common_utils/test_fips.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py new file mode 100644 index 00000000000..33f35388567 --- /dev/null +++ b/litellm/proxy/common_utils/fips.py @@ -0,0 +1,151 @@ +import hashlib +import os +from collections.abc import Callable +from dataclasses import dataclass +from typing import Final + +from typing_extensions import assert_never + +FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" +SSL_VERIFY_ENV_VAR: Final = "SSL_VERIFY" +SSL_VERIFY_SETTING: Final = "litellm_settings.ssl_verify" +REFUSAL_PREFIX: Final = "LiteLLM proxy refused to start" + +_TRUE_VALUES: Final = frozenset({"true", "1", "yes", "on"}) +_FALSE_VALUES: Final = frozenset({"false", "0", "no", "off", ""}) + + +@dataclass(frozen=True, slots=True) +class FipsModeOff: + pass + + +@dataclass(frozen=True, slots=True) +class FipsModeOn: + pass + + +@dataclass(frozen=True, slots=True) +class MalformedFipsMode: + value: str + + +FipsModeSetting = FipsModeOff | FipsModeOn | MalformedFipsMode + + +@dataclass(frozen=True, slots=True) +class ProviderDoesNotEnforceFips: + pass + + +@dataclass(frozen=True, slots=True) +class TlsVerificationDisabled: + sources: tuple[str, ...] + + +FipsBootRefusal = MalformedFipsMode | ProviderDoesNotEnforceFips | TlsVerificationDisabled +FipsBootVerdict = FipsModeOff | FipsModeOn | FipsBootRefusal + + +class FipsModeError(Exception): + pass + + +def parse_fips_mode(raw: str | None) -> FipsModeSetting: + if raw is None: + return FipsModeOff() + normalized: Final = raw.strip().lower() + if normalized in _TRUE_VALUES: + return FipsModeOn() + if normalized in _FALSE_VALUES: + return FipsModeOff() + return MalformedFipsMode(value=raw) + + +def is_fips_mode(environ: Callable[[str], str | None] = os.environ.get) -> bool: + return isinstance(parse_fips_mode(environ(FIPS_MODE_ENV_VAR)), FipsModeOn) + + +def openssl_enforces_fips() -> bool: + """MD5 is not an approved digest, so an enforcing FIPS provider refuses it even when asked for security use.""" + try: + hashlib.md5(b"", usedforsecurity=True) + except ValueError: + return True + return False + + +def fips_boot_verdict( + *, + raw_fips_mode: str | None, + provider_enforces_fips: Callable[[], bool], + ssl_verify_environment: str | None, + ssl_verify_setting: object, +) -> FipsBootVerdict: + setting: Final = parse_fips_mode(raw_fips_mode) + match setting: + case FipsModeOff() | MalformedFipsMode(): + return setting + case FipsModeOn(): + pass + case _: + assert_never(setting) + disabled: Final = tuple( + source + for source, off in ( + (SSL_VERIFY_ENV_VAR, _is_off(ssl_verify_environment)), + (SSL_VERIFY_SETTING, _is_off(ssl_verify_setting)), + ) + if off + ) + if disabled: + return TlsVerificationDisabled(sources=disabled) + if not provider_enforces_fips(): + return ProviderDoesNotEnforceFips() + return setting + + +def enforce_fips_boot_verdict(verdict: FipsBootVerdict, announce: Callable[[str], object]) -> None: + match verdict: + case FipsModeOff() | FipsModeOn(): + return + case MalformedFipsMode() | ProviderDoesNotEnforceFips() | TlsVerificationDisabled(): + message: Final = render_refusal(verdict) + announce(f"\n{message}\n\n") + raise FipsModeError(message) + case _: + assert_never(verdict) + + +def render_refusal(refusal: FipsBootRefusal) -> str: + match refusal: + case MalformedFipsMode(value=value): + return ( + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR}={value} is not a boolean.\n" + f"Set {FIPS_MODE_ENV_VAR} to true or false, or unset it." + ) + case ProviderDoesNotEnforceFips(): + return ( + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but this Python does not enforce FIPS.\n" + "Its OpenSSL still allows non-approved algorithms (MD5 succeeded), so passwords and keys would be\n" + "protected with algorithms the FIPS 140-3 policy forbids. Run the proxy from a FIPS image whose\n" + f"OpenSSL FIPS provider is enabled, or unset {FIPS_MODE_ENV_VAR} on a non-FIPS runtime." + ) + case TlsVerificationDisabled(sources=sources): + return ( + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by " + f"{' and '.join(sources)}.\nFIPS deployments must verify upstream certificates, so remove the " + "override or point ssl_verify at a CA bundle instead." + ) + case _: + assert_never(refusal) + + +def _is_off(value: object) -> bool: + match value: + case bool(): + return value is False + case str(): + return value.strip().lower() in _FALSE_VALUES - {""} + case _: + return False diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dab7decd4dc..7e2e063ae88 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -404,6 +404,14 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id +from litellm.proxy.common_utils.fips import ( + FIPS_MODE_ENV_VAR, + SSL_VERIFY_ENV_VAR, + enforce_fips_boot_verdict, + fips_boot_verdict, + is_fips_mode, + openssl_enforces_fips, +) from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -1277,6 +1285,16 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: if isinstance(worker_config, dict): await initialize_from_worker_config(worker_config) + enforce_fips_boot_verdict( + fips_boot_verdict( + raw_fips_mode=os.getenv(FIPS_MODE_ENV_VAR), + provider_enforces_fips=openssl_enforces_fips, + ssl_verify_environment=os.getenv(SSL_VERIFY_ENV_VAR), + ssl_verify_setting=litellm.ssl_verify, + ), + announce=announce_on_stderr_at_exit, + ) + enforce_master_key_boot_verdict( await with_stored_secrets_counted( master_key_boot_verdict( @@ -1316,10 +1334,21 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: try: result: Final = await migrate_passwords_to_scrypt_async(prisma_client) verbose_proxy_logger.info("Password migration: %s", result) + except ValueError as e: + verbose_proxy_logger.error( + "Password migration failed, so plaintext passwords stay unhashed in the database: %s. " + "This is what an OpenSSL FIPS provider reports when the hashing algorithm is not approved.", + e, + ) + if is_fips_mode(): + raise except Exception as e: verbose_proxy_logger.warning("Password migration skipped: %s", e) - asyncio.create_task(_run_pw_migration()) + if is_fips_mode(): + await _run_pw_migration() + else: + asyncio.create_task(_run_pw_migration()) async def _run_agent_grant_id_migration() -> None: from litellm.proxy.agent_endpoints.agent_registry import ( diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 5c44beaa570..6dec960097b 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -68,8 +68,15 @@ def owned_proxy( yield owned.gateway +@dataclass(frozen=True, slots=True) +class LaunchedProxy: + port: int + process: subprocess.Popen[bytes] + log: Path + + @contextmanager -def owned_proxy_process( +def launched_proxy( gateway: Gateway, directory: Path, overrides: Mapping[str, str], @@ -77,7 +84,7 @@ def owned_proxy_process( config: Path | None = None, remove_environment: tuple[str, ...] = (), workers: int = 1, -) -> Iterator[OwnedProxy]: +) -> Iterator[LaunchedProxy]: with socket.socket() as reserve: reserve.bind(("127.0.0.1", 0)) port: Final = reserve.getsockname()[1] @@ -116,18 +123,7 @@ def owned_proxy_process( start_new_session=True, ) try: - with httpx.Client(base_url=f"http://127.0.0.1:{port}", timeout=15, trust_env=False) as client: - deadline: Final = time.monotonic() + 70 - while True: - assert process.poll() is None, "Owned proxy exited before readiness" - try: - if client.get("/health/readiness", timeout=2).status_code == 200: - break - except httpx.TransportError: - pass - assert time.monotonic() < deadline, "Owned proxy readiness deadline exceeded" - time.sleep(0.1) - yield OwnedProxy(Gateway(client, gateway.key, gateway.upstream_url), process, log_path) + yield LaunchedProxy(port, process, log_path) finally: root_stopped: Final = stop_root_process(process) residual: Final = group_members(process.pid) @@ -142,3 +138,55 @@ def owned_proxy_process( survivors: Final = group_members(process.pid) assert not survivors, "Owned proxy child survived cleanup" assert root_stopped and not remaining, "Owned proxy required forced cleanup" + + +def refused_boot_log( + gateway: Gateway, + directory: Path, + overrides: Mapping[str, str], + *, + config: Path | None = None, +) -> str: + """Start the proxy and return its log once it exits non-zero instead of becoming ready.""" + with launched_proxy(gateway, directory, overrides, config=config) as launched: + with httpx.Client(base_url=f"http://127.0.0.1:{launched.port}", timeout=15, trust_env=False) as client: + deadline: Final = time.monotonic() + 70 + while launched.process.poll() is None: + try: + ready: Final = client.get("/health/readiness", timeout=2).status_code == 200 + except httpx.TransportError: + ready = False + assert not ready, f"Proxy became ready instead of refusing to boot:\n{launched.log.read_text()}" + assert time.monotonic() < deadline, "Proxy neither exited nor became ready within the deadline" + time.sleep(0.1) + assert launched.process.returncode != 0, ( + f"Proxy exited 0 instead of refusing to boot:\n{launched.log.read_text()}" + ) + return launched.log.read_text() + + +@contextmanager +def owned_proxy_process( + gateway: Gateway, + directory: Path, + overrides: Mapping[str, str], + *, + config: Path | None = None, + remove_environment: tuple[str, ...] = (), + workers: int = 1, +) -> Iterator[OwnedProxy]: + with launched_proxy( + gateway, directory, overrides, config=config, remove_environment=remove_environment, workers=workers + ) as launched: + with httpx.Client(base_url=f"http://127.0.0.1:{launched.port}", timeout=15, trust_env=False) as client: + deadline: Final = time.monotonic() + 70 + while True: + assert launched.process.poll() is None, "Owned proxy exited before readiness" + try: + if client.get("/health/readiness", timeout=2).status_code == 200: + break + except httpx.TransportError: + pass + assert time.monotonic() < deadline, "Owned proxy readiness deadline exceeded" + time.sleep(0.1) + yield OwnedProxy(Gateway(client, gateway.key, gateway.upstream_url), launched.process, launched.log) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py new file mode 100644 index 00000000000..5cf69f21ab3 --- /dev/null +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -0,0 +1,66 @@ +"""LITELLM_FIPS_MODE is a boot gate: the proxy refuses to serve unless the process really enforces FIPS. + +Every leg launches the real proxy binary against the suite's Postgres and asserts on what an operator sees: +exit status and the refusal text in the log. Nothing is patched inside the proxy. +""" + +import hashlib +from pathlib import Path +from typing import Final + +import pytest +import yaml + +from tests.integration._support.client import Gateway +from tests.integration._support.process import owned_proxy, refused_boot_log + +REFUSAL: Final = "LiteLLM proxy refused to start" + + +def _this_python_enforces_fips() -> bool: + try: + hashlib.md5(b"probe", usedforsecurity=True) + except ValueError: + return True + return False + + +@pytest.mark.covers("other.configuration.fips_mode.refuses_boot_when_provider_does_not_enforce_fips") +def test_fips_mode_refuses_to_serve_when_this_python_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None: + if _this_python_enforces_fips(): + pytest.skip("Runner OpenSSL enforces FIPS, so this leg cannot observe the non-enforcing refusal") + log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}) + assert REFUSAL in log, log + assert "LITELLM_FIPS_MODE" in log and "does not enforce FIPS" in log, log + + +@pytest.mark.covers("other.configuration.fips_mode.refuses_boot_with_tls_verification_disabled") +@pytest.mark.parametrize("source", ("environment", "config")) +def test_fips_mode_refuses_to_serve_with_tls_verification_disabled( + gateway: Gateway, tmp_path: Path, source: str +) -> None: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + path: Final = tmp_path / "ssl_verify_off.yaml" + settings: Final = {**config.get("litellm_settings", {}), "ssl_verify": False} + path.write_text(yaml.safe_dump({**config, "litellm_settings": settings})) + log: Final = ( + refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true", "SSL_VERIFY": "false"}) + if source == "environment" + else refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}, config=path) + ) + assert REFUSAL in log, log + assert "TLS certificate verification is disabled" in log, log + assert ("SSL_VERIFY" if source == "environment" else "litellm_settings.ssl_verify") in log, log + + +@pytest.mark.covers("other.configuration.fips_mode.refuses_boot_on_malformed_value") +def test_fips_mode_refuses_to_serve_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: + log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}) + assert REFUSAL in log, log + assert "LITELLM_FIPS_MODE=enforced" in log and "true or false" in log, log + + +@pytest.mark.covers("other.configuration.fips_mode.off_leaves_boot_unchanged") +def test_fips_mode_off_serves_even_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: + with owned_proxy(gateway, tmp_path, {"LITELLM_FIPS_MODE": "false", "SSL_VERIFY": "false"}) as candidate: + assert candidate.client.get("/health/readiness").status_code == 200 diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index a8d9cf1df8b..673f62fd731 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -76,6 +76,21 @@ "tests/integration/configuration/test_effective_settings.py::test_credential_value_update_and_model_reload_reach_provider": [ "mgmt.credential.update.saved_value_reaches_wire" ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_when_this_python_does_not_enforce_fips": [ + "other.configuration.fips_mode.refuses_boot_when_provider_does_not_enforce_fips" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_with_tls_verification_disabled[environment]": [ + "other.configuration.fips_mode.refuses_boot_with_tls_verification_disabled" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_with_tls_verification_disabled[config]": [ + "other.configuration.fips_mode.refuses_boot_with_tls_verification_disabled" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_on_a_value_that_is_not_a_boolean": [ + "other.configuration.fips_mode.refuses_boot_on_malformed_value" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_off_serves_even_with_tls_verification_disabled": [ + "other.configuration.fips_mode.off_leaves_boot_unchanged" + ], "tests/integration/management/test_partial_update_sequences.py::test_denied_key_update_preserves_saved_grants_and_serving": [ "mgmt.key.update.denied_request_preserves_effective_state" ], diff --git a/tests/test_litellm/proxy/common_utils/test_fips.py b/tests/test_litellm/proxy/common_utils/test_fips.py new file mode 100644 index 00000000000..e934cafbdda --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_fips.py @@ -0,0 +1,118 @@ +import hashlib + +import pytest + +from litellm.proxy.common_utils.fips import ( + FipsModeError, + FipsModeOff, + FipsModeOn, + MalformedFipsMode, + ProviderDoesNotEnforceFips, + TlsVerificationDisabled, + enforce_fips_boot_verdict, + fips_boot_verdict, + is_fips_mode, + openssl_enforces_fips, + parse_fips_mode, +) + + +def _verdict( + raw: str | None, + *, + enforcing: bool = True, + ssl_env: str | None = None, + ssl_setting: object = True, +): + return fips_boot_verdict( + raw_fips_mode=raw, + provider_enforces_fips=lambda: enforcing, + ssl_verify_environment=ssl_env, + ssl_verify_setting=ssl_setting, + ) + + +@pytest.mark.parametrize("raw", [None, "false", "0", "no", "off", "", " False "]) +def test_unset_and_false_spellings_leave_fips_mode_off(raw): + assert parse_fips_mode(raw) == FipsModeOff() + assert is_fips_mode({"LITELLM_FIPS_MODE": raw}.get) is False + + +@pytest.mark.parametrize("raw", ["true", "1", "yes", "on", " TRUE "]) +def test_true_spellings_turn_fips_mode_on(raw): + assert parse_fips_mode(raw) == FipsModeOn() + assert is_fips_mode({"LITELLM_FIPS_MODE": raw}.get) is True + + +@pytest.mark.parametrize("raw", ["enforced", "2", "strict", "yes please"]) +def test_anything_else_is_malformed_and_refused_with_the_offending_value(raw): + assert parse_fips_mode(raw) == MalformedFipsMode(value=raw) + with pytest.raises(FipsModeError) as refused: + enforce_fips_boot_verdict(_verdict(raw, enforcing=False), announce=lambda _: None) + assert f"LITELLM_FIPS_MODE={raw} is not a boolean" in str(refused.value) + assert "true or false" in str(refused.value) + + +def test_off_never_consults_the_provider_or_tls_settings(): + def explode() -> bool: + raise AssertionError("provider probe must not run while FIPS mode is off") + + verdict = fips_boot_verdict( + raw_fips_mode=None, provider_enforces_fips=explode, ssl_verify_environment="false", ssl_verify_setting=False + ) + assert verdict == FipsModeOff() + enforce_fips_boot_verdict(verdict, announce=lambda _: pytest.fail("nothing to announce when off")) + + +def test_on_with_an_enforcing_provider_and_verified_tls_boots(): + verdict = _verdict("true", enforcing=True, ssl_env="true", ssl_setting="/etc/ssl/certs/ca.pem") + assert verdict == FipsModeOn() + enforce_fips_boot_verdict(verdict, announce=lambda _: pytest.fail("nothing to announce when on")) + + +def test_on_with_a_non_enforcing_provider_is_refused_and_names_the_fix(): + announced = [] + with pytest.raises(FipsModeError) as refused: + enforce_fips_boot_verdict(_verdict("true", enforcing=False), announce=announced.append) + message = str(refused.value) + assert message.startswith("LiteLLM proxy refused to start") + assert "LITELLM_FIPS_MODE is on but this Python does not enforce FIPS" in message + assert "FIPS image" in message + assert announced == [f"\n{message}\n\n"] + + +@pytest.mark.parametrize( + "ssl_env, ssl_setting, sources", + [ + ("false", True, ("SSL_VERIFY",)), + ("0", True, ("SSL_VERIFY",)), + (None, False, ("litellm_settings.ssl_verify",)), + (None, "False", ("litellm_settings.ssl_verify",)), + ("no", False, ("SSL_VERIFY", "litellm_settings.ssl_verify")), + ], +) +def test_disabled_tls_verification_is_refused_naming_every_source(ssl_env, ssl_setting, sources): + verdict = _verdict("true", enforcing=True, ssl_env=ssl_env, ssl_setting=ssl_setting) + assert verdict == TlsVerificationDisabled(sources=sources) + with pytest.raises(FipsModeError) as refused: + enforce_fips_boot_verdict(verdict, announce=lambda _: None) + assert "TLS certificate verification is disabled by " + " and ".join(sources) in str(refused.value) + + +@pytest.mark.parametrize("ssl_setting", [True, "true", "/etc/ssl/certs/ca.pem", None, ""]) +def test_verified_or_custom_bundle_tls_settings_are_not_treated_as_disabled(ssl_setting): + assert _verdict("true", enforcing=True, ssl_setting=ssl_setting) == FipsModeOn() + + +def test_disabled_tls_is_reported_before_the_provider_so_operators_see_config_mistakes_first(): + assert _verdict("true", enforcing=False, ssl_env="false") == TlsVerificationDisabled(sources=("SSL_VERIFY",)) + assert _verdict("true", enforcing=False) == ProviderDoesNotEnforceFips() + + +def test_provider_probe_agrees_with_whether_md5_is_usable_for_security_here(): + try: + hashlib.md5(b"", usedforsecurity=True) + except ValueError: + assert openssl_enforces_fips() is True + else: + assert openssl_enforces_fips() is False diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 89fd9c5c9d4..42ad00d6b13 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1681,6 +1681,110 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_this_python_does_not_enforce_fips(monkeypatch, tmp_path): + from fastapi import FastAPI + + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + _, announced = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError): + async with proxy_startup_event(FastAPI()): + pass + + assert len(announced) == 1 + assert "does not enforce FIPS" in announced[0] + + +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path): + import yaml + from fastapi import FastAPI + + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + config_path, announced = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + config_path.write_text( + yaml.dump({"general_settings": {"master_key": "sk-a-safe-master-key"}, "litellm_settings": {"ssl_verify": False}}) + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError): + async with proxy_startup_event(FastAPI()): + pass + + assert "TLS certificate verification is disabled by litellm_settings.ssl_verify" in announced[0] + + +class _PrismaClientWhoseUserTableCannotHash: + class _Table: + async def find_many(self, where): + raise ValueError("[digital envelope routines] unsupported") + + class _Db: + litellm_usertable = None + + def __init__(self): + self.db = self._Db() + self.db.litellm_usertable = self._Table() + self.writer_db = self.db + + async def connect(self): + pass + + async def disconnect(self): + pass + + async def check_view_exists(self): + pass + + async def health_check(self): + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fips_mode", ["true", "false"]) +async def test_proxy_startup_surfaces_a_password_migration_crypto_failure(monkeypatch, tmp_path, caplog, fips_mode): + from fastapi import FastAPI + + from litellm.proxy.proxy_server import proxy_startup_event + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable") + cannot_hash = _PrismaClientWhoseUserTableCannotHash() + + async def connected(**kwargs): + return cannot_hash + + monkeypatch.setattr("litellm.proxy.proxy_server.ProxyStartupEvent._setup_prisma_client", connected) + monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True) + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + if fips_mode == "true": + with pytest.raises(ValueError, match="digital envelope routines"): + async with proxy_startup_event(FastAPI()): + pass + else: + async with proxy_startup_event(FastAPI()): + await asyncio.sleep(0) + + failures = [r.getMessage() for r in caplog.records if "Password migration failed" in r.getMessage()] + assert len(failures) == 1 + assert "plaintext passwords stay unhashed" in failures[0] + assert "digital envelope routines" in failures[0] + + class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): self._ciphertext = ciphertext From 5c237840f4055a5a0058d09598ec51a66fae8e27 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:31:01 +0000 Subject: [PATCH 02/15] refactor(proxy): match ssl_verify off detection to runtime str_to_bool semantics Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_utils/fips.py | 4 +++- tests/integration/_support/process.py | 15 ++++++++++----- .../test_litellm/proxy/common_utils/test_fips.py | 6 +++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 33f35388567..8a3c26b3220 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -6,6 +6,8 @@ from typing import Final from typing_extensions import assert_never +from litellm.secret_managers.main import str_to_bool + FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" SSL_VERIFY_ENV_VAR: Final = "SSL_VERIFY" SSL_VERIFY_SETTING: Final = "litellm_settings.ssl_verify" @@ -146,6 +148,6 @@ def _is_off(value: object) -> bool: case bool(): return value is False case str(): - return value.strip().lower() in _FALSE_VALUES - {""} + return str_to_bool(value) is False case _: return False diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 6dec960097b..f65ab0e8093 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -140,6 +140,13 @@ def launched_proxy( assert root_stopped and not remaining, "Owned proxy required forced cleanup" +def _is_ready(client: httpx.Client) -> bool: + try: + return client.get("/health/readiness", timeout=2).status_code == 200 + except httpx.TransportError: + return False + + def refused_boot_log( gateway: Gateway, directory: Path, @@ -152,11 +159,9 @@ def refused_boot_log( with httpx.Client(base_url=f"http://127.0.0.1:{launched.port}", timeout=15, trust_env=False) as client: deadline: Final = time.monotonic() + 70 while launched.process.poll() is None: - try: - ready: Final = client.get("/health/readiness", timeout=2).status_code == 200 - except httpx.TransportError: - ready = False - assert not ready, f"Proxy became ready instead of refusing to boot:\n{launched.log.read_text()}" + assert not _is_ready(client), ( + f"Proxy became ready instead of refusing to boot:\n{launched.log.read_text()}" + ) assert time.monotonic() < deadline, "Proxy neither exited nor became ready within the deadline" time.sleep(0.1) assert launched.process.returncode != 0, ( diff --git a/tests/test_litellm/proxy/common_utils/test_fips.py b/tests/test_litellm/proxy/common_utils/test_fips.py index e934cafbdda..4b50ae4ad20 100644 --- a/tests/test_litellm/proxy/common_utils/test_fips.py +++ b/tests/test_litellm/proxy/common_utils/test_fips.py @@ -85,10 +85,10 @@ def test_on_with_a_non_enforcing_provider_is_refused_and_names_the_fix(): "ssl_env, ssl_setting, sources", [ ("false", True, ("SSL_VERIFY",)), - ("0", True, ("SSL_VERIFY",)), + (" FALSE ", True, ("SSL_VERIFY",)), (None, False, ("litellm_settings.ssl_verify",)), (None, "False", ("litellm_settings.ssl_verify",)), - ("no", False, ("SSL_VERIFY", "litellm_settings.ssl_verify")), + ("false", False, ("SSL_VERIFY", "litellm_settings.ssl_verify")), ], ) def test_disabled_tls_verification_is_refused_naming_every_source(ssl_env, ssl_setting, sources): @@ -99,7 +99,7 @@ def test_disabled_tls_verification_is_refused_naming_every_source(ssl_env, ssl_s assert "TLS certificate verification is disabled by " + " and ".join(sources) in str(refused.value) -@pytest.mark.parametrize("ssl_setting", [True, "true", "/etc/ssl/certs/ca.pem", None, ""]) +@pytest.mark.parametrize("ssl_setting", [True, "true", "/etc/ssl/certs/ca.pem", None, "", "0", "no"]) def test_verified_or_custom_bundle_tls_settings_are_not_treated_as_disabled(ssl_setting): assert _verdict("true", enforcing=True, ssl_setting=ssl_setting) == FipsModeOn() From 6ae5df66c1d24ba5ac460c2338ea01585753b04a Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:45:39 +0000 Subject: [PATCH 03/15] refactor(proxy): drop tautological fips probe test and satisfy CodeQL return checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_utils/fips.py | 23 ++++++++----------- .../proxy/common_utils/test_fips.py | 12 ---------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 8a3c26b3220..07bd4d2462c 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -121,9 +121,9 @@ def enforce_fips_boot_verdict(verdict: FipsBootVerdict, announce: Callable[[str] def render_refusal(refusal: FipsBootRefusal) -> str: match refusal: - case MalformedFipsMode(value=value): + case MalformedFipsMode(): return ( - f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR}={value} is not a boolean.\n" + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR}={refusal.value} is not a boolean.\n" f"Set {FIPS_MODE_ENV_VAR} to true or false, or unset it." ) case ProviderDoesNotEnforceFips(): @@ -133,21 +133,18 @@ def render_refusal(refusal: FipsBootRefusal) -> str: "protected with algorithms the FIPS 140-3 policy forbids. Run the proxy from a FIPS image whose\n" f"OpenSSL FIPS provider is enabled, or unset {FIPS_MODE_ENV_VAR} on a non-FIPS runtime." ) - case TlsVerificationDisabled(sources=sources): + case TlsVerificationDisabled(): return ( f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by " - f"{' and '.join(sources)}.\nFIPS deployments must verify upstream certificates, so remove the " + f"{' and '.join(refusal.sources)}.\nFIPS deployments must verify upstream certificates, so remove the " "override or point ssl_verify at a CA bundle instead." ) - case _: - assert_never(refusal) + return assert_never(refusal) def _is_off(value: object) -> bool: - match value: - case bool(): - return value is False - case str(): - return str_to_bool(value) is False - case _: - return False + if isinstance(value, bool): + return value is False + if isinstance(value, str): + return str_to_bool(value) is False + return False diff --git a/tests/test_litellm/proxy/common_utils/test_fips.py b/tests/test_litellm/proxy/common_utils/test_fips.py index 4b50ae4ad20..0e8138856c0 100644 --- a/tests/test_litellm/proxy/common_utils/test_fips.py +++ b/tests/test_litellm/proxy/common_utils/test_fips.py @@ -1,5 +1,3 @@ -import hashlib - import pytest from litellm.proxy.common_utils.fips import ( @@ -12,7 +10,6 @@ from litellm.proxy.common_utils.fips import ( enforce_fips_boot_verdict, fips_boot_verdict, is_fips_mode, - openssl_enforces_fips, parse_fips_mode, ) @@ -107,12 +104,3 @@ def test_verified_or_custom_bundle_tls_settings_are_not_treated_as_disabled(ssl_ def test_disabled_tls_is_reported_before_the_provider_so_operators_see_config_mistakes_first(): assert _verdict("true", enforcing=False, ssl_env="false") == TlsVerificationDisabled(sources=("SSL_VERIFY",)) assert _verdict("true", enforcing=False) == ProviderDoesNotEnforceFips() - - -def test_provider_probe_agrees_with_whether_md5_is_usable_for_security_here(): - try: - hashlib.md5(b"", usedforsecurity=True) - except ValueError: - assert openssl_enforces_fips() is True - else: - assert openssl_enforces_fips() is False From e6a5d32cd9925caedbd1eb067b86772031dcf556 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 06:07:49 +0000 Subject: [PATCH 04/15] test(proxy): inject the fake Prisma client through the module boundary instead of patching _setup_prisma_client Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/proxy/test_proxy_server.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 42ad00d6b13..39e4a417d19 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1733,7 +1733,7 @@ class _PrismaClientWhoseUserTableCannotHash: class _Db: litellm_usertable = None - def __init__(self): + def __init__(self, database_url, proxy_logging_obj): self.db = self._Db() self.db.litellm_usertable = self._Table() self.writer_db = self.db @@ -1744,6 +1744,9 @@ class _PrismaClientWhoseUserTableCannotHash: async def disconnect(self): pass + def start_view_setup_task(self): + pass + async def check_view_exists(self): pass @@ -1761,12 +1764,7 @@ async def test_proxy_startup_surfaces_a_password_migration_crypto_failure(monkey _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable") - cannot_hash = _PrismaClientWhoseUserTableCannotHash() - - async def connected(**kwargs): - return cannot_hash - - monkeypatch.setattr("litellm.proxy.proxy_server.ProxyStartupEvent._setup_prisma_client", connected) + monkeypatch.setattr("litellm.proxy.proxy_server.PrismaClient", _PrismaClientWhoseUserTableCannotHash) monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True) monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) From 29945f1b66ee9ae8ba393c2d7f2ee20d3b625a17 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:06:13 +0000 Subject: [PATCH 05/15] test(integration): cover outbound TLS cipher list at the handler and proxy edges Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/_support/tls.py | 194 +++++++++++ .../configuration/test_tls_cipher_proxy.py | 188 +++++++++++ tests/integration/sdk/test_tls_cipher_wire.py | 314 ++++++++++++++++++ 3 files changed, 696 insertions(+) create mode 100644 tests/integration/_support/tls.py create mode 100644 tests/integration/configuration/test_tls_cipher_proxy.py create mode 100644 tests/integration/sdk/test_tls_cipher_wire.py diff --git a/tests/integration/_support/tls.py b/tests/integration/_support/tls.py new file mode 100644 index 00000000000..1abc8273ffe --- /dev/null +++ b/tests/integration/_support/tls.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import datetime +import ipaddress +import json +import ssl +import threading +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final, cast + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID +from pydantic import JsonValue, TypeAdapter + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def write_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]: + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + now: Final = datetime.datetime.now(datetime.timezone.utc) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) + cert: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=7)) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .sign(key, hashes.SHA256()) + ) + cert_file: Final = cert_dir / "cert.pem" + key_file: Final = cert_dir / "key.pem" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) + return cert_file, key_file + + +@dataclass(frozen=True, slots=True) +class TlsPeerRequest: + path: str + user: str | None + cipher: str + + +def _completion_body(identifier: str, model: str) -> dict[str, JsonValue]: + return { + "id": identifier, + "object": "chat.completion", + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "tls ok"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + + +def _chunk_body( + identifier: str, model: str, content: str, role: str | None, finish_reason: str | None +) -> dict[str, JsonValue]: + delta: dict[str, JsonValue] = {"content": content} + if role is not None: + delta["role"] = role + return { + "id": identifier, + "object": "chat.completion.chunk", + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + + +class _CipherPeer(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + + def __init__(self, address: tuple[str, int], owner: TlsPeer) -> None: + super().__init__(address, _CipherPeerHandler) + self.owner = owner + + +def _negotiated_cipher(handler: BaseHTTPRequestHandler) -> str: + info: Final = cast(ssl.SSLSocket, handler.connection).cipher() + return info[0] if info else "" + + +class _CipherPeerHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + peer: Final = cast(_CipherPeer, self.server).owner + body: Final = JSON_OBJECT.validate_json(self.rfile.read(int(self.headers["Content-Length"]))) + user_value: Final = body.get("user") + user: Final = user_value if isinstance(user_value, str) else None + peer.record(TlsPeerRequest(path=self.path, user=user, cipher=_negotiated_cipher(self))) + identifier: Final = f"chatcmpl-{user or ''}" + model_value: Final = body.get("model") + model: Final = model_value if isinstance(model_value, str) else "gpt-4o-mini" + if body.get("stream") is True: + chunks: Final = ( + f"data: {json.dumps(_chunk_body(identifier, model, 'tls', 'assistant', None))}\n\n" + + f"data: {json.dumps(_chunk_body(identifier, model, ' ok', None, 'stop'))}\n\n" + + "data: [DONE]\n\n" + ).encode() + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Content-Length", str(len(chunks))) + self.end_headers() + self.wfile.write(chunks) + return + payload: Final = json.dumps(_completion_body(identifier, model)).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, format: str, *args: object) -> None: + return + + +class TlsPeer: + def __init__(self, cipher: str, cert_file: Path, key_file: Path) -> None: + self.cipher = cipher + self.cert_file = cert_file + self.key_file = key_file + self.port = 0 + self._server: _CipherPeer | None = None + self._thread: threading.Thread | None = None + self._lock = threading.Lock() + self._requests: list[TlsPeerRequest] = [] + + @property + def url(self) -> str: + return f"https://127.0.0.1:{self.port}" + + def record(self, request: TlsPeerRequest) -> None: + with self._lock: + self._requests.append(request) + + def received(self) -> tuple[TlsPeerRequest, ...]: + with self._lock: + return tuple(self._requests) + + def start(self) -> None: + server: Final = _CipherPeer(("127.0.0.1", self.port), self) + context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(str(self.cert_file), str(self.key_file)) + context.maximum_version = ssl.TLSVersion.TLSv1_2 + context.set_ciphers(self.cipher) + server.socket = context.wrap_socket(server.socket, server_side=True) + self.port = server.server_address[1] + self._server = server + self._thread = threading.Thread(target=server.serve_forever, daemon=True) + self._thread.start() + + def stop(self) -> None: + if self._server is None: + return + self._server.shutdown() + self._server.server_close() + if self._thread is not None: + self._thread.join(timeout=10) + self._server = None + self._thread = None + + +@contextmanager +def tls_peer(cipher: str, cert_file: Path, key_file: Path) -> Generator[TlsPeer, None, None]: + peer: Final = TlsPeer(cipher, cert_file, key_file) + peer.start() + try: + yield peer + finally: + peer.stop() diff --git a/tests/integration/configuration/test_tls_cipher_proxy.py b/tests/integration/configuration/test_tls_cipher_proxy.py new file mode 100644 index 00000000000..e118d858a86 --- /dev/null +++ b/tests/integration/configuration/test_tls_cipher_proxy.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import asyncio +import uuid +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +import yaml +from openai import APIStatusError, AsyncOpenAI, OpenAI +from pydantic import JsonValue, TypeAdapter + +from tests.integration._support.client import Gateway, eventually, gateway_from_environment +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy +from tests.integration._support.tls import TlsPeer, tls_peer, write_self_signed_cert + +CHACHA20: Final = "ECDHE-RSA-CHACHA20-POLY1305" +NONPFS_GCM: Final = "AES256-GCM-SHA384" +PFS_GCM: Final = "ECDHE-RSA-AES256-GCM-SHA384" +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +@dataclass(frozen=True, slots=True) +class Peers: + gcm: TlsPeer + chacha: TlsPeer + nonpfs: TlsPeer + + +@pytest.fixture(scope="module") +def tls_cert(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]: + return write_self_signed_cert(tmp_path_factory.mktemp("tlscerts")) + + +@pytest.fixture(scope="module") +def tls_peers(tls_cert: tuple[Path, Path]) -> Iterator[Peers]: + cert_file, key_file = tls_cert + with ( + tls_peer(PFS_GCM, cert_file, key_file) as gcm, + tls_peer(CHACHA20, cert_file, key_file) as chacha, + tls_peer(NONPFS_GCM, cert_file, key_file) as nonpfs, + ): + yield Peers(gcm=gcm, chacha=chacha, nonpfs=nonpfs) + + +@pytest.fixture(scope="module") +def cipher_proxy( + tls_cert: tuple[Path, Path], tls_peers: Peers, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Gateway]: + cert_file, _ = tls_cert + directory: Final = tmp_path_factory.mktemp("tlsproxy") + base_config: Final = JSON_OBJECT.validate_python( + yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + ) + config: Final = { + **base_config, + "model_list": [ + { + "model_name": "tls-peer", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{tls_peers.gcm.url}/v1", + "api_key": "sk-peer", + }, + }, + { + "model_name": "tls-chacha-peer", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{tls_peers.chacha.url}/v1", + "api_key": "sk-peer", + }, + }, + { + "model_name": "tls-nonpfs-peer", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{tls_peers.nonpfs.url}/v1", + "api_key": "sk-peer", + }, + }, + ], + } + config_path: Final = directory / "tls_cipher_config.yaml" + config_path.write_text(yaml.safe_dump(config)) + with gateway_from_environment() as gateway: + with owned_proxy(gateway, directory, {"SSL_CERT_FILE": str(cert_file)}, config=config_path) as candidate: + yield candidate + + +def _sync_completion(proxy: Gateway, model: str, user: str, *, stream: bool = False) -> str: + with OpenAI(api_key=proxy.key, base_url=f"{proxy.client.base_url}/v1", max_retries=0, timeout=60) as client: + if stream: + chunks: Final = list( + client.chat.completions.create( + model=model, messages=[{"role": "user", "content": "tls"}], user=user, stream=True + ) + ) + assert chunks, "stream produced no chunks" + assert all(chunk.id == f"chatcmpl-{user}" for chunk in chunks), chunks[0].id + return str(chunks[0].id) + response: Final = client.chat.completions.create( + model=model, messages=[{"role": "user", "content": "tls"}], user=user + ) + return str(response.id) + + +async def _async_completion(proxy: Gateway, model: str, user: str, *, stream: bool = False) -> str: + async with AsyncOpenAI( + api_key=proxy.key, base_url=f"{proxy.client.base_url}/v1", max_retries=0, timeout=60 + ) as client: + if stream: + chunk_ids: Final = [ + chunk.id + async for chunk in await client.chat.completions.create( + model=model, messages=[{"role": "user", "content": "tls"}], user=user, stream=True + ) + ] + assert chunk_ids, "stream produced no chunks" + assert all(chunk_id == f"chatcmpl-{user}" for chunk_id in chunk_ids), chunk_ids[0] + return str(chunk_ids[0]) + response: Final = await client.chat.completions.create( + model=model, messages=[{"role": "user", "content": "tls"}], user=user + ) + return str(response.id) + + +def test_proxy_round_trips_over_pfs_gcm_peer_and_writes_spend(cipher_proxy: Gateway, tls_peers: Peers) -> None: + marker: Final = uuid.uuid4().hex[:8] + sync_id: Final = _sync_completion(cipher_proxy, "tls-peer", f"user13-sync-{marker}") + stream_id: Final = _sync_completion(cipher_proxy, "tls-peer", f"user13-stream-{marker}", stream=True) + async_id: Final = asyncio.run(_async_completion(cipher_proxy, "tls-peer", f"user13-async-{marker}")) + async_stream_id: Final = asyncio.run( + _async_completion(cipher_proxy, "tls-peer", f"user13-astream-{marker}", stream=True) + ) + users: Final = {f"user13-{leg}-{marker}" for leg in ("sync", "stream", "async", "astream")} + recorded: Final = tls_peers.gcm.received() + assert {record.user for record in recorded} == users, recorded + assert all(record.cipher == PFS_GCM for record in recorded), recorded + spend_rows: Final = tuple( + eventually( + lambda response_id=response_id: read_rows( + 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (response_id,) + ), + lambda values: len(values) == 1, + seconds=70, + ) + for response_id in (sync_id, stream_id, async_id, async_stream_id) + ) + assert [rows[0]["request_id"] for rows in spend_rows] == [sync_id, stream_id, async_id, async_stream_id] + + +def test_proxy_reports_ssl_error_for_chacha20_only_peer(cipher_proxy: Gateway, tls_peers: Peers) -> None: + marker: Final = uuid.uuid4().hex[:8] + before: Final = len(tls_peers.chacha.received()) + with pytest.raises(APIStatusError) as raised: + _sync_completion(cipher_proxy, "tls-chacha-peer", f"user14-{marker}") + assert raised.value.status_code >= 400 + assert "ssl" in str(raised.value).lower() or "connection error" in str(raised.value).lower(), str(raised.value) + assert len(tls_peers.chacha.received()) == before + readiness: Final = cipher_proxy.client.get("/health/readiness") + assert readiness.status_code == 200, readiness.text + recovered: Final = _sync_completion(cipher_proxy, "tls-peer", f"user14-ok-{marker}") + assert recovered == f"chatcmpl-user14-ok-{marker}" + + +def test_proxy_reports_ssl_error_for_non_pfs_peer(cipher_proxy: Gateway, tls_peers: Peers) -> None: + marker: Final = uuid.uuid4().hex[:8] + before: Final = len(tls_peers.nonpfs.received()) + response: Final = cipher_proxy.request( + "POST", + "/v1/chat/completions", + { + "model": "tls-nonpfs-peer", + "messages": [{"role": "user", "content": "tls"}], + "user": f"user15-{marker}", + }, + ) + assert response.status_code >= 400, response.text + assert "ssl" in response.text.lower() or "connection error" in response.text.lower(), response.text + assert len(tls_peers.nonpfs.received()) == before + readiness: Final = cipher_proxy.client.get("/health/readiness") + assert readiness.status_code == 200, readiness.text + recovered: Final = _sync_completion(cipher_proxy, "tls-peer", f"user15-ok-{marker}") + assert recovered == f"chatcmpl-user15-ok-{marker}" diff --git a/tests/integration/sdk/test_tls_cipher_wire.py b/tests/integration/sdk/test_tls_cipher_wire.py new file mode 100644 index 00000000000..62df651a293 --- /dev/null +++ b/tests/integration/sdk/test_tls_cipher_wire.py @@ -0,0 +1,314 @@ +from __future__ import annotations + +import asyncio +import os +import socket +import subprocess +import sys +import textwrap +import threading +import time +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Final + +import httpx +import pytest +from pydantic import JsonValue, TypeAdapter + +if TYPE_CHECKING: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + +from tests.integration._support.client import eventually +from tests.integration._support.tls import TlsPeer, tls_peer, write_self_signed_cert + +CHACHA20: Final = "ECDHE-RSA-CHACHA20-POLY1305" +AES256_GCM: Final = "AES256-GCM-SHA384" +AES128_GCM: Final = "AES128-GCM-SHA256" +CBC_SHA256: Final = "ECDHE-RSA-AES128-SHA256" +CBC_SHA384: Final = "ECDHE-RSA-AES256-SHA384" +PFS_GCM: Final = "ECDHE-RSA-AES256-GCM-SHA384" +BURST_SIZE: Final = 30 +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + +OVERRIDE_EXCHANGE: Final = textwrap.dedent( + """ + import sys + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + handler = HTTPHandler(ssl_verify=sys.argv[2]) + try: + response = handler.post(f"{sys.argv[1]}/v1/chat/completions", json={"user": "u-override"}) + print(response.json()["id"]) + finally: + handler.close() + """ +) + + +@pytest.fixture(scope="module") +def tls_cert(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]: + return write_self_signed_cert(tmp_path_factory.mktemp("tlscerts")) + + +@dataclass(frozen=True, slots=True) +class BurstOutcome: + user: str + when: float + response_id: str | None + error: Exception | None + + +def _sync_post(cert_file: Path, url: str, user: str) -> str: + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + handler: Final = HTTPHandler(ssl_verify=str(cert_file)) + try: + response: Final = handler.client.post(f"{url}/v1/chat/completions", json={"user": user}) + return str(JSON_OBJECT.validate_json(response.content)["id"]) + finally: + handler.close() + + +def _async_post(cert_file: Path, url: str, user: str) -> str: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + async def _run() -> str: + handler: Final = AsyncHTTPHandler(ssl_verify=str(cert_file)) + try: + response: Final = await handler.client.post(f"{url}/v1/chat/completions", json={"user": user}) + return str(JSON_OBJECT.validate_json(response.content)["id"]) + finally: + await handler.close() + + return asyncio.run(_run()) + + +def _assert_served(peer: TlsPeer, user: str, response_id: str) -> None: + records: Final = peer.received() + assert response_id == f"chatcmpl-{user}" + assert [(record.user, record.cipher) for record in records] == [(user, peer.cipher)], records + + +def test_sync_handler_refuses_chacha20_only_peer(tls_cert: tuple[Path, Path]) -> None: + cert_file, key_file = tls_cert + with tls_peer(CHACHA20, cert_file, key_file) as peer: + with pytest.raises(httpx.ConnectError, match="SSL"): + _sync_post(cert_file, peer.url, "u-cell1") + assert peer.received() == (), peer.received() + + +def test_async_httpx_handler_refuses_chacha20_only_peer( + monkeypatch: pytest.MonkeyPatch, tls_cert: tuple[Path, Path] +) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + cert_file, key_file = tls_cert + with tls_peer(CHACHA20, cert_file, key_file) as peer: + with pytest.raises(httpx.ConnectError, match="SSL"): + _async_post(cert_file, peer.url, "u-cell2") + assert peer.received() == (), peer.received() + + +def test_async_aiohttp_handler_refuses_chacha20_only_peer(tls_cert: tuple[Path, Path]) -> None: + cert_file, key_file = tls_cert + with tls_peer(CHACHA20, cert_file, key_file) as peer: + with pytest.raises(httpx.ProtocolError, match="SSL"): + _async_post(cert_file, peer.url, "u-cell3") + assert peer.received() == (), peer.received() + + +def test_sync_handler_refuses_non_pfs_aes256_gcm_peer(tls_cert: tuple[Path, Path]) -> None: + cert_file, key_file = tls_cert + with tls_peer(AES256_GCM, cert_file, key_file) as peer: + with pytest.raises(httpx.ConnectError, match="SSL"): + _sync_post(cert_file, peer.url, "u-cell4") + assert peer.received() == (), peer.received() + + +def test_async_httpx_handler_refuses_non_pfs_aes128_gcm_peer( + monkeypatch: pytest.MonkeyPatch, tls_cert: tuple[Path, Path] +) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + cert_file, key_file = tls_cert + with tls_peer(AES128_GCM, cert_file, key_file) as peer: + with pytest.raises(httpx.ConnectError, match="SSL"): + _async_post(cert_file, peer.url, "u-cell5") + assert peer.received() == (), peer.received() + + +def test_sync_handler_accepts_cbc_peer_by_default(tls_cert: tuple[Path, Path]) -> None: + cert_file, key_file = tls_cert + with tls_peer(CBC_SHA256, cert_file, key_file) as peer: + _assert_served(peer, "u-cell6", _sync_post(cert_file, peer.url, "u-cell6")) + + +def test_sync_handler_refuses_cbc_peer_in_fips_mode( + monkeypatch: pytest.MonkeyPatch, tls_cert: tuple[Path, Path] +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + cert_file, key_file = tls_cert + with tls_peer(CBC_SHA256, cert_file, key_file) as peer: + with pytest.raises(httpx.ConnectError, match="SSL"): + _sync_post(cert_file, peer.url, "u-cell7") + assert peer.received() == (), peer.received() + + +def test_async_httpx_handler_refuses_cbc_sha384_peer_in_fips_mode( + monkeypatch: pytest.MonkeyPatch, tls_cert: tuple[Path, Path] +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + cert_file, key_file = tls_cert + with tls_peer(CBC_SHA384, cert_file, key_file) as peer: + with pytest.raises(httpx.ConnectError, match="SSL"): + _async_post(cert_file, peer.url, "u-cell8") + assert peer.received() == (), peer.received() + + +def test_handlers_accept_pfs_gcm_peer_in_default_and_fips_modes( + monkeypatch: pytest.MonkeyPatch, tls_cert: tuple[Path, Path] +) -> None: + cert_file, key_file = tls_cert + with tls_peer(PFS_GCM, cert_file, key_file) as peer: + served: Final = _sync_post(cert_file, peer.url, "u-cell9-sync") + assert served == "chatcmpl-u-cell9-sync" + served_async: Final = _async_post(cert_file, peer.url, "u-cell9-async") + assert served_async == "chatcmpl-u-cell9-async" + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + served_fips: Final = _sync_post(cert_file, peer.url, "u-cell9-fips") + assert served_fips == "chatcmpl-u-cell9-fips" + served_async_fips: Final = _async_post(cert_file, peer.url, "u-cell9-async-fips") + assert served_async_fips == "chatcmpl-u-cell9-async-fips" + assert {record.cipher for record in peer.received()} == {PFS_GCM} + + +def _override_exchange(url: str, cert_file: Path, overrides: Mapping[str, str]) -> str: + completed: Final = subprocess.run( + [sys.executable, "-P", "-c", OVERRIDE_EXCHANGE, url, str(cert_file)], + env={**os.environ, **overrides}, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert completed.returncode == 0, completed.stderr + return completed.stdout.strip() + + +def test_ssl_ciphers_env_override_keeps_chacha20(tls_cert: tuple[Path, Path]) -> None: + cert_file, key_file = tls_cert + with tls_peer(CHACHA20, cert_file, key_file) as peer: + assert _override_exchange(peer.url, cert_file, {"LITELLM_SSL_CIPHERS": CHACHA20}) == "chatcmpl-u-override" + assert [record.cipher for record in peer.received()] == [CHACHA20] + + +def test_ssl_ciphers_override_wins_over_fips_mode(tls_cert: tuple[Path, Path]) -> None: + cert_file, key_file = tls_cert + with tls_peer(CHACHA20, cert_file, key_file) as peer: + result: Final = _override_exchange( + peer.url, cert_file, {"LITELLM_SSL_CIPHERS": CHACHA20, "LITELLM_FIPS_MODE": "true"} + ) + assert result == "chatcmpl-u-override" + assert [record.cipher for record in peer.received()] == [CHACHA20] + + +def test_cipher_list_follows_fips_mode_toggle_in_one_process( + monkeypatch: pytest.MonkeyPatch, tls_cert: tuple[Path, Path] +) -> None: + cert_file, key_file = tls_cert + with tls_peer(CBC_SHA256, cert_file, key_file) as peer: + assert _sync_post(cert_file, peer.url, "u-cell12-before") == "chatcmpl-u-cell12-before" + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + with pytest.raises(httpx.ConnectError, match="SSL"): + _sync_post(cert_file, peer.url, "u-cell12-fips") + monkeypatch.delenv("LITELLM_FIPS_MODE") + assert _sync_post(cert_file, peer.url, "u-cell12-after") == "chatcmpl-u-cell12-after" + assert [record.user for record in peer.received()] == ["u-cell12-before", "u-cell12-after"] + + +def _assert_burst_outcomes(outcomes: Mapping[int, BurstOutcome]) -> None: + for index, outcome in outcomes.items(): + if outcome.error is None: + assert outcome.response_id == f"chatcmpl-user-{index}", outcome + continue + message: str = str(outcome.error).lower() + assert "handshake" not in message and "cipher" not in message, ( + f"user-{index} failed with a cipher class error: {outcome.error!r}" + ) + assert isinstance(outcome.error, httpx.HTTPError), f"user-{index}: {outcome.error!r}" + + +def _peer_ready(port: int) -> bool: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.5): + return True + except OSError: + return False + + +def test_burst_survives_peer_restart_without_cipher_errors(tls_cert: tuple[Path, Path]) -> None: + cert_file, key_file = tls_cert + with tls_peer(PFS_GCM, cert_file, key_file) as peer: + second_wave: Final = threading.Event() + outcomes: Final[dict[int, BurstOutcome]] = {} + + def _one(index: int) -> None: + if index >= 10: + second_wave.wait(timeout=30) + try: + response_id: Final = _sync_post(cert_file, peer.url, f"user-{index}") + outcomes[index] = BurstOutcome(f"user-{index}", time.monotonic(), response_id, None) + except Exception as error: + outcomes[index] = BurstOutcome(f"user-{index}", time.monotonic(), None, error) + + async def _async_burst() -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + handler: Final = AsyncHTTPHandler(ssl_verify=str(cert_file)) + try: + await asyncio.to_thread(second_wave.wait, 30) + await asyncio.gather(*(_async_one(handler, index) for index in range(15, BURST_SIZE))) + finally: + await handler.close() + + async def _async_one(handler: AsyncHTTPHandler, index: int) -> None: + try: + response: Final = await handler.client.post( + f"{peer.url}/v1/chat/completions", json={"user": f"user-{index}"} + ) + outcomes[index] = BurstOutcome( + f"user-{index}", time.monotonic(), str(JSON_OBJECT.validate_json(response.content)["id"]), None + ) + except Exception as error: + outcomes[index] = BurstOutcome(f"user-{index}", time.monotonic(), None, error) + + def _async_thread() -> None: + asyncio.run(_async_burst()) + + async_thread: Final = threading.Thread(target=_async_thread, daemon=True) + with ThreadPoolExecutor(max_workers=15) as pool: + async_thread.start() + futures: Final = tuple(pool.submit(_one, index) for index in range(15)) + eventually(lambda: len(peer.received()) >= 3, lambda ready: ready, seconds=15) + stopped_at: Final = time.monotonic() + peer.stop() + peer.start() + eventually(lambda: _peer_ready(peer.port), lambda ready: ready, seconds=15) + restarted_at: Final = time.monotonic() + second_wave.set() + for future in futures: + future.result(timeout=60) + async_thread.join(timeout=60) + + assert len(outcomes) == BURST_SIZE + _assert_burst_outcomes(outcomes) + succeeded: Final = {o.user for o in outcomes.values() if o.error is None} + recorded: Final = peer.received() + assert sorted(record.user or "" for record in recorded) == sorted(succeeded) + assert all(record.cipher == PFS_GCM for record in recorded) + for user in succeeded: + assert sum(1 for record in recorded if record.user == user) == 1, user + assert any(o.error is None and o.when < stopped_at for o in outcomes.values()), "no success before restart" + assert any(o.error is None and o.when > restarted_at for o in outcomes.values()), "no success after restart" From 769172960871c823fb8065e28460b32cb81305ec Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:06:13 +0000 Subject: [PATCH 06/15] feat(tls): drop ChaCha20 and non-PFS suites from the default cipher list and add a FIPS list Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 30 +++------ litellm/llms/custom_httpx/http_handler.py | 22 ++++--- .../llms/custom_httpx/test_http_handler.py | 66 ++++++++++++++++--- tests/test_litellm/test_constants.py | 50 ++++++++++---- 4 files changed, 119 insertions(+), 49 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 3ff80d8b7dd..4dc315a0041 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -339,28 +339,18 @@ MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolve REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged" REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged" -# SSL/TLS cipher configuration for faster handshakes -# Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones -# This balances performance with broad compatibility +FIPS_SSL_CIPHERS: Final = os.getenv( + "LITELLM_SSL_CIPHERS", + "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:" + "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:" + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256", +) DEFAULT_SSL_CIPHERS: Final = os.getenv( "LITELLM_SSL_CIPHERS", - # Priority 1: TLS 1.3 ciphers (fastest, ~50ms handshake) - "TLS_AES_256_GCM_SHA384:" # Fastest observed in testing - "TLS_AES_128_GCM_SHA256:" # Slightly faster than 256-bit - "TLS_CHACHA20_POLY1305_SHA256:" # Fast on ARM/mobile - # Priority 2: TLS 1.2 ECDHE+GCM (fast, ~100ms handshake, widely supported) - "ECDHE-RSA-AES256-GCM-SHA384:" - "ECDHE-RSA-AES128-GCM-SHA256:" - "ECDHE-ECDSA-AES256-GCM-SHA384:" - "ECDHE-ECDSA-AES128-GCM-SHA256:" - # Priority 3: Additional modern ciphers (good balance) - "ECDHE-RSA-CHACHA20-POLY1305:" - "ECDHE-ECDSA-CHACHA20-POLY1305:" - # Priority 4: Widely compatible fallbacks (slower but universally supported) - "ECDHE-RSA-AES256-SHA384:" # Common fallback - "ECDHE-RSA-AES128-SHA256:" # Very widely supported - "AES256-GCM-SHA384:" # Non-PFS fallback (compatibility) - "AES128-GCM-SHA256", # Last resort (maximum compatibility) + "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:" + "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:" + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:" + "ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256", ) ########### v2 Architecture constants for managing writing updates to the database ########### diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 312f46bb021..60cb10d382e 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -36,6 +36,7 @@ from litellm.constants import ( AIOHTTP_TTL_DNS_CACHE, COMPLETION_HTTP_FALLBACK_SECONDS, DEFAULT_SSL_CIPHERS, + FIPS_SSL_CIPHERS, HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, ) from litellm.litellm_core_utils.logging_utils import track_llm_api_timing @@ -274,15 +275,22 @@ def _prepare_request_data_and_content( # Cache for SSL contexts to avoid creating duplicate contexts with the same configuration -# Key: tuple of (cafile, ssl_security_level, ssl_ecdh_curve) +# Key: tuple of (cafile, ssl_security_level, ssl_ecdh_curve, ssl_ciphers) # Value: ssl.SSLContext -_ssl_context_cache: Final[dict[tuple[str | None, str | None, str | None], ssl.SSLContext]] = {} +_ssl_context_cache: Final[dict[tuple[str | None, str | None, str | None, str], ssl.SSLContext]] = {} + + +def _outbound_ssl_ciphers() -> str: + from litellm.proxy.common_utils.fips import is_fips_mode + + return FIPS_SSL_CIPHERS if is_fips_mode() else DEFAULT_SSL_CIPHERS def _create_ssl_context( cafile: str | None, ssl_security_level: str | None, ssl_ecdh_curve: str | None, + ssl_ciphers: str, ) -> ssl.SSLContext: """ Create an SSL context with the given configuration. @@ -299,9 +307,7 @@ def _create_ssl_context( # User provided custom cipher configuration (e.g., via SSL_SECURITY_LEVEL env var) custom_ssl_context.set_ciphers(ssl_security_level) else: - # Use optimized cipher list that strongly prefers fast ciphers - # but falls back to widely compatible ones - custom_ssl_context.set_ciphers(DEFAULT_SSL_CIPHERS) + custom_ssl_context.set_ciphers(ssl_ciphers) # Configure ECDH curve for key exchange (e.g., to disable PQC and improve performance) # Set SSL_ECDH_CURVE env var or litellm.ssl_ecdh_curve to 'X25519' to disable PQC @@ -416,15 +422,15 @@ def get_ssl_configuration( cafile = certifi.where() if ssl_verify is not False: - # Create cache key from configuration parameters - cache_key: Final = (cafile, ssl_security_level, ssl_ecdh_curve) + ssl_ciphers: Final = _outbound_ssl_ciphers() + cache_key: Final = (cafile, ssl_security_level, ssl_ecdh_curve, ssl_ciphers) - # Check if we have a cached SSL context for this configuration if cache_key not in _ssl_context_cache: _ssl_context_cache[cache_key] = _create_ssl_context( cafile=cafile, ssl_security_level=ssl_security_level, ssl_ecdh_curve=ssl_ecdh_curve, + ssl_ciphers=ssl_ciphers, ) # Return the cached SSL context diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 8358d15d30e..e131f79f2bd 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1,6 +1,5 @@ import asyncio import gc -import io import os import pathlib import ssl @@ -321,6 +320,45 @@ def test_get_ssl_configuration_integration(): assert ssl_context.verify_mode is not None +def _tls12_cipher_names(cipher_string: str) -> set[str]: + return {name for name in cipher_string.split(":") if name and not name.startswith("TLS_")} + + +def _context_tls12_cipher_names(context: ssl.SSLContext) -> set[str]: + return {cipher["name"] for cipher in context.get_ciphers() if cipher["protocol"] == "TLSv1.2"} + + +def _supported_tls12_cipher_names() -> set[str]: + unrestricted = ssl.create_default_context() + return _context_tls12_cipher_names(unrestricted) + + +def test_get_ssl_configuration_uses_fips_cipher_list_only_in_fips_mode(monkeypatch: pytest.MonkeyPatch): + from litellm.constants import DEFAULT_SSL_CIPHERS, FIPS_SSL_CIPHERS + from litellm.llms.custom_httpx import http_handler + + monkeypatch.setattr(http_handler, "_ssl_context_cache", {}) + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + monkeypatch.delenv("SSL_SECURITY_LEVEL", raising=False) + monkeypatch.delenv("SSL_ECDH_CURVE", raising=False) + + supported = _supported_tls12_cipher_names() + + default_context = get_ssl_configuration() + assert isinstance(default_context, ssl.SSLContext) + assert _context_tls12_cipher_names(default_context) == (_tls12_cipher_names(DEFAULT_SSL_CIPHERS) & supported) + assert get_ssl_configuration() is default_context + + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + fips_context = get_ssl_configuration() + assert fips_context is not default_context + assert isinstance(fips_context, ssl.SSLContext) + fips_names = _context_tls12_cipher_names(fips_context) + assert fips_names == (_tls12_cipher_names(FIPS_SSL_CIPHERS) & supported) + assert all("-GCM-" in name for name in fips_names) + assert get_ssl_configuration() is fips_context + + # Session Reuse Tests class MockClientSession: """Mock ClientSession that is not callable""" @@ -367,9 +405,11 @@ async def test_async_handler_with_shared_session(): async def test_get_async_httpx_client_with_shared_session(): """Test get_async_httpx_client with shared session""" from litellm.llms.custom_httpx.http_handler import ( - get_async_httpx_client, AsyncHTTPHandler as AsyncHTTPHandlerReload, ) + from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + ) from litellm.types.utils import LlmProviders # Create a mock shared session @@ -391,9 +431,11 @@ async def test_get_async_httpx_client_with_shared_session(): async def test_get_async_httpx_client_without_shared_session(): """Test get_async_httpx_client without shared session (backward compatibility)""" from litellm.llms.custom_httpx.http_handler import ( - get_async_httpx_client, AsyncHTTPHandler as AsyncHTTPHandlerReload, ) + from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + ) from litellm.types.utils import LlmProviders # Test without shared session @@ -429,6 +471,7 @@ async def test_session_reuse_chain(): def test_shared_session_parameter_in_acompletion(): """Test that acompletion function accepts shared_session parameter""" import inspect + from litellm.main import acompletion # Get the function signature @@ -446,6 +489,7 @@ def test_shared_session_parameter_in_acompletion(): def test_shared_session_parameter_in_completion(): """Test that completion function accepts shared_session parameter""" import inspect + from litellm.main import completion # Get the function signature @@ -464,9 +508,11 @@ def test_shared_session_parameter_in_completion(): async def test_session_reuse_integration(): """Integration test for session reuse functionality""" from litellm.llms.custom_httpx.http_handler import ( - get_async_httpx_client, AsyncHTTPHandler as AsyncHTTPHandlerReload, ) + from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + ) from litellm.types.utils import LlmProviders # Create a mock session @@ -1439,7 +1485,6 @@ async def test_connection_error_retry_forwards_content(method: str): await handler.close() - @pytest.fixture def forward_proxy_server(): """Plain HTTP forward proxy that records the absolute URIs it is asked to fetch.""" @@ -1570,9 +1615,7 @@ def private_ca_tls_upstream(tmp_path: pathlib.Path): ca_pem.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) key_pem = tmp_path / "key.pem" key_pem.write_bytes( - key.private_bytes( - serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() - ) + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) ) class OkTlsHandler(BaseHTTPRequestHandler): @@ -1747,8 +1790,11 @@ async def test_bounded_get_preserves_sdk_redirect_auth_and_query_handling(respx_ handler = AsyncHTTPHandler() try: response = await handler.get( - "https://example.com/spec.json?original=1", max_response_bytes=100, follow_redirects=True, - headers={"Authorization": "Bearer sentinel", "Accept-Encoding": "gzip"}, timeout=2.0, + "https://example.com/spec.json?original=1", + max_response_bytes=100, + follow_redirects=True, + headers={"Authorization": "Bearer sentinel", "Accept-Encoding": "gzip"}, + timeout=2.0, ) finally: await handler.close() diff --git a/tests/test_litellm/test_constants.py b/tests/test_litellm/test_constants.py index 12e473f68a4..199a518ec49 100644 --- a/tests/test_litellm/test_constants.py +++ b/tests/test_litellm/test_constants.py @@ -1,17 +1,9 @@ import ast -import inspect -import json -from unittest import mock - -import httpx -import pytest -import respx -from fastapi.testclient import TestClient - - import importlib +import inspect + +import pytest -import litellm from litellm import constants @@ -68,3 +60,39 @@ def _build_constant_env_var_map() -> dict[str, str]: env_var_map[constant_name] = env_var_name return env_var_map + + +def _cipher_suites(cipher_string: str) -> list[str]: + return [suite for suite in cipher_string.split(":") if suite] + + +def test_default_ssl_ciphers_excludes_chacha20(): + assert all("CHACHA" not in suite for suite in _cipher_suites(constants.DEFAULT_SSL_CIPHERS)) + + +def test_default_ssl_ciphers_tls12_suites_all_offer_forward_secrecy(): + tls12_suites = [suite for suite in _cipher_suites(constants.DEFAULT_SSL_CIPHERS) if not suite.startswith("TLS_")] + assert tls12_suites + assert all(suite.startswith("ECDHE-") for suite in tls12_suites) + + +def test_fips_ssl_ciphers_is_a_subset_of_default(): + assert set(_cipher_suites(constants.FIPS_SSL_CIPHERS)) <= set(_cipher_suites(constants.DEFAULT_SSL_CIPHERS)) + + +def test_fips_ssl_ciphers_tls12_suites_are_all_gcm(): + tls12_suites = [suite for suite in _cipher_suites(constants.FIPS_SSL_CIPHERS) if not suite.startswith("TLS_")] + assert tls12_suites + assert all("-GCM-" in suite for suite in tls12_suites) + + +def test_ssl_ciphers_env_override_applies_to_both_lists(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_SSL_CIPHERS", "ECDHE-RSA-AES128-GCM-SHA256") + importlib.reload(constants) + try: + assert constants.DEFAULT_SSL_CIPHERS == "ECDHE-RSA-AES128-GCM-SHA256" + assert constants.FIPS_SSL_CIPHERS == "ECDHE-RSA-AES128-GCM-SHA256" + finally: + monkeypatch.delenv("LITELLM_SSL_CIPHERS") + importlib.reload(constants) + assert constants.DEFAULT_SSL_CIPHERS != "ECDHE-RSA-AES128-GCM-SHA256" From 96eadd943e55e60ce2d49ec9f3c5dc3f3c2867da Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:08:17 +0000 Subject: [PATCH 07/15] test(http_handler): keep the cipher list tests to the changed behavior only Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/custom_httpx/test_http_handler.py | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index e131f79f2bd..b2293ed3ded 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1,5 +1,6 @@ import asyncio import gc +import io import os import pathlib import ssl @@ -404,11 +405,9 @@ async def test_async_handler_with_shared_session(): @pytest.mark.asyncio async def test_get_async_httpx_client_with_shared_session(): """Test get_async_httpx_client with shared session""" - from litellm.llms.custom_httpx.http_handler import ( - AsyncHTTPHandler as AsyncHTTPHandlerReload, - ) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, + AsyncHTTPHandler as AsyncHTTPHandlerReload, ) from litellm.types.utils import LlmProviders @@ -430,11 +429,9 @@ async def test_get_async_httpx_client_with_shared_session(): @pytest.mark.asyncio async def test_get_async_httpx_client_without_shared_session(): """Test get_async_httpx_client without shared session (backward compatibility)""" - from litellm.llms.custom_httpx.http_handler import ( - AsyncHTTPHandler as AsyncHTTPHandlerReload, - ) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, + AsyncHTTPHandler as AsyncHTTPHandlerReload, ) from litellm.types.utils import LlmProviders @@ -471,7 +468,6 @@ async def test_session_reuse_chain(): def test_shared_session_parameter_in_acompletion(): """Test that acompletion function accepts shared_session parameter""" import inspect - from litellm.main import acompletion # Get the function signature @@ -489,7 +485,6 @@ def test_shared_session_parameter_in_acompletion(): def test_shared_session_parameter_in_completion(): """Test that completion function accepts shared_session parameter""" import inspect - from litellm.main import completion # Get the function signature @@ -507,11 +502,9 @@ def test_shared_session_parameter_in_completion(): @pytest.mark.asyncio async def test_session_reuse_integration(): """Integration test for session reuse functionality""" - from litellm.llms.custom_httpx.http_handler import ( - AsyncHTTPHandler as AsyncHTTPHandlerReload, - ) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, + AsyncHTTPHandler as AsyncHTTPHandlerReload, ) from litellm.types.utils import LlmProviders @@ -1485,6 +1478,7 @@ async def test_connection_error_retry_forwards_content(method: str): await handler.close() + @pytest.fixture def forward_proxy_server(): """Plain HTTP forward proxy that records the absolute URIs it is asked to fetch.""" @@ -1615,7 +1609,9 @@ def private_ca_tls_upstream(tmp_path: pathlib.Path): ca_pem.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) key_pem = tmp_path / "key.pem" key_pem.write_bytes( - key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ) ) class OkTlsHandler(BaseHTTPRequestHandler): @@ -1790,11 +1786,8 @@ async def test_bounded_get_preserves_sdk_redirect_auth_and_query_handling(respx_ handler = AsyncHTTPHandler() try: response = await handler.get( - "https://example.com/spec.json?original=1", - max_response_bytes=100, - follow_redirects=True, - headers={"Authorization": "Bearer sentinel", "Accept-Encoding": "gzip"}, - timeout=2.0, + "https://example.com/spec.json?original=1", max_response_bytes=100, follow_redirects=True, + headers={"Authorization": "Bearer sentinel", "Accept-Encoding": "gzip"}, timeout=2.0, ) finally: await handler.close() From a5004e248c08d1d27c2e8083bfbabd7354285d08 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:37:04 +0000 Subject: [PATCH 08/15] test(integration): make the cipher burst test wait for client-side successes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/sdk/test_tls_cipher_wire.py | 59 +++++++++++++------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/tests/integration/sdk/test_tls_cipher_wire.py b/tests/integration/sdk/test_tls_cipher_wire.py index 62df651a293..e3094ece812 100644 --- a/tests/integration/sdk/test_tls_cipher_wire.py +++ b/tests/integration/sdk/test_tls_cipher_wire.py @@ -95,7 +95,7 @@ def _assert_served(peer: TlsPeer, user: str, response_id: str) -> None: def test_sync_handler_refuses_chacha20_only_peer(tls_cert: tuple[Path, Path]) -> None: cert_file, key_file = tls_cert with tls_peer(CHACHA20, cert_file, key_file) as peer: - with pytest.raises(httpx.ConnectError, match="SSL"): + with pytest.raises(httpx.ConnectError, match=r"(?i)ssl|handshake"): _sync_post(cert_file, peer.url, "u-cell1") assert peer.received() == (), peer.received() @@ -106,7 +106,7 @@ def test_async_httpx_handler_refuses_chacha20_only_peer( monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") cert_file, key_file = tls_cert with tls_peer(CHACHA20, cert_file, key_file) as peer: - with pytest.raises(httpx.ConnectError, match="SSL"): + with pytest.raises(httpx.ConnectError, match=r"(?i)ssl|handshake"): _async_post(cert_file, peer.url, "u-cell2") assert peer.received() == (), peer.received() @@ -114,7 +114,7 @@ def test_async_httpx_handler_refuses_chacha20_only_peer( def test_async_aiohttp_handler_refuses_chacha20_only_peer(tls_cert: tuple[Path, Path]) -> None: cert_file, key_file = tls_cert with tls_peer(CHACHA20, cert_file, key_file) as peer: - with pytest.raises(httpx.ProtocolError, match="SSL"): + with pytest.raises(httpx.ProtocolError, match=r"(?i)ssl|handshake"): _async_post(cert_file, peer.url, "u-cell3") assert peer.received() == (), peer.received() @@ -122,7 +122,7 @@ def test_async_aiohttp_handler_refuses_chacha20_only_peer(tls_cert: tuple[Path, def test_sync_handler_refuses_non_pfs_aes256_gcm_peer(tls_cert: tuple[Path, Path]) -> None: cert_file, key_file = tls_cert with tls_peer(AES256_GCM, cert_file, key_file) as peer: - with pytest.raises(httpx.ConnectError, match="SSL"): + with pytest.raises(httpx.ConnectError, match=r"(?i)ssl|handshake"): _sync_post(cert_file, peer.url, "u-cell4") assert peer.received() == (), peer.received() @@ -133,7 +133,7 @@ def test_async_httpx_handler_refuses_non_pfs_aes128_gcm_peer( monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") cert_file, key_file = tls_cert with tls_peer(AES128_GCM, cert_file, key_file) as peer: - with pytest.raises(httpx.ConnectError, match="SSL"): + with pytest.raises(httpx.ConnectError, match=r"(?i)ssl|handshake"): _async_post(cert_file, peer.url, "u-cell5") assert peer.received() == (), peer.received() @@ -150,7 +150,7 @@ def test_sync_handler_refuses_cbc_peer_in_fips_mode( monkeypatch.setenv("LITELLM_FIPS_MODE", "true") cert_file, key_file = tls_cert with tls_peer(CBC_SHA256, cert_file, key_file) as peer: - with pytest.raises(httpx.ConnectError, match="SSL"): + with pytest.raises(httpx.ConnectError, match=r"(?i)ssl|handshake"): _sync_post(cert_file, peer.url, "u-cell7") assert peer.received() == (), peer.received() @@ -162,7 +162,7 @@ def test_async_httpx_handler_refuses_cbc_sha384_peer_in_fips_mode( monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") cert_file, key_file = tls_cert with tls_peer(CBC_SHA384, cert_file, key_file) as peer: - with pytest.raises(httpx.ConnectError, match="SSL"): + with pytest.raises(httpx.ConnectError, match=r"(?i)ssl|handshake"): _async_post(cert_file, peer.url, "u-cell8") assert peer.received() == (), peer.received() @@ -221,7 +221,7 @@ def test_cipher_list_follows_fips_mode_toggle_in_one_process( with tls_peer(CBC_SHA256, cert_file, key_file) as peer: assert _sync_post(cert_file, peer.url, "u-cell12-before") == "chatcmpl-u-cell12-before" monkeypatch.setenv("LITELLM_FIPS_MODE", "true") - with pytest.raises(httpx.ConnectError, match="SSL"): + with pytest.raises(httpx.ConnectError, match=r"(?i)ssl|handshake"): _sync_post(cert_file, peer.url, "u-cell12-fips") monkeypatch.delenv("LITELLM_FIPS_MODE") assert _sync_post(cert_file, peer.url, "u-cell12-after") == "chatcmpl-u-cell12-after" @@ -253,15 +253,24 @@ def test_burst_survives_peer_restart_without_cipher_errors(tls_cert: tuple[Path, with tls_peer(PFS_GCM, cert_file, key_file) as peer: second_wave: Final = threading.Event() outcomes: Final[dict[int, BurstOutcome]] = {} + outcomes_lock: Final = threading.Lock() + + def _record(index: int, outcome: BurstOutcome) -> None: + with outcomes_lock: + outcomes[index] = outcome + + def _snapshot() -> tuple[BurstOutcome, ...]: + with outcomes_lock: + return tuple(outcomes.values()) def _one(index: int) -> None: if index >= 10: second_wave.wait(timeout=30) try: response_id: Final = _sync_post(cert_file, peer.url, f"user-{index}") - outcomes[index] = BurstOutcome(f"user-{index}", time.monotonic(), response_id, None) + _record(index, BurstOutcome(f"user-{index}", time.monotonic(), response_id, None)) except Exception as error: - outcomes[index] = BurstOutcome(f"user-{index}", time.monotonic(), None, error) + _record(index, BurstOutcome(f"user-{index}", time.monotonic(), None, error)) async def _async_burst() -> None: from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -278,11 +287,14 @@ def test_burst_survives_peer_restart_without_cipher_errors(tls_cert: tuple[Path, response: Final = await handler.client.post( f"{peer.url}/v1/chat/completions", json={"user": f"user-{index}"} ) - outcomes[index] = BurstOutcome( - f"user-{index}", time.monotonic(), str(JSON_OBJECT.validate_json(response.content)["id"]), None + _record( + index, + BurstOutcome( + f"user-{index}", time.monotonic(), str(JSON_OBJECT.validate_json(response.content)["id"]), None + ), ) except Exception as error: - outcomes[index] = BurstOutcome(f"user-{index}", time.monotonic(), None, error) + _record(index, BurstOutcome(f"user-{index}", time.monotonic(), None, error)) def _async_thread() -> None: asyncio.run(_async_burst()) @@ -291,7 +303,11 @@ def test_burst_survives_peer_restart_without_cipher_errors(tls_cert: tuple[Path, with ThreadPoolExecutor(max_workers=15) as pool: async_thread.start() futures: Final = tuple(pool.submit(_one, index) for index in range(15)) - eventually(lambda: len(peer.received()) >= 3, lambda ready: ready, seconds=15) + eventually( + lambda: sum(1 for outcome in _snapshot() if outcome.error is None) >= 3, + lambda ready: ready, + seconds=15, + ) stopped_at: Final = time.monotonic() peer.stop() peer.start() @@ -302,13 +318,18 @@ def test_burst_survives_peer_restart_without_cipher_errors(tls_cert: tuple[Path, future.result(timeout=60) async_thread.join(timeout=60) - assert len(outcomes) == BURST_SIZE - _assert_burst_outcomes(outcomes) - succeeded: Final = {o.user for o in outcomes.values() if o.error is None} + finished: Final = _snapshot() + assert len(finished) == BURST_SIZE + _assert_burst_outcomes({int(outcome.user.removeprefix("user-")): outcome for outcome in finished}) + succeeded: Final = {outcome.user for outcome in finished if outcome.error is None} recorded: Final = peer.received() assert sorted(record.user or "" for record in recorded) == sorted(succeeded) assert all(record.cipher == PFS_GCM for record in recorded) for user in succeeded: assert sum(1 for record in recorded if record.user == user) == 1, user - assert any(o.error is None and o.when < stopped_at for o in outcomes.values()), "no success before restart" - assert any(o.error is None and o.when > restarted_at for o in outcomes.values()), "no success after restart" + assert any(outcome.error is None and outcome.when < stopped_at for outcome in finished), ( + "no success before restart" + ) + assert any(outcome.error is None and outcome.when > restarted_at for outcome in finished), ( + "no success after restart" + ) From 693d834e448990d53256a4cf0d92bd12bc778d3c Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 13:20:09 +0000 Subject: [PATCH 09/15] test(integration): cover responses, messages and proxy chaos for the TLS cipher lists Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/_support/tls.py | 37 +++ .../configuration/test_tls_cipher_proxy.py | 269 +++++++++++++++++- 2 files changed, 305 insertions(+), 1 deletion(-) diff --git a/tests/integration/_support/tls.py b/tests/integration/_support/tls.py index 1abc8273ffe..06296ece85a 100644 --- a/tests/integration/_support/tls.py +++ b/tests/integration/_support/tls.py @@ -75,6 +75,34 @@ def _completion_body(identifier: str, model: str) -> dict[str, JsonValue]: } +def _response_body(identifier: str, model: str) -> dict[str, JsonValue]: + return { + "id": identifier, + "object": "response", + "created_at": 1, + "status": "completed", + "model": model, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "temperature": 1.0, + "truncation": "disabled", + "store": True, + "metadata": {}, + "output": [ + { + "type": "message", + "id": "msg_x", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "tls ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 5, "output_tokens": 2, "total_tokens": 7}, + } + + def _chunk_body( identifier: str, model: str, content: str, role: str | None, finish_reason: str | None ) -> dict[str, JsonValue]: @@ -115,6 +143,15 @@ class _CipherPeerHandler(BaseHTTPRequestHandler): identifier: Final = f"chatcmpl-{user or ''}" model_value: Final = body.get("model") model: Final = model_value if isinstance(model_value, str) else "gpt-4o-mini" + if self.path.endswith("/responses"): + response_id: Final = f"resp_{user or ''}" + responses_payload: Final = json.dumps(_response_body(response_id, model)).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(responses_payload))) + self.end_headers() + self.wfile.write(responses_payload) + return if body.get("stream") is True: chunks: Final = ( f"data: {json.dumps(_chunk_body(identifier, model, 'tls', 'assistant', None))}\n\n" diff --git a/tests/integration/configuration/test_tls_cipher_proxy.py b/tests/integration/configuration/test_tls_cipher_proxy.py index e118d858a86..2b10228e4fa 100644 --- a/tests/integration/configuration/test_tls_cipher_proxy.py +++ b/tests/integration/configuration/test_tls_cipher_proxy.py @@ -1,20 +1,25 @@ from __future__ import annotations import asyncio +import threading import uuid from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path from typing import Final +import httpx +import psutil import pytest import yaml +from anthropic import Anthropic +from anthropic import APIStatusError as AnthropicAPIStatusError from openai import APIStatusError, AsyncOpenAI, OpenAI from pydantic import JsonValue, TypeAdapter from tests.integration._support.client import Gateway, eventually, gateway_from_environment from tests.integration._support.database import read_rows -from tests.integration._support.process import owned_proxy +from tests.integration._support.process import OwnedProxy, owned_proxy, owned_proxy_process from tests.integration._support.tls import TlsPeer, tls_peer, write_self_signed_cert CHACHA20: Final = "ECDHE-RSA-CHACHA20-POLY1305" @@ -55,8 +60,10 @@ def cipher_proxy( base_config: Final = JSON_OBJECT.validate_python( yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) ) + base_settings: Final = JSON_OBJECT.validate_python(base_config.get("litellm_settings", {})) config: Final = { **base_config, + "litellm_settings": {**base_settings, "cache": False}, "model_list": [ { "model_name": "tls-peer", @@ -186,3 +193,263 @@ def test_proxy_reports_ssl_error_for_non_pfs_peer(cipher_proxy: Gateway, tls_pee assert readiness.status_code == 200, readiness.text recovered: Final = _sync_completion(cipher_proxy, "tls-peer", f"user15-ok-{marker}") assert recovered == f"chatcmpl-user15-ok-{marker}" + + +@pytest.fixture(scope="module") +def cipher_owned_proxy( + tls_cert: tuple[Path, Path], tls_peers: Peers, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[OwnedProxy]: + cert_file, _ = tls_cert + directory: Final = tmp_path_factory.mktemp("tlsproxyworkers") + base_config: Final = JSON_OBJECT.validate_python( + yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + ) + base_settings: Final = JSON_OBJECT.validate_python(base_config.get("litellm_settings", {})) + config: Final = { + **base_config, + "litellm_settings": {**base_settings, "cache": False}, + "model_list": [ + { + "model_name": "tls-peer", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{tls_peers.gcm.url}/v1", + "api_key": "sk-peer", + }, + } + ], + } + config_path: Final = directory / "tls_cipher_workers_config.yaml" + config_path.write_text(yaml.safe_dump(config)) + with gateway_from_environment() as gateway: + with owned_proxy_process( + gateway, directory, {"SSL_CERT_FILE": str(cert_file)}, config=config_path, workers=2 + ) as owned: + yield owned + + +def _responses_request(proxy: Gateway, model: str, user: str) -> str: + response: Final = proxy.request("POST", "/v1/responses", {"model": model, "input": "tls", "user": user}) + assert response.status_code == 200, response.text + return str(JSON_OBJECT.validate_json(response.text)["id"]) + + +def _messages_request(proxy: Gateway, model: str) -> str: + with Anthropic(api_key=proxy.key, base_url=str(proxy.client.base_url), max_retries=0, timeout=60) as client: + message: Final = client.messages.create( + model=model, max_tokens=8, messages=[{"role": "user", "content": "tls"}] + ) + return message.id + + +def test_proxy_round_trips_responses_over_pfs_gcm_peer_and_writes_spend( + cipher_proxy: Gateway, tls_peers: Peers +) -> None: + marker: Final = uuid.uuid4().hex[:8] + user: Final = f"user19-resp-{marker}" + response_id: Final = _responses_request(cipher_proxy, "tls-peer", user) + assert response_id.startswith("resp_"), response_id + recorded: Final = tuple(record for record in tls_peers.gcm.received() if record.path.endswith("/responses")) + assert [record.user for record in recorded] == [user], recorded + assert all(record.cipher == PFS_GCM for record in recorded), recorded + spend_rows: Final = eventually( + lambda: read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (response_id,)), + lambda values: len(values) == 1, + seconds=70, + ) + assert spend_rows[0]["request_id"] == response_id + + +def test_proxy_reports_ssl_error_for_responses_on_chacha20_only_peer(cipher_proxy: Gateway, tls_peers: Peers) -> None: + before: Final = len(tls_peers.chacha.received()) + response: Final = cipher_proxy.request( + "POST", + "/v1/responses", + {"model": "tls-chacha-peer", "input": "tls", "user": f"user20-{uuid.uuid4().hex[:8]}"}, + ) + assert response.status_code >= 400, response.text + assert "ssl" in response.text.lower() or "connection error" in response.text.lower(), response.text + assert len(tls_peers.chacha.received()) == before + readiness: Final = cipher_proxy.client.get("/health/readiness") + assert readiness.status_code == 200, readiness.text + + +def test_proxy_round_trips_messages_over_pfs_gcm_peer(cipher_proxy: Gateway, tls_peers: Peers) -> None: + before: Final = len(tls_peers.gcm.received()) + message_id: Final = _messages_request(cipher_proxy, "tls-peer") + assert message_id, "messages response carried no id" + recorded: Final = tls_peers.gcm.received() + assert len(recorded) - before == 1, recorded[before:] + assert recorded[-1].cipher == PFS_GCM, recorded[-1] + + +def test_proxy_reports_ssl_error_for_messages_on_chacha20_only_peer(cipher_proxy: Gateway, tls_peers: Peers) -> None: + before: Final = len(tls_peers.chacha.received()) + with pytest.raises(AnthropicAPIStatusError) as raised: + _messages_request(cipher_proxy, "tls-chacha-peer") + assert raised.value.status_code >= 400 + assert "ssl" in str(raised.value).lower() or "connection error" in str(raised.value).lower(), str(raised.value) + assert len(tls_peers.chacha.received()) == before + readiness: Final = cipher_proxy.client.get("/health/readiness") + assert readiness.status_code == 200, readiness.text + + +@dataclass(frozen=True, slots=True) +class _BurstOutcome: + user: str | None + status: int + body: str + + +def _chaos_request(proxy: Gateway, kind: str, index: int, marker: str) -> _BurstOutcome: + user: Final = f"chaos-{kind}-{index}-{marker}" + if kind == "chat": + try: + chat_response: Final = proxy.request( + "POST", + "/v1/chat/completions", + { + "model": "tls-peer", + "messages": [{"role": "user", "content": "tls"}], + "user": user, + "num_retries": 0, + }, + ) + return _BurstOutcome(user=user, status=chat_response.status_code, body=chat_response.text) + except httpx.HTTPError: + return _BurstOutcome(user=user, status=0, body="") + if kind == "stream": + try: + stream_response: Final = proxy.request( + "POST", + "/v1/chat/completions", + { + "model": "tls-peer", + "messages": [{"role": "user", "content": "tls"}], + "user": user, + "stream": True, + "num_retries": 0, + }, + ) + return _BurstOutcome(user=user, status=stream_response.status_code, body=stream_response.text) + except httpx.HTTPError: + return _BurstOutcome(user=user, status=0, body="") + if kind == "responses": + try: + responses_response: Final = proxy.request( + "POST", + "/v1/responses", + {"model": "tls-peer", "input": "tls", "user": user, "num_retries": 0}, + ) + return _BurstOutcome(user=user, status=responses_response.status_code, body=responses_response.text) + except httpx.HTTPError: + return _BurstOutcome(user=user, status=0, body="") + try: + messages_response: Final = proxy.request( + "POST", + "/v1/messages", + { + "model": "tls-peer", + "max_tokens": 8, + "messages": [{"role": "user", "content": f"tls {user}"}], + "num_retries": 0, + }, + headers={"anthropic-version": "2023-06-01"}, + ) + return _BurstOutcome(user=None, status=messages_response.status_code, body=messages_response.text) + except httpx.HTTPError: + return _BurstOutcome(user=None, status=0, body="") + + +def test_proxy_burst_survives_peer_restart_with_every_outcome_recorded(cipher_proxy: Gateway, tls_peers: Peers) -> None: + marker: Final = uuid.uuid4().hex[:8] + outcomes_lock: Final = threading.Lock() + outcomes: Final[list[_BurstOutcome]] = [] + kinds: Final = ("chat", "stream", "responses", "messages", "chat", "stream") + + def _run(kind: str, index: int) -> None: + outcome: Final = _chaos_request(cipher_proxy, kind, index, marker) + with outcomes_lock: + outcomes.append(outcome) + + def _snapshot() -> tuple[_BurstOutcome, ...]: + with outcomes_lock: + return tuple(outcomes) + + workers: Final = [ + threading.Thread(target=_run, args=(kind, index), daemon=True) for index in range(5) for kind in kinds + ] + before: Final = len(tls_peers.gcm.received()) + for worker in workers: + worker.start() + eventually( + lambda: sum(1 for outcome in _snapshot() if outcome.status == 200), + lambda successes: successes >= 3, + seconds=60, + ) + tls_peers.gcm.stop() + tls_peers.gcm.start() + for worker in workers: + worker.join(timeout=120) + finished: Final = _snapshot() + assert len(finished) == 30, finished + assert not any(worker.is_alive() for worker in workers), "burst thread still running" + assert not any(outcome.status == 0 for outcome in finished), finished + records: Final = tls_peers.gcm.received()[before:] + assert all(record.cipher == PFS_GCM for record in records), records + successful_users: Final = { + outcome.user for outcome in finished if outcome.status == 200 and outcome.user is not None + } + failed_users: Final = {outcome.user for outcome in finished if outcome.status != 200 and outcome.user is not None} + assert all(any(record.user == user for record in records) for user in successful_users), (successful_users, records) + assert all(not any(record.user == user for record in records) for user in failed_users), (failed_users, records) + recorded_users: Final = {record.user for record in records if record.user is not None} + assert recorded_users <= successful_users, (recorded_users - successful_users, records) + anonymous: Final = sum(1 for record in records if record.user is None) + messages_ok: Final = sum(1 for outcome in finished if outcome.user is None and outcome.status == 200) + assert anonymous == messages_ok, (anonymous, messages_ok, records) + error_bodies: Final = tuple(outcome.body for outcome in finished if outcome.status >= 400) + assert all("ssl" not in body.lower() and "cipher" not in body.lower() for body in error_bodies), error_bodies + readiness: Final = cipher_proxy.client.get("/health/readiness") + assert readiness.status_code == 200, readiness.text + + +def test_proxy_burst_survives_losing_one_worker(cipher_owned_proxy: OwnedProxy, tls_peers: Peers) -> None: + proxy: Final = cipher_owned_proxy.gateway + marker: Final = uuid.uuid4().hex[:8] + outcomes_lock: Final = threading.Lock() + outcomes: Final[list[_BurstOutcome]] = [] + + def _run(index: int) -> None: + outcome: Final = _chaos_request(proxy, "chat", index, marker) + with outcomes_lock: + outcomes.append(outcome) + + def _snapshot() -> tuple[_BurstOutcome, ...]: + with outcomes_lock: + return tuple(outcomes) + + workers: Final = [threading.Thread(target=_run, args=(index,), daemon=True) for index in range(12)] + for worker in workers: + worker.start() + eventually( + lambda: sum(1 for outcome in _snapshot() if outcome.status == 200), + lambda successes: successes >= 2, + seconds=60, + ) + children: Final = psutil.Process(cipher_owned_proxy.process.pid).children() + assert children, "two-worker proxy has no child processes to kill" + children[0].kill() + for worker in workers: + worker.join(timeout=120) + finished: Final = _snapshot() + assert len(finished) == 12, finished + assert any(outcome.status == 200 for outcome in finished), finished + + def _ready() -> bool: + try: + return proxy.client.get("/health/readiness").status_code == 200 + except httpx.TransportError: + return False + + eventually(_ready, lambda ready: ready, seconds=60) From 4ca2493f73c1f29c2e047d76051f8a359011389f Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 14:58:48 +0000 Subject: [PATCH 10/15] test(integration): make the TLS cipher chaos cells immutable and window scoped Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../configuration/test_tls_cipher_proxy.py | 127 +++++++++--------- 1 file changed, 60 insertions(+), 67 deletions(-) diff --git a/tests/integration/configuration/test_tls_cipher_proxy.py b/tests/integration/configuration/test_tls_cipher_proxy.py index 2b10228e4fa..e44b7f3d43a 100644 --- a/tests/integration/configuration/test_tls_cipher_proxy.py +++ b/tests/integration/configuration/test_tls_cipher_proxy.py @@ -1,9 +1,10 @@ from __future__ import annotations import asyncio -import threading +import time import uuid from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path from typing import Final @@ -247,9 +248,12 @@ def test_proxy_round_trips_responses_over_pfs_gcm_peer_and_writes_spend( ) -> None: marker: Final = uuid.uuid4().hex[:8] user: Final = f"user19-resp-{marker}" + before: Final = len(tls_peers.gcm.received()) response_id: Final = _responses_request(cipher_proxy, "tls-peer", user) assert response_id.startswith("resp_"), response_id - recorded: Final = tuple(record for record in tls_peers.gcm.received() if record.path.endswith("/responses")) + recorded: Final = tuple( + record for record in tls_peers.gcm.received()[before:] if record.path.endswith("/responses") + ) assert [record.user for record in recorded] == [user], recorded assert all(record.cipher == PFS_GCM for record in recorded), recorded spend_rows: Final = eventually( @@ -299,6 +303,7 @@ class _BurstOutcome: user: str | None status: int body: str + finished_at: float def _chaos_request(proxy: Gateway, kind: str, index: int, marker: str) -> _BurstOutcome: @@ -315,9 +320,11 @@ def _chaos_request(proxy: Gateway, kind: str, index: int, marker: str) -> _Burst "num_retries": 0, }, ) - return _BurstOutcome(user=user, status=chat_response.status_code, body=chat_response.text) + return _BurstOutcome( + user=user, status=chat_response.status_code, body=chat_response.text, finished_at=time.monotonic() + ) except httpx.HTTPError: - return _BurstOutcome(user=user, status=0, body="") + return _BurstOutcome(user=user, status=0, body="", finished_at=time.monotonic()) if kind == "stream": try: stream_response: Final = proxy.request( @@ -331,9 +338,11 @@ def _chaos_request(proxy: Gateway, kind: str, index: int, marker: str) -> _Burst "num_retries": 0, }, ) - return _BurstOutcome(user=user, status=stream_response.status_code, body=stream_response.text) + return _BurstOutcome( + user=user, status=stream_response.status_code, body=stream_response.text, finished_at=time.monotonic() + ) except httpx.HTTPError: - return _BurstOutcome(user=user, status=0, body="") + return _BurstOutcome(user=user, status=0, body="", finished_at=time.monotonic()) if kind == "responses": try: responses_response: Final = proxy.request( @@ -341,9 +350,14 @@ def _chaos_request(proxy: Gateway, kind: str, index: int, marker: str) -> _Burst "/v1/responses", {"model": "tls-peer", "input": "tls", "user": user, "num_retries": 0}, ) - return _BurstOutcome(user=user, status=responses_response.status_code, body=responses_response.text) + return _BurstOutcome( + user=user, + status=responses_response.status_code, + body=responses_response.text, + finished_at=time.monotonic(), + ) except httpx.HTTPError: - return _BurstOutcome(user=user, status=0, body="") + return _BurstOutcome(user=user, status=0, body="", finished_at=time.monotonic()) try: messages_response: Final = proxy.request( "POST", @@ -356,44 +370,34 @@ def _chaos_request(proxy: Gateway, kind: str, index: int, marker: str) -> _Burst }, headers={"anthropic-version": "2023-06-01"}, ) - return _BurstOutcome(user=None, status=messages_response.status_code, body=messages_response.text) + return _BurstOutcome( + user=None, + status=messages_response.status_code, + body=messages_response.text, + finished_at=time.monotonic(), + ) except httpx.HTTPError: - return _BurstOutcome(user=None, status=0, body="") + return _BurstOutcome(user=None, status=0, body="", finished_at=time.monotonic()) def test_proxy_burst_survives_peer_restart_with_every_outcome_recorded(cipher_proxy: Gateway, tls_peers: Peers) -> None: marker: Final = uuid.uuid4().hex[:8] - outcomes_lock: Final = threading.Lock() - outcomes: Final[list[_BurstOutcome]] = [] kinds: Final = ("chat", "stream", "responses", "messages", "chat", "stream") - - def _run(kind: str, index: int) -> None: - outcome: Final = _chaos_request(cipher_proxy, kind, index, marker) - with outcomes_lock: - outcomes.append(outcome) - - def _snapshot() -> tuple[_BurstOutcome, ...]: - with outcomes_lock: - return tuple(outcomes) - - workers: Final = [ - threading.Thread(target=_run, args=(kind, index), daemon=True) for index in range(5) for kind in kinds - ] before: Final = len(tls_peers.gcm.received()) - for worker in workers: - worker.start() - eventually( - lambda: sum(1 for outcome in _snapshot() if outcome.status == 200), - lambda successes: successes >= 3, - seconds=60, - ) - tls_peers.gcm.stop() - tls_peers.gcm.start() - for worker in workers: - worker.join(timeout=120) - finished: Final = _snapshot() + executor: Final = ThreadPoolExecutor(max_workers=30) + with executor: + futures: Final = tuple( + executor.submit(_chaos_request, cipher_proxy, kind, index, marker) for index in range(5) for kind in kinds + ) + eventually( + lambda: sum(1 for future in futures if future.done() and future.result().status == 200), + lambda successes: successes >= 3, + seconds=60, + ) + tls_peers.gcm.stop() + tls_peers.gcm.start() + finished: Final = tuple(future.result(timeout=120) for future in futures) assert len(finished) == 30, finished - assert not any(worker.is_alive() for worker in workers), "burst thread still running" assert not any(outcome.status == 0 for outcome in finished), finished records: Final = tls_peers.gcm.received()[before:] assert all(record.cipher == PFS_GCM for record in records), records @@ -408,8 +412,10 @@ def test_proxy_burst_survives_peer_restart_with_every_outcome_recorded(cipher_pr anonymous: Final = sum(1 for record in records if record.user is None) messages_ok: Final = sum(1 for outcome in finished if outcome.user is None and outcome.status == 200) assert anonymous == messages_ok, (anonymous, messages_ok, records) - error_bodies: Final = tuple(outcome.body for outcome in finished if outcome.status >= 400) - assert all("ssl" not in body.lower() and "cipher" not in body.lower() for body in error_bodies), error_bodies + after_id: Final = _sync_completion(cipher_proxy, "tls-peer", f"chaos-after-{marker}") + assert after_id == f"chatcmpl-chaos-after-{marker}" + last_record: Final = tls_peers.gcm.received()[-1] + assert last_record.cipher == PFS_GCM and last_record.user == f"chaos-after-{marker}", last_record readiness: Final = cipher_proxy.client.get("/health/readiness") assert readiness.status_code == 200, readiness.text @@ -417,34 +423,21 @@ def test_proxy_burst_survives_peer_restart_with_every_outcome_recorded(cipher_pr def test_proxy_burst_survives_losing_one_worker(cipher_owned_proxy: OwnedProxy, tls_peers: Peers) -> None: proxy: Final = cipher_owned_proxy.gateway marker: Final = uuid.uuid4().hex[:8] - outcomes_lock: Final = threading.Lock() - outcomes: Final[list[_BurstOutcome]] = [] - - def _run(index: int) -> None: - outcome: Final = _chaos_request(proxy, "chat", index, marker) - with outcomes_lock: - outcomes.append(outcome) - - def _snapshot() -> tuple[_BurstOutcome, ...]: - with outcomes_lock: - return tuple(outcomes) - - workers: Final = [threading.Thread(target=_run, args=(index,), daemon=True) for index in range(12)] - for worker in workers: - worker.start() - eventually( - lambda: sum(1 for outcome in _snapshot() if outcome.status == 200), - lambda successes: successes >= 2, - seconds=60, - ) - children: Final = psutil.Process(cipher_owned_proxy.process.pid).children() - assert children, "two-worker proxy has no child processes to kill" - children[0].kill() - for worker in workers: - worker.join(timeout=120) - finished: Final = _snapshot() + executor: Final = ThreadPoolExecutor(max_workers=12) + with executor: + futures: Final = tuple(executor.submit(_chaos_request, proxy, "chat", index, marker) for index in range(12)) + eventually( + lambda: sum(1 for future in futures if future.done() and future.result().status == 200), + lambda successes: successes >= 2, + seconds=60, + ) + children: Final = psutil.Process(cipher_owned_proxy.process.pid).children() + assert children, "two-worker proxy has no child processes to kill" + killed_at: Final = time.monotonic() + children[0].kill() + finished: Final = tuple(future.result(timeout=120) for future in futures) assert len(finished) == 12, finished - assert any(outcome.status == 200 for outcome in finished), finished + assert any(outcome.status == 200 and outcome.finished_at > killed_at for outcome in finished), finished def _ready() -> bool: try: From 495418966a51f40f0fb3eecfe6705a813f46cc8b Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 15:10:43 +0000 Subject: [PATCH 11/15] test(integration): bound chaos peer records instead of assuming no lost responses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../configuration/test_tls_cipher_proxy.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/integration/configuration/test_tls_cipher_proxy.py b/tests/integration/configuration/test_tls_cipher_proxy.py index e44b7f3d43a..b261af86207 100644 --- a/tests/integration/configuration/test_tls_cipher_proxy.py +++ b/tests/integration/configuration/test_tls_cipher_proxy.py @@ -401,17 +401,21 @@ def test_proxy_burst_survives_peer_restart_with_every_outcome_recorded(cipher_pr assert not any(outcome.status == 0 for outcome in finished), finished records: Final = tls_peers.gcm.received()[before:] assert all(record.cipher == PFS_GCM for record in records), records + submitted_users: Final = {outcome.user for outcome in finished if outcome.user is not None} successful_users: Final = { outcome.user for outcome in finished if outcome.status == 200 and outcome.user is not None } - failed_users: Final = {outcome.user for outcome in finished if outcome.status != 200 and outcome.user is not None} assert all(any(record.user == user for record in records) for user in successful_users), (successful_users, records) - assert all(not any(record.user == user for record in records) for user in failed_users), (failed_users, records) recorded_users: Final = {record.user for record in records if record.user is not None} - assert recorded_users <= successful_users, (recorded_users - successful_users, records) + assert recorded_users <= submitted_users, (recorded_users - submitted_users, records) + assert all(sum(1 for record in records if record.user == user) <= 3 for user in submitted_users), ( + submitted_users, + records, + ) anonymous: Final = sum(1 for record in records if record.user is None) + messages_outcomes: Final = sum(1 for outcome in finished if outcome.user is None) messages_ok: Final = sum(1 for outcome in finished if outcome.user is None and outcome.status == 200) - assert anonymous == messages_ok, (anonymous, messages_ok, records) + assert messages_ok <= anonymous <= messages_outcomes * 3, (anonymous, messages_ok, records) after_id: Final = _sync_completion(cipher_proxy, "tls-peer", f"chaos-after-{marker}") assert after_id == f"chatcmpl-chaos-after-{marker}" last_record: Final = tls_peers.gcm.received()[-1] From 8e70e83f773ba496bc49f1c7bf479568dd68fd39 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 15:15:52 +0000 Subject: [PATCH 12/15] test(integration): slow half the kill-worker burst so the kill lands mid-flight Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/_support/tls.py | 3 +++ .../configuration/test_tls_cipher_proxy.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/integration/_support/tls.py b/tests/integration/_support/tls.py index 06296ece85a..fd06f5053ae 100644 --- a/tests/integration/_support/tls.py +++ b/tests/integration/_support/tls.py @@ -5,6 +5,7 @@ import ipaddress import json import ssl import threading +import time from collections.abc import Generator from contextlib import contextmanager from dataclasses import dataclass @@ -140,6 +141,8 @@ class _CipherPeerHandler(BaseHTTPRequestHandler): user_value: Final = body.get("user") user: Final = user_value if isinstance(user_value, str) else None peer.record(TlsPeerRequest(path=self.path, user=user, cipher=_negotiated_cipher(self))) + if user is not None and "slow" in user: + time.sleep(5) identifier: Final = f"chatcmpl-{user or ''}" model_value: Final = body.get("model") model: Final = model_value if isinstance(model_value, str) else "gpt-4o-mini" diff --git a/tests/integration/configuration/test_tls_cipher_proxy.py b/tests/integration/configuration/test_tls_cipher_proxy.py index b261af86207..1a492a4dfd6 100644 --- a/tests/integration/configuration/test_tls_cipher_proxy.py +++ b/tests/integration/configuration/test_tls_cipher_proxy.py @@ -308,7 +308,7 @@ class _BurstOutcome: def _chaos_request(proxy: Gateway, kind: str, index: int, marker: str) -> _BurstOutcome: user: Final = f"chaos-{kind}-{index}-{marker}" - if kind == "chat": + if kind in ("chat", "slow"): try: chat_response: Final = proxy.request( "POST", @@ -429,7 +429,10 @@ def test_proxy_burst_survives_losing_one_worker(cipher_owned_proxy: OwnedProxy, marker: Final = uuid.uuid4().hex[:8] executor: Final = ThreadPoolExecutor(max_workers=12) with executor: - futures: Final = tuple(executor.submit(_chaos_request, proxy, "chat", index, marker) for index in range(12)) + futures: Final = tuple( + executor.submit(_chaos_request, proxy, "slow" if index % 2 else "chat", index, marker) + for index in range(12) + ) eventually( lambda: sum(1 for future in futures if future.done() and future.result().status == 200), lambda successes: successes >= 2, @@ -441,7 +444,9 @@ def test_proxy_burst_survives_losing_one_worker(cipher_owned_proxy: OwnedProxy, children[0].kill() finished: Final = tuple(future.result(timeout=120) for future in futures) assert len(finished) == 12, finished - assert any(outcome.status == 200 and outcome.finished_at > killed_at for outcome in finished), finished + assert any(outcome.finished_at > killed_at for outcome in finished), finished + post_kill_id: Final = _sync_completion(proxy, "tls-peer", f"post-kill-{marker}") + assert post_kill_id == f"chatcmpl-post-kill-{marker}" def _ready() -> bool: try: From d459735b8b5f21e21cbdc70b27fa0eaa7954dbaf Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 17:43:20 +0000 Subject: [PATCH 13/15] test(integration): fail the restart chaos cell on any failure before the peer outage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/configuration/test_tls_cipher_proxy.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/configuration/test_tls_cipher_proxy.py b/tests/integration/configuration/test_tls_cipher_proxy.py index 1a492a4dfd6..aefe2074445 100644 --- a/tests/integration/configuration/test_tls_cipher_proxy.py +++ b/tests/integration/configuration/test_tls_cipher_proxy.py @@ -394,10 +394,15 @@ def test_proxy_burst_survives_peer_restart_with_every_outcome_recorded(cipher_pr lambda successes: successes >= 3, seconds=60, ) + stopped_at: Final = time.monotonic() tls_peers.gcm.stop() tls_peers.gcm.start() finished: Final = tuple(future.result(timeout=120) for future in futures) assert len(finished) == 30, finished + assert all(outcome.finished_at >= stopped_at for outcome in finished if outcome.status != 200), ( + stopped_at, + finished, + ) assert not any(outcome.status == 0 for outcome in finished), finished records: Final = tls_peers.gcm.received()[before:] assert all(record.cipher == PFS_GCM for record in records), records From d4041ddb8d2367efa0b51d7c2eabcaebbc1b1cd5 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 19:15:03 +0000 Subject: [PATCH 14/15] test(integration): record TLS peer requests as an immutable tuple Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/_support/tls.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/_support/tls.py b/tests/integration/_support/tls.py index fd06f5053ae..e40b92d3dd4 100644 --- a/tests/integration/_support/tls.py +++ b/tests/integration/_support/tls.py @@ -187,7 +187,7 @@ class TlsPeer: self._server: _CipherPeer | None = None self._thread: threading.Thread | None = None self._lock = threading.Lock() - self._requests: list[TlsPeerRequest] = [] + self._requests: tuple[TlsPeerRequest, ...] = () @property def url(self) -> str: @@ -195,11 +195,11 @@ class TlsPeer: def record(self, request: TlsPeerRequest) -> None: with self._lock: - self._requests.append(request) + self._requests = (*self._requests, request) def received(self) -> tuple[TlsPeerRequest, ...]: with self._lock: - return tuple(self._requests) + return self._requests def start(self) -> None: server: Final = _CipherPeer(("127.0.0.1", self.port), self) From 2277895897f1761bb7802dbc24276ec37d5d6fb8 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 5 Oct 2026 09:40:37 +0000 Subject: [PATCH 15/15] test(tls): window the gcm peer history in the proxy round-trip cell Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/configuration/test_tls_cipher_proxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/configuration/test_tls_cipher_proxy.py b/tests/integration/configuration/test_tls_cipher_proxy.py index aefe2074445..aac030f0712 100644 --- a/tests/integration/configuration/test_tls_cipher_proxy.py +++ b/tests/integration/configuration/test_tls_cipher_proxy.py @@ -138,6 +138,7 @@ async def _async_completion(proxy: Gateway, model: str, user: str, *, stream: bo def test_proxy_round_trips_over_pfs_gcm_peer_and_writes_spend(cipher_proxy: Gateway, tls_peers: Peers) -> None: marker: Final = uuid.uuid4().hex[:8] + before: Final = len(tls_peers.gcm.received()) sync_id: Final = _sync_completion(cipher_proxy, "tls-peer", f"user13-sync-{marker}") stream_id: Final = _sync_completion(cipher_proxy, "tls-peer", f"user13-stream-{marker}", stream=True) async_id: Final = asyncio.run(_async_completion(cipher_proxy, "tls-peer", f"user13-async-{marker}")) @@ -145,7 +146,7 @@ def test_proxy_round_trips_over_pfs_gcm_peer_and_writes_spend(cipher_proxy: Gate _async_completion(cipher_proxy, "tls-peer", f"user13-astream-{marker}", stream=True) ) users: Final = {f"user13-{leg}-{marker}" for leg in ("sync", "stream", "async", "astream")} - recorded: Final = tls_peers.gcm.received() + recorded: Final = tls_peers.gcm.received()[before:] assert {record.user for record in recorded} == users, recorded assert all(record.cipher == PFS_GCM for record in recorded), recorded spend_rows: Final = tuple(