From 401c320c45101d7e29234232d74ea189bcc2fac5 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:14:22 +0000 Subject: [PATCH] feat(fips): build grpcio from source against system OpenSSL and drop its scanner waiver Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docker/Dockerfile.fips | 37 ++++ docker/scan_bundled_crypto.sh | 1 - .../test_fips_bundled_crypto_scan.py | 88 ++++++++ .../test_proxy_without_grpc_stack.py | 199 ++++++++++++++++++ 4 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 tests/integration/configuration/test_fips_bundled_crypto_scan.py create mode 100644 tests/integration/configuration/test_proxy_without_grpc_stack.py diff --git a/docker/Dockerfile.fips b/docker/Dockerfile.fips index 1aa0f893e1d..bb95ddf6ebc 100644 --- a/docker/Dockerfile.fips +++ b/docker/Dockerfile.fips @@ -36,6 +36,35 @@ RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${ make -j"$(nproc)" pgbouncer && \ install -m 0755 pgbouncer /usr/local/bin/pgbouncer +# grpcio's PyPI wheel statically bundles BoringSSL, which is not the validated +# module. Rebuild it from source against the FIPS system OpenSSL instead +# (GRPC_PYTHON_BUILD_SYSTEM_OPENSSL), so cygrpc links libcrypto.so.3 like +# every other consumer in the image. The version is read from uv.lock through +# its own tiny stage so this long compile is cached until grpcio itself moves. +FROM $LITELLM_BUILD_IMAGE AS grpcio-version +COPY uv.lock /uv.lock +RUN awk '/^name = "grpcio"$/ { found = 1 } found && /^version = / { gsub(/"/, "", $3); print $3; exit }' /uv.lock \ + > /grpcio-version && test -s /grpcio-version + +FROM $LITELLM_BUILD_IMAGE AS grpcio-builder +USER root +RUN --mount=type=secret,id=cgr-token \ + HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \ + apk add --no-cache build-base python-3.13 python-3.13-dev openssl-dev zlib-dev binutils +COPY --from=uvbin /uv /usr/local/bin/uv +COPY --from=grpcio-version /grpcio-version /grpcio-version +ENV UV_PYTHON_DOWNLOADS=0 +RUN uv venv --seed --python python3.13 /grpcio-venv && \ + GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \ + GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 \ + GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS="$(nproc)" \ + /grpcio-venv/bin/python -m pip wheel --no-binary grpcio --no-deps --no-cache-dir \ + --wheel-dir /wheels "grpcio==$(cat /grpcio-version)" && \ + /grpcio-venv/bin/python -m zipfile -e /wheels/grpcio-*.whl /wheels/check && \ + objdump -p /wheels/check/grpc/_cython/cygrpc*.so | grep -q 'NEEDED.*libcrypto\.so\.3' && \ + ! grep -aq BoringSSL /wheels/check/grpc/_cython/cygrpc*.so && \ + rm -rf /wheels/check + FROM --platform=$BUILDPLATFORM $UI_BUILD_IMAGE AS ui-builder ENV NEXT_TELEMETRY_DISABLED=1 \ @@ -62,6 +91,7 @@ RUN --mount=type=secret,id=cgr-token \ HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \ apk add --no-cache \ bash \ + binutils \ gcc \ python-3.13 \ python-3.13-dev \ @@ -107,6 +137,11 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra bedrock-realtime \ --python python3.13 +COPY --from=grpcio-builder /wheels /wheels +RUN uv pip install --python /app/.venv/bin/python --no-deps --reinstall --no-cache /wheels/grpcio-*.whl && \ + strip --strip-unneeded /app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc*.so && \ + rm -rf /wheels + RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma @@ -155,6 +190,8 @@ RUN --mount=type=secret,id=cgr-token \ HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \ apk add --no-cache binutils RUN openssl-fips-test +RUN python -c "import grpc._cython.cygrpc" && \ + objdump -p /app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc*.so | grep -q 'NEEDED.*libcrypto\.so\.3' RUN chmod +x docker/scan_bundled_crypto.sh && \ docker/scan_bundled_crypto.sh /app/.venv /opt/prisma /usr/local/bin diff --git a/docker/scan_bundled_crypto.sh b/docker/scan_bundled_crypto.sh index fb8bd53b4ce..d6e926dff83 100755 --- a/docker/scan_bundled_crypto.sh +++ b/docker/scan_bundled_crypto.sh @@ -51,7 +51,6 @@ crypto_markers() { is_waived() { case "$1" in */cryptography/hazmat/bindings/_rust.abi3.so) ;; - */grpc/_cython/cygrpc*.so) ;; */hf_xet/hf_xet.abi3.so) ;; */_awscrt.abi3.so) ;; */pyroscope/_native__lib*.so) ;; diff --git a/tests/integration/configuration/test_fips_bundled_crypto_scan.py b/tests/integration/configuration/test_fips_bundled_crypto_scan.py new file mode 100644 index 00000000000..4f23ccccf87 --- /dev/null +++ b/tests/integration/configuration/test_fips_bundled_crypto_scan.py @@ -0,0 +1,88 @@ +"""The FIPS image bundled-crypto gate fails unreviewed or bundled crypto and passes a system-OpenSSL build.""" + +import _ssl +import importlib +import os +import shutil +import subprocess +from pathlib import Path +from typing import Final + +REPO_ROOT: Final = Path(__file__).resolve().parents[3] +SCANNER: Final = REPO_ROOT / "docker" / "scan_bundled_crypto.sh" +SOURCE_INVENTORY: Final = REPO_ROOT / "docker" / "fips_elf_inventory.txt" +CYGRPC_INVENTORY_PATH: Final = ( + "app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc.cpython-313-x86_64-linux-gnu.so" +) +CYGRPC_SO: Final = Path(str(importlib.import_module("grpc._cython.cygrpc").__file__)) + + +def _inventory_for(root: Path) -> Path: + entries: Final = tuple(f"{root}{line}" for line in SOURCE_INVENTORY.read_text().splitlines() if line.strip()) + inventory: Final = root.parent / "fips_elf_inventory.txt" + inventory.write_text("\n".join(entries) + "\n") + return inventory + + +def _run_scan(root: Path, report: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["sh", str(SCANNER), str(root)], + cwd=REPO_ROOT, + env={**os.environ, "SCAN_INVENTORY": str(_inventory_for(root)), "SCAN_REPORT": str(report)}, + capture_output=True, + text=True, + timeout=60, + ) + + +def _plant(root: Path, relative: str, source: Path) -> None: + target: Final = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(source, target) + + +def test_bundled_boringssl_grpcio_fails_the_fips_gate(tmp_path: Path) -> None: + root: Final = tmp_path / "image" + _plant(root, CYGRPC_INVENTORY_PATH, CYGRPC_SO) + result: Final = _run_scan(root, tmp_path / "scan-report.txt") + assert result.returncode == 1, result.stdout + result.stderr + violations: Final = tuple(line for line in result.stdout.splitlines() if line.startswith("VIOLATION ")) + assert any("cygrpc" in line and "BoringSSL" in line for line in violations), result.stdout + assert not any("cygrpc" in line for line in result.stdout.splitlines() if "WAIVED" in line), result.stdout + + +def test_system_openssl_linked_grpcio_at_its_inventory_path_is_clean(tmp_path: Path) -> None: + ssl_extension: Final = Path(_ssl.__file__) + headers: Final = subprocess.run( + ["objdump", "-p", str(ssl_extension)], capture_output=True, text=True, timeout=60, check=True + ).stdout + assert "libcrypto.so.3" in headers, ( + f"_ssl at {ssl_extension} is not linked against system libcrypto.so.3, " + f"so it cannot stand in for a system-OpenSSL cygrpc: {headers}" + ) + root: Final = tmp_path / "image" + _plant(root, CYGRPC_INVENTORY_PATH, ssl_extension) + report: Final = tmp_path / "scan-report.txt" + result: Final = _run_scan(root, report) + assert result.returncode == 0, result.stdout + result.stderr + assert "PASS: all 1 ELF binaries accounted for" in result.stdout, result.stdout + assert any(line.startswith("CLEAN ") and "cygrpc" in line for line in report.read_text().splitlines()), ( + report.read_text() + ) + + +def test_unreviewed_elf_outside_the_inventory_fails_the_gate(tmp_path: Path) -> None: + root: Final = tmp_path / "image" + _plant(root, "app/.venv/lib/python3.13/site-packages/somepkg/native.so", Path(_ssl.__file__)) + result: Final = _run_scan(root, tmp_path / "scan-report.txt") + assert result.returncode == 1, result.stdout + result.stderr + violations: Final = tuple(line for line in result.stdout.splitlines() if line.startswith("VIOLATION ")) + assert any("[unreviewed]" in line and "native.so" in line for line in violations), result.stdout + + +def test_empty_tree_passes(tmp_path: Path) -> None: + root: Final = tmp_path / "image" + root.mkdir() + result: Final = _run_scan(root, tmp_path / "scan-report.txt") + assert result.returncode == 0, result.stdout + result.stderr + assert "PASS: all 0 ELF binaries" in result.stdout, result.stdout diff --git a/tests/integration/configuration/test_proxy_without_grpc_stack.py b/tests/integration/configuration/test_proxy_without_grpc_stack.py new file mode 100644 index 00000000000..de2ae3ff3b6 --- /dev/null +++ b/tests/integration/configuration/test_proxy_without_grpc_stack.py @@ -0,0 +1,199 @@ +"""The proxy serves chat completions, messages and responses with grpcio, polars and google.cloud absent.""" + +import json +import os +import subprocess +import sys +import uuid +from collections.abc import Generator, Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Final + +import httpx +import pytest +from pydantic import JsonValue + +from tests.integration._support.client import JSON_OBJECT, gateway_from_environment, object_value +from tests.integration._support.process import OwnedProxy, owned_proxy_process +from tests.integration._support.upstream import delete_scenario, register_scenario +from tests.integration.cost_calculation.cost_tracking_case import JsonResponse, SseResponse + +SITECUSTOMIZE: Final = """ +import importlib.abc +import sys + +_BLOCKED_TOP_LEVEL = ("grpc", "polars") +_BLOCKED_GOOGLE = ("google.cloud", "google.api_core") + + +class _BlockedImports(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split(".", 1)[0] in _BLOCKED_TOP_LEVEL: + raise ModuleNotFoundError(f"blocked by the no-grpc-stack test: {fullname}") + if any(fullname == prefix or fullname.startswith(prefix + ".") for prefix in _BLOCKED_GOOGLE): + raise ModuleNotFoundError(f"blocked by the no-grpc-stack test: {fullname}") + return None + + +sys.meta_path.insert(0, _BlockedImports()) +""" + + +@contextmanager +def _python_without_grpc_stack(directory: Path) -> Generator[str, None, None]: + shim: Final = directory / "shim" + shim.mkdir() + (shim / "sitecustomize.py").write_text(SITECUSTOMIZE) + pythonpath: Final = f"{shim}{os.pathsep}{os.environ['PYTHONPATH']}" + probe: Final = subprocess.run( + [sys.executable, "-c", "import grpc"], + env={**os.environ, "PYTHONPATH": pythonpath}, + capture_output=True, + text=True, + timeout=60, + ) + output: Final = probe.stdout + probe.stderr + assert probe.returncode != 0 and "grpc" in output, ( + f"The import shim did not block grpc, so this test would prove nothing: {output}" + ) + yield pythonpath + + +@pytest.fixture(scope="module") +def proxy_without_grpc_stack(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]: + directory: Final = tmp_path_factory.mktemp("no-grpc-stack") + with _python_without_grpc_stack(directory) as pythonpath, gateway_from_environment() as gateway: + with owned_proxy_process(gateway, directory, {"PYTHONPATH": pythonpath}, workers=2) as owned: + yield owned + + +def test_chat_completions_serves_without_the_grpc_stack(proxy_without_grpc_stack: OwnedProxy) -> None: + owned: Final = proxy_without_grpc_stack + with ( + owned.gateway.scenario() as scenario, + httpx.Client(base_url=owned.gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model() + response: Final = owned.gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "no grpc stack"}]}, + ) + assert response.status_code == 200, response.text + body: Final = JSON_OBJECT.validate_json(response.content) + assert isinstance(body["id"], str) and body["id"].startswith("chatcmpl-"), response.text + observed: Final = upstream.get("/__observations") + observed.raise_for_status() + observed_requests: Final = JSON_OBJECT.validate_json(observed.content)["requests"] + assert isinstance(observed_requests, list) and len(observed_requests) == 1, observed_requests + received: Final = object_value(observed_requests[0]) + assert received["path"] == "/v1/chat/completions", observed_requests + received_body: Final = object_value(received["body"]) + assert received_body["model"] == "gpt-4o-mini", observed_requests + assert received_body["messages"] == [{"role": "user", "content": "no grpc stack"}], observed_requests + log: Final = owned.log.read_text() + leaked: Final = tuple( + line for line in log.splitlines() if "ModuleNotFoundError" in line and ("grpc" in line or "polars" in line) + ) + assert not leaked, f"Proxy logged a blocked import while serving without the grpc stack: {leaked}" + + +def test_streaming_chat_completion_serves_without_the_grpc_stack(proxy_without_grpc_stack: OwnedProxy) -> None: + owned: Final = proxy_without_grpc_stack + scenario_id: Final = f"nogrpcstream{uuid.uuid4().hex}" + chunk: Final[dict[str, JsonValue]] = { + "id": f"chatcmpl-{scenario_id}", + "object": "chat.completion.chunk", + "created": 1789788253, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": "streamed without grpc"}, "finish_reason": None} + ], + } + done: Final[dict[str, JsonValue]] = { + **chunk, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + } + handle: Final = register_scenario( + scenario_id, + SseResponse( + content_type="text/event-stream", + frames=(f"data: {json.dumps(chunk)}", f"data: {json.dumps(done)}", "data: [DONE]"), + ), + ) + try: + with owned.gateway.scenario() as scenario: + model: Final = scenario.model(api_base=f"{handle.api_base()}/v1") + response: Final = owned.gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "stream without grpc"}], "stream": True}, + ) + assert response.status_code == 200, response.text + chunks: Final = tuple( + JSON_OBJECT.validate_json(line.removeprefix("data: ").encode()) + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + assert chunks, response.text + assert chunks[-1]["id"] == f"chatcmpl-{scenario_id}", response.text + finally: + delete_scenario(handle) + + +def test_messages_and_responses_serve_without_the_grpc_stack(proxy_without_grpc_stack: OwnedProxy) -> None: + owned: Final = proxy_without_grpc_stack + scenario_id: Final = f"nogrpc{uuid.uuid4().hex}" + handle: Final = register_scenario( + scenario_id, + JsonResponse( + content_type="application/json", + body={ + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": "msg_upstream", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "served without grpc", "annotations": []}], + } + ], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + }, + ), + ) + try: + with owned.gateway.scenario() as scenario: + model: Final = scenario.model(api_base=handle.api_base()) + responses: Final = owned.gateway.request( + "POST", "/v1/responses", {"model": model, "input": "respond without grpc"} + ) + assert responses.status_code == 200, responses.text + responses_body: Final = JSON_OBJECT.validate_json(responses.content) + response_id: Final = responses_body["id"] + assert isinstance(response_id, str) and response_id.startswith("resp_"), responses.text + output: Final = responses_body["output"] + assert isinstance(output, list) and output, responses.text + output_content: Final = object_value(output[0])["content"] + assert isinstance(output_content, list) and output_content, responses.text + assert object_value(output_content[0])["text"] == "served without grpc", responses.text + messages: Final = owned.gateway.request( + "POST", + "/v1/messages", + {"model": model, "max_tokens": 64, "messages": [{"role": "user", "content": "respond without grpc"}]}, + ) + assert messages.status_code == 200, messages.text + payload: Final = JSON_OBJECT.validate_json(messages.content) + assert isinstance(payload["id"], str) and payload["id"], messages.text + content: Final = payload["content"] + assert isinstance(content, list) and len(content) == 1, messages.text + block: Final = object_value(content[0]) + assert block["type"] == "text" and block["text"] == "served without grpc", messages.text + finally: + delete_scenario(handle)