feat(fips): build grpcio from source against system OpenSSL and drop its scanner waiver

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-24 08:14:22 +00:00
parent 53cb1576eb
commit 401c320c45
4 changed files with 324 additions and 1 deletions

View file

@ -36,6 +36,35 @@ RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${
make -j"$(nproc)" pgbouncer && \
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
# grpcio's PyPI wheel statically bundles BoringSSL, which is not the validated
# module. Rebuild it from source against the FIPS system OpenSSL instead
# (GRPC_PYTHON_BUILD_SYSTEM_OPENSSL), so cygrpc links libcrypto.so.3 like
# every other consumer in the image. The version is read from uv.lock through
# its own tiny stage so this long compile is cached until grpcio itself moves.
FROM $LITELLM_BUILD_IMAGE AS grpcio-version
COPY uv.lock /uv.lock
RUN awk '/^name = "grpcio"$/ { found = 1 } found && /^version = / { gsub(/"/, "", $3); print $3; exit }' /uv.lock \
> /grpcio-version && test -s /grpcio-version
FROM $LITELLM_BUILD_IMAGE AS grpcio-builder
USER root
RUN --mount=type=secret,id=cgr-token \
HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
apk add --no-cache build-base python-3.13 python-3.13-dev openssl-dev zlib-dev binutils
COPY --from=uvbin /uv /usr/local/bin/uv
COPY --from=grpcio-version /grpcio-version /grpcio-version
ENV UV_PYTHON_DOWNLOADS=0
RUN uv venv --seed --python python3.13 /grpcio-venv && \
GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 \
GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS="$(nproc)" \
/grpcio-venv/bin/python -m pip wheel --no-binary grpcio --no-deps --no-cache-dir \
--wheel-dir /wheels "grpcio==$(cat /grpcio-version)" && \
/grpcio-venv/bin/python -m zipfile -e /wheels/grpcio-*.whl /wheels/check && \
objdump -p /wheels/check/grpc/_cython/cygrpc*.so | grep -q 'NEEDED.*libcrypto\.so\.3' && \
! grep -aq BoringSSL /wheels/check/grpc/_cython/cygrpc*.so && \
rm -rf /wheels/check
FROM --platform=$BUILDPLATFORM $UI_BUILD_IMAGE AS ui-builder
ENV NEXT_TELEMETRY_DISABLED=1 \
@ -62,6 +91,7 @@ RUN --mount=type=secret,id=cgr-token \
HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
apk add --no-cache \
bash \
binutils \
gcc \
python-3.13 \
python-3.13-dev \
@ -107,6 +137,11 @@ RUN uv sync --frozen --no-default-groups --no-editable \
--extra bedrock-realtime \
--python python3.13
COPY --from=grpcio-builder /wheels /wheels
RUN uv pip install --python /app/.venv/bin/python --no-deps --reinstall --no-cache /wheels/grpcio-*.whl && \
strip --strip-unneeded /app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc*.so && \
rm -rf /wheels
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
@ -155,6 +190,8 @@ RUN --mount=type=secret,id=cgr-token \
HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
apk add --no-cache binutils
RUN openssl-fips-test
RUN python -c "import grpc._cython.cygrpc" && \
objdump -p /app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc*.so | grep -q 'NEEDED.*libcrypto\.so\.3'
RUN chmod +x docker/scan_bundled_crypto.sh && \
docker/scan_bundled_crypto.sh /app/.venv /opt/prisma /usr/local/bin

View file

@ -51,7 +51,6 @@ crypto_markers() {
is_waived() {
case "$1" in
*/cryptography/hazmat/bindings/_rust.abi3.so) ;;
*/grpc/_cython/cygrpc*.so) ;;
*/hf_xet/hf_xet.abi3.so) ;;
*/_awscrt.abi3.so) ;;
*/pyroscope/_native__lib*.so) ;;

View file

@ -0,0 +1,88 @@
"""The FIPS image bundled-crypto gate fails unreviewed or bundled crypto and passes a system-OpenSSL build."""
import _ssl
import importlib
import os
import shutil
import subprocess
from pathlib import Path
from typing import Final
REPO_ROOT: Final = Path(__file__).resolve().parents[3]
SCANNER: Final = REPO_ROOT / "docker" / "scan_bundled_crypto.sh"
SOURCE_INVENTORY: Final = REPO_ROOT / "docker" / "fips_elf_inventory.txt"
CYGRPC_INVENTORY_PATH: Final = (
"app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc.cpython-313-x86_64-linux-gnu.so"
)
CYGRPC_SO: Final = Path(str(importlib.import_module("grpc._cython.cygrpc").__file__))
def _inventory_for(root: Path) -> Path:
entries: Final = tuple(f"{root}{line}" for line in SOURCE_INVENTORY.read_text().splitlines() if line.strip())
inventory: Final = root.parent / "fips_elf_inventory.txt"
inventory.write_text("\n".join(entries) + "\n")
return inventory
def _run_scan(root: Path, report: Path) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["sh", str(SCANNER), str(root)],
cwd=REPO_ROOT,
env={**os.environ, "SCAN_INVENTORY": str(_inventory_for(root)), "SCAN_REPORT": str(report)},
capture_output=True,
text=True,
timeout=60,
)
def _plant(root: Path, relative: str, source: Path) -> None:
target: Final = root / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(source, target)
def test_bundled_boringssl_grpcio_fails_the_fips_gate(tmp_path: Path) -> None:
root: Final = tmp_path / "image"
_plant(root, CYGRPC_INVENTORY_PATH, CYGRPC_SO)
result: Final = _run_scan(root, tmp_path / "scan-report.txt")
assert result.returncode == 1, result.stdout + result.stderr
violations: Final = tuple(line for line in result.stdout.splitlines() if line.startswith("VIOLATION "))
assert any("cygrpc" in line and "BoringSSL" in line for line in violations), result.stdout
assert not any("cygrpc" in line for line in result.stdout.splitlines() if "WAIVED" in line), result.stdout
def test_system_openssl_linked_grpcio_at_its_inventory_path_is_clean(tmp_path: Path) -> None:
ssl_extension: Final = Path(_ssl.__file__)
headers: Final = subprocess.run(
["objdump", "-p", str(ssl_extension)], capture_output=True, text=True, timeout=60, check=True
).stdout
assert "libcrypto.so.3" in headers, (
f"_ssl at {ssl_extension} is not linked against system libcrypto.so.3, "
f"so it cannot stand in for a system-OpenSSL cygrpc: {headers}"
)
root: Final = tmp_path / "image"
_plant(root, CYGRPC_INVENTORY_PATH, ssl_extension)
report: Final = tmp_path / "scan-report.txt"
result: Final = _run_scan(root, report)
assert result.returncode == 0, result.stdout + result.stderr
assert "PASS: all 1 ELF binaries accounted for" in result.stdout, result.stdout
assert any(line.startswith("CLEAN ") and "cygrpc" in line for line in report.read_text().splitlines()), (
report.read_text()
)
def test_unreviewed_elf_outside_the_inventory_fails_the_gate(tmp_path: Path) -> None:
root: Final = tmp_path / "image"
_plant(root, "app/.venv/lib/python3.13/site-packages/somepkg/native.so", Path(_ssl.__file__))
result: Final = _run_scan(root, tmp_path / "scan-report.txt")
assert result.returncode == 1, result.stdout + result.stderr
violations: Final = tuple(line for line in result.stdout.splitlines() if line.startswith("VIOLATION "))
assert any("[unreviewed]" in line and "native.so" in line for line in violations), result.stdout
def test_empty_tree_passes(tmp_path: Path) -> None:
root: Final = tmp_path / "image"
root.mkdir()
result: Final = _run_scan(root, tmp_path / "scan-report.txt")
assert result.returncode == 0, result.stdout + result.stderr
assert "PASS: all 0 ELF binaries" in result.stdout, result.stdout

View file

@ -0,0 +1,199 @@
"""The proxy serves chat completions, messages and responses with grpcio, polars and google.cloud absent."""
import json
import os
import subprocess
import sys
import uuid
from collections.abc import Generator, Iterator
from contextlib import contextmanager
from pathlib import Path
from typing import Final
import httpx
import pytest
from pydantic import JsonValue
from tests.integration._support.client import JSON_OBJECT, gateway_from_environment, object_value
from tests.integration._support.process import OwnedProxy, owned_proxy_process
from tests.integration._support.upstream import delete_scenario, register_scenario
from tests.integration.cost_calculation.cost_tracking_case import JsonResponse, SseResponse
SITECUSTOMIZE: Final = """
import importlib.abc
import sys
_BLOCKED_TOP_LEVEL = ("grpc", "polars")
_BLOCKED_GOOGLE = ("google.cloud", "google.api_core")
class _BlockedImports(importlib.abc.MetaPathFinder):
def find_spec(self, fullname, path=None, target=None):
if fullname.split(".", 1)[0] in _BLOCKED_TOP_LEVEL:
raise ModuleNotFoundError(f"blocked by the no-grpc-stack test: {fullname}")
if any(fullname == prefix or fullname.startswith(prefix + ".") for prefix in _BLOCKED_GOOGLE):
raise ModuleNotFoundError(f"blocked by the no-grpc-stack test: {fullname}")
return None
sys.meta_path.insert(0, _BlockedImports())
"""
@contextmanager
def _python_without_grpc_stack(directory: Path) -> Generator[str, None, None]:
shim: Final = directory / "shim"
shim.mkdir()
(shim / "sitecustomize.py").write_text(SITECUSTOMIZE)
pythonpath: Final = f"{shim}{os.pathsep}{os.environ['PYTHONPATH']}"
probe: Final = subprocess.run(
[sys.executable, "-c", "import grpc"],
env={**os.environ, "PYTHONPATH": pythonpath},
capture_output=True,
text=True,
timeout=60,
)
output: Final = probe.stdout + probe.stderr
assert probe.returncode != 0 and "grpc" in output, (
f"The import shim did not block grpc, so this test would prove nothing: {output}"
)
yield pythonpath
@pytest.fixture(scope="module")
def proxy_without_grpc_stack(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
directory: Final = tmp_path_factory.mktemp("no-grpc-stack")
with _python_without_grpc_stack(directory) as pythonpath, gateway_from_environment() as gateway:
with owned_proxy_process(gateway, directory, {"PYTHONPATH": pythonpath}, workers=2) as owned:
yield owned
def test_chat_completions_serves_without_the_grpc_stack(proxy_without_grpc_stack: OwnedProxy) -> None:
owned: Final = proxy_without_grpc_stack
with (
owned.gateway.scenario() as scenario,
httpx.Client(base_url=owned.gateway.upstream_url, timeout=5, trust_env=False) as upstream,
):
model: Final = scenario.model()
response: Final = owned.gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": "no grpc stack"}]},
)
assert response.status_code == 200, response.text
body: Final = JSON_OBJECT.validate_json(response.content)
assert isinstance(body["id"], str) and body["id"].startswith("chatcmpl-"), response.text
observed: Final = upstream.get("/__observations")
observed.raise_for_status()
observed_requests: Final = JSON_OBJECT.validate_json(observed.content)["requests"]
assert isinstance(observed_requests, list) and len(observed_requests) == 1, observed_requests
received: Final = object_value(observed_requests[0])
assert received["path"] == "/v1/chat/completions", observed_requests
received_body: Final = object_value(received["body"])
assert received_body["model"] == "gpt-4o-mini", observed_requests
assert received_body["messages"] == [{"role": "user", "content": "no grpc stack"}], observed_requests
log: Final = owned.log.read_text()
leaked: Final = tuple(
line for line in log.splitlines() if "ModuleNotFoundError" in line and ("grpc" in line or "polars" in line)
)
assert not leaked, f"Proxy logged a blocked import while serving without the grpc stack: {leaked}"
def test_streaming_chat_completion_serves_without_the_grpc_stack(proxy_without_grpc_stack: OwnedProxy) -> None:
owned: Final = proxy_without_grpc_stack
scenario_id: Final = f"nogrpcstream{uuid.uuid4().hex}"
chunk: Final[dict[str, JsonValue]] = {
"id": f"chatcmpl-{scenario_id}",
"object": "chat.completion.chunk",
"created": 1789788253,
"model": "gpt-4o-mini",
"choices": [
{"index": 0, "delta": {"role": "assistant", "content": "streamed without grpc"}, "finish_reason": None}
],
}
done: Final[dict[str, JsonValue]] = {
**chunk,
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
}
handle: Final = register_scenario(
scenario_id,
SseResponse(
content_type="text/event-stream",
frames=(f"data: {json.dumps(chunk)}", f"data: {json.dumps(done)}", "data: [DONE]"),
),
)
try:
with owned.gateway.scenario() as scenario:
model: Final = scenario.model(api_base=f"{handle.api_base()}/v1")
response: Final = owned.gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": "stream without grpc"}], "stream": True},
)
assert response.status_code == 200, response.text
chunks: Final = tuple(
JSON_OBJECT.validate_json(line.removeprefix("data: ").encode())
for line in response.text.splitlines()
if line.startswith("data: ") and line != "data: [DONE]"
)
assert chunks, response.text
assert chunks[-1]["id"] == f"chatcmpl-{scenario_id}", response.text
finally:
delete_scenario(handle)
def test_messages_and_responses_serve_without_the_grpc_stack(proxy_without_grpc_stack: OwnedProxy) -> None:
owned: Final = proxy_without_grpc_stack
scenario_id: Final = f"nogrpc{uuid.uuid4().hex}"
handle: Final = register_scenario(
scenario_id,
JsonResponse(
content_type="application/json",
body={
"id": "resp_$REQUEST_ID",
"object": "response",
"created_at": 1789788253,
"status": "completed",
"model": "gpt-4o-mini",
"output": [
{
"type": "message",
"id": "msg_upstream",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "served without grpc", "annotations": []}],
}
],
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
},
),
)
try:
with owned.gateway.scenario() as scenario:
model: Final = scenario.model(api_base=handle.api_base())
responses: Final = owned.gateway.request(
"POST", "/v1/responses", {"model": model, "input": "respond without grpc"}
)
assert responses.status_code == 200, responses.text
responses_body: Final = JSON_OBJECT.validate_json(responses.content)
response_id: Final = responses_body["id"]
assert isinstance(response_id, str) and response_id.startswith("resp_"), responses.text
output: Final = responses_body["output"]
assert isinstance(output, list) and output, responses.text
output_content: Final = object_value(output[0])["content"]
assert isinstance(output_content, list) and output_content, responses.text
assert object_value(output_content[0])["text"] == "served without grpc", responses.text
messages: Final = owned.gateway.request(
"POST",
"/v1/messages",
{"model": model, "max_tokens": 64, "messages": [{"role": "user", "content": "respond without grpc"}]},
)
assert messages.status_code == 200, messages.text
payload: Final = JSON_OBJECT.validate_json(messages.content)
assert isinstance(payload["id"], str) and payload["id"], messages.text
content: Final = payload["content"]
assert isinstance(content, list) and len(content) == 1, messages.text
block: Final = object_value(content[0])
assert block["type"] == "text" and block["text"] == "served without grpc", messages.text
finally:
delete_scenario(handle)