mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
feat(fips): build grpcio from source against system OpenSSL and drop its scanner waiver
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
53cb1576eb
commit
401c320c45
4 changed files with 324 additions and 1 deletions
|
|
@ -36,6 +36,35 @@ RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${
|
|||
make -j"$(nproc)" pgbouncer && \
|
||||
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
# grpcio's PyPI wheel statically bundles BoringSSL, which is not the validated
|
||||
# module. Rebuild it from source against the FIPS system OpenSSL instead
|
||||
# (GRPC_PYTHON_BUILD_SYSTEM_OPENSSL), so cygrpc links libcrypto.so.3 like
|
||||
# every other consumer in the image. The version is read from uv.lock through
|
||||
# its own tiny stage so this long compile is cached until grpcio itself moves.
|
||||
FROM $LITELLM_BUILD_IMAGE AS grpcio-version
|
||||
COPY uv.lock /uv.lock
|
||||
RUN awk '/^name = "grpcio"$/ { found = 1 } found && /^version = / { gsub(/"/, "", $3); print $3; exit }' /uv.lock \
|
||||
> /grpcio-version && test -s /grpcio-version
|
||||
|
||||
FROM $LITELLM_BUILD_IMAGE AS grpcio-builder
|
||||
USER root
|
||||
RUN --mount=type=secret,id=cgr-token \
|
||||
HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
|
||||
apk add --no-cache build-base python-3.13 python-3.13-dev openssl-dev zlib-dev binutils
|
||||
COPY --from=uvbin /uv /usr/local/bin/uv
|
||||
COPY --from=grpcio-version /grpcio-version /grpcio-version
|
||||
ENV UV_PYTHON_DOWNLOADS=0
|
||||
RUN uv venv --seed --python python3.13 /grpcio-venv && \
|
||||
GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
|
||||
GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 \
|
||||
GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS="$(nproc)" \
|
||||
/grpcio-venv/bin/python -m pip wheel --no-binary grpcio --no-deps --no-cache-dir \
|
||||
--wheel-dir /wheels "grpcio==$(cat /grpcio-version)" && \
|
||||
/grpcio-venv/bin/python -m zipfile -e /wheels/grpcio-*.whl /wheels/check && \
|
||||
objdump -p /wheels/check/grpc/_cython/cygrpc*.so | grep -q 'NEEDED.*libcrypto\.so\.3' && \
|
||||
! grep -aq BoringSSL /wheels/check/grpc/_cython/cygrpc*.so && \
|
||||
rm -rf /wheels/check
|
||||
|
||||
FROM --platform=$BUILDPLATFORM $UI_BUILD_IMAGE AS ui-builder
|
||||
|
||||
ENV NEXT_TELEMETRY_DISABLED=1 \
|
||||
|
|
@ -62,6 +91,7 @@ RUN --mount=type=secret,id=cgr-token \
|
|||
HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
|
||||
apk add --no-cache \
|
||||
bash \
|
||||
binutils \
|
||||
gcc \
|
||||
python-3.13 \
|
||||
python-3.13-dev \
|
||||
|
|
@ -107,6 +137,11 @@ RUN uv sync --frozen --no-default-groups --no-editable \
|
|||
--extra bedrock-realtime \
|
||||
--python python3.13
|
||||
|
||||
COPY --from=grpcio-builder /wheels /wheels
|
||||
RUN uv pip install --python /app/.venv/bin/python --no-deps --reinstall --no-cache /wheels/grpcio-*.whl && \
|
||||
strip --strip-unneeded /app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc*.so && \
|
||||
rm -rf /wheels
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
prisma generate --schema=./schema.prisma
|
||||
|
|
@ -155,6 +190,8 @@ RUN --mount=type=secret,id=cgr-token \
|
|||
HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
|
||||
apk add --no-cache binutils
|
||||
RUN openssl-fips-test
|
||||
RUN python -c "import grpc._cython.cygrpc" && \
|
||||
objdump -p /app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc*.so | grep -q 'NEEDED.*libcrypto\.so\.3'
|
||||
RUN chmod +x docker/scan_bundled_crypto.sh && \
|
||||
docker/scan_bundled_crypto.sh /app/.venv /opt/prisma /usr/local/bin
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,6 @@ crypto_markers() {
|
|||
is_waived() {
|
||||
case "$1" in
|
||||
*/cryptography/hazmat/bindings/_rust.abi3.so) ;;
|
||||
*/grpc/_cython/cygrpc*.so) ;;
|
||||
*/hf_xet/hf_xet.abi3.so) ;;
|
||||
*/_awscrt.abi3.so) ;;
|
||||
*/pyroscope/_native__lib*.so) ;;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
"""The FIPS image bundled-crypto gate fails unreviewed or bundled crypto and passes a system-OpenSSL build."""
|
||||
|
||||
import _ssl
|
||||
import importlib
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
REPO_ROOT: Final = Path(__file__).resolve().parents[3]
|
||||
SCANNER: Final = REPO_ROOT / "docker" / "scan_bundled_crypto.sh"
|
||||
SOURCE_INVENTORY: Final = REPO_ROOT / "docker" / "fips_elf_inventory.txt"
|
||||
CYGRPC_INVENTORY_PATH: Final = (
|
||||
"app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc.cpython-313-x86_64-linux-gnu.so"
|
||||
)
|
||||
CYGRPC_SO: Final = Path(str(importlib.import_module("grpc._cython.cygrpc").__file__))
|
||||
|
||||
|
||||
def _inventory_for(root: Path) -> Path:
|
||||
entries: Final = tuple(f"{root}{line}" for line in SOURCE_INVENTORY.read_text().splitlines() if line.strip())
|
||||
inventory: Final = root.parent / "fips_elf_inventory.txt"
|
||||
inventory.write_text("\n".join(entries) + "\n")
|
||||
return inventory
|
||||
|
||||
|
||||
def _run_scan(root: Path, report: Path) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["sh", str(SCANNER), str(root)],
|
||||
cwd=REPO_ROOT,
|
||||
env={**os.environ, "SCAN_INVENTORY": str(_inventory_for(root)), "SCAN_REPORT": str(report)},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
|
||||
def _plant(root: Path, relative: str, source: Path) -> None:
|
||||
target: Final = root / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy(source, target)
|
||||
|
||||
|
||||
def test_bundled_boringssl_grpcio_fails_the_fips_gate(tmp_path: Path) -> None:
|
||||
root: Final = tmp_path / "image"
|
||||
_plant(root, CYGRPC_INVENTORY_PATH, CYGRPC_SO)
|
||||
result: Final = _run_scan(root, tmp_path / "scan-report.txt")
|
||||
assert result.returncode == 1, result.stdout + result.stderr
|
||||
violations: Final = tuple(line for line in result.stdout.splitlines() if line.startswith("VIOLATION "))
|
||||
assert any("cygrpc" in line and "BoringSSL" in line for line in violations), result.stdout
|
||||
assert not any("cygrpc" in line for line in result.stdout.splitlines() if "WAIVED" in line), result.stdout
|
||||
|
||||
|
||||
def test_system_openssl_linked_grpcio_at_its_inventory_path_is_clean(tmp_path: Path) -> None:
|
||||
ssl_extension: Final = Path(_ssl.__file__)
|
||||
headers: Final = subprocess.run(
|
||||
["objdump", "-p", str(ssl_extension)], capture_output=True, text=True, timeout=60, check=True
|
||||
).stdout
|
||||
assert "libcrypto.so.3" in headers, (
|
||||
f"_ssl at {ssl_extension} is not linked against system libcrypto.so.3, "
|
||||
f"so it cannot stand in for a system-OpenSSL cygrpc: {headers}"
|
||||
)
|
||||
root: Final = tmp_path / "image"
|
||||
_plant(root, CYGRPC_INVENTORY_PATH, ssl_extension)
|
||||
report: Final = tmp_path / "scan-report.txt"
|
||||
result: Final = _run_scan(root, report)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert "PASS: all 1 ELF binaries accounted for" in result.stdout, result.stdout
|
||||
assert any(line.startswith("CLEAN ") and "cygrpc" in line for line in report.read_text().splitlines()), (
|
||||
report.read_text()
|
||||
)
|
||||
|
||||
|
||||
def test_unreviewed_elf_outside_the_inventory_fails_the_gate(tmp_path: Path) -> None:
|
||||
root: Final = tmp_path / "image"
|
||||
_plant(root, "app/.venv/lib/python3.13/site-packages/somepkg/native.so", Path(_ssl.__file__))
|
||||
result: Final = _run_scan(root, tmp_path / "scan-report.txt")
|
||||
assert result.returncode == 1, result.stdout + result.stderr
|
||||
violations: Final = tuple(line for line in result.stdout.splitlines() if line.startswith("VIOLATION "))
|
||||
assert any("[unreviewed]" in line and "native.so" in line for line in violations), result.stdout
|
||||
|
||||
|
||||
def test_empty_tree_passes(tmp_path: Path) -> None:
|
||||
root: Final = tmp_path / "image"
|
||||
root.mkdir()
|
||||
result: Final = _run_scan(root, tmp_path / "scan-report.txt")
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert "PASS: all 0 ELF binaries" in result.stdout, result.stdout
|
||||
199
tests/integration/configuration/test_proxy_without_grpc_stack.py
Normal file
199
tests/integration/configuration/test_proxy_without_grpc_stack.py
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
"""The proxy serves chat completions, messages and responses with grpcio, polars and google.cloud absent."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from collections.abc import Generator, Iterator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
|
||||
from tests.integration._support.client import JSON_OBJECT, gateway_from_environment, object_value
|
||||
from tests.integration._support.process import OwnedProxy, owned_proxy_process
|
||||
from tests.integration._support.upstream import delete_scenario, register_scenario
|
||||
from tests.integration.cost_calculation.cost_tracking_case import JsonResponse, SseResponse
|
||||
|
||||
SITECUSTOMIZE: Final = """
|
||||
import importlib.abc
|
||||
import sys
|
||||
|
||||
_BLOCKED_TOP_LEVEL = ("grpc", "polars")
|
||||
_BLOCKED_GOOGLE = ("google.cloud", "google.api_core")
|
||||
|
||||
|
||||
class _BlockedImports(importlib.abc.MetaPathFinder):
|
||||
def find_spec(self, fullname, path=None, target=None):
|
||||
if fullname.split(".", 1)[0] in _BLOCKED_TOP_LEVEL:
|
||||
raise ModuleNotFoundError(f"blocked by the no-grpc-stack test: {fullname}")
|
||||
if any(fullname == prefix or fullname.startswith(prefix + ".") for prefix in _BLOCKED_GOOGLE):
|
||||
raise ModuleNotFoundError(f"blocked by the no-grpc-stack test: {fullname}")
|
||||
return None
|
||||
|
||||
|
||||
sys.meta_path.insert(0, _BlockedImports())
|
||||
"""
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _python_without_grpc_stack(directory: Path) -> Generator[str, None, None]:
|
||||
shim: Final = directory / "shim"
|
||||
shim.mkdir()
|
||||
(shim / "sitecustomize.py").write_text(SITECUSTOMIZE)
|
||||
pythonpath: Final = f"{shim}{os.pathsep}{os.environ['PYTHONPATH']}"
|
||||
probe: Final = subprocess.run(
|
||||
[sys.executable, "-c", "import grpc"],
|
||||
env={**os.environ, "PYTHONPATH": pythonpath},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
output: Final = probe.stdout + probe.stderr
|
||||
assert probe.returncode != 0 and "grpc" in output, (
|
||||
f"The import shim did not block grpc, so this test would prove nothing: {output}"
|
||||
)
|
||||
yield pythonpath
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def proxy_without_grpc_stack(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
|
||||
directory: Final = tmp_path_factory.mktemp("no-grpc-stack")
|
||||
with _python_without_grpc_stack(directory) as pythonpath, gateway_from_environment() as gateway:
|
||||
with owned_proxy_process(gateway, directory, {"PYTHONPATH": pythonpath}, workers=2) as owned:
|
||||
yield owned
|
||||
|
||||
|
||||
def test_chat_completions_serves_without_the_grpc_stack(proxy_without_grpc_stack: OwnedProxy) -> None:
|
||||
owned: Final = proxy_without_grpc_stack
|
||||
with (
|
||||
owned.gateway.scenario() as scenario,
|
||||
httpx.Client(base_url=owned.gateway.upstream_url, timeout=5, trust_env=False) as upstream,
|
||||
):
|
||||
model: Final = scenario.model()
|
||||
response: Final = owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": "no grpc stack"}]},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = JSON_OBJECT.validate_json(response.content)
|
||||
assert isinstance(body["id"], str) and body["id"].startswith("chatcmpl-"), response.text
|
||||
observed: Final = upstream.get("/__observations")
|
||||
observed.raise_for_status()
|
||||
observed_requests: Final = JSON_OBJECT.validate_json(observed.content)["requests"]
|
||||
assert isinstance(observed_requests, list) and len(observed_requests) == 1, observed_requests
|
||||
received: Final = object_value(observed_requests[0])
|
||||
assert received["path"] == "/v1/chat/completions", observed_requests
|
||||
received_body: Final = object_value(received["body"])
|
||||
assert received_body["model"] == "gpt-4o-mini", observed_requests
|
||||
assert received_body["messages"] == [{"role": "user", "content": "no grpc stack"}], observed_requests
|
||||
log: Final = owned.log.read_text()
|
||||
leaked: Final = tuple(
|
||||
line for line in log.splitlines() if "ModuleNotFoundError" in line and ("grpc" in line or "polars" in line)
|
||||
)
|
||||
assert not leaked, f"Proxy logged a blocked import while serving without the grpc stack: {leaked}"
|
||||
|
||||
|
||||
def test_streaming_chat_completion_serves_without_the_grpc_stack(proxy_without_grpc_stack: OwnedProxy) -> None:
|
||||
owned: Final = proxy_without_grpc_stack
|
||||
scenario_id: Final = f"nogrpcstream{uuid.uuid4().hex}"
|
||||
chunk: Final[dict[str, JsonValue]] = {
|
||||
"id": f"chatcmpl-{scenario_id}",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1789788253,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{"index": 0, "delta": {"role": "assistant", "content": "streamed without grpc"}, "finish_reason": None}
|
||||
],
|
||||
}
|
||||
done: Final[dict[str, JsonValue]] = {
|
||||
**chunk,
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
}
|
||||
handle: Final = register_scenario(
|
||||
scenario_id,
|
||||
SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=(f"data: {json.dumps(chunk)}", f"data: {json.dumps(done)}", "data: [DONE]"),
|
||||
),
|
||||
)
|
||||
try:
|
||||
with owned.gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=f"{handle.api_base()}/v1")
|
||||
response: Final = owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": "stream without grpc"}], "stream": True},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
chunks: Final = tuple(
|
||||
JSON_OBJECT.validate_json(line.removeprefix("data: ").encode())
|
||||
for line in response.text.splitlines()
|
||||
if line.startswith("data: ") and line != "data: [DONE]"
|
||||
)
|
||||
assert chunks, response.text
|
||||
assert chunks[-1]["id"] == f"chatcmpl-{scenario_id}", response.text
|
||||
finally:
|
||||
delete_scenario(handle)
|
||||
|
||||
|
||||
def test_messages_and_responses_serve_without_the_grpc_stack(proxy_without_grpc_stack: OwnedProxy) -> None:
|
||||
owned: Final = proxy_without_grpc_stack
|
||||
scenario_id: Final = f"nogrpc{uuid.uuid4().hex}"
|
||||
handle: Final = register_scenario(
|
||||
scenario_id,
|
||||
JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "resp_$REQUEST_ID",
|
||||
"object": "response",
|
||||
"created_at": 1789788253,
|
||||
"status": "completed",
|
||||
"model": "gpt-4o-mini",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_upstream",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "served without grpc", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
},
|
||||
),
|
||||
)
|
||||
try:
|
||||
with owned.gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=handle.api_base())
|
||||
responses: Final = owned.gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": "respond without grpc"}
|
||||
)
|
||||
assert responses.status_code == 200, responses.text
|
||||
responses_body: Final = JSON_OBJECT.validate_json(responses.content)
|
||||
response_id: Final = responses_body["id"]
|
||||
assert isinstance(response_id, str) and response_id.startswith("resp_"), responses.text
|
||||
output: Final = responses_body["output"]
|
||||
assert isinstance(output, list) and output, responses.text
|
||||
output_content: Final = object_value(output[0])["content"]
|
||||
assert isinstance(output_content, list) and output_content, responses.text
|
||||
assert object_value(output_content[0])["text"] == "served without grpc", responses.text
|
||||
messages: Final = owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{"model": model, "max_tokens": 64, "messages": [{"role": "user", "content": "respond without grpc"}]},
|
||||
)
|
||||
assert messages.status_code == 200, messages.text
|
||||
payload: Final = JSON_OBJECT.validate_json(messages.content)
|
||||
assert isinstance(payload["id"], str) and payload["id"], messages.text
|
||||
content: Final = payload["content"]
|
||||
assert isinstance(content, list) and len(content) == 1, messages.text
|
||||
block: Final = object_value(content[0])
|
||||
assert block["type"] == "text" and block["text"] == "served without grpc", messages.text
|
||||
finally:
|
||||
delete_scenario(handle)
|
||||
Loading…
Add table
Reference in a new issue