test(docker): gate the componentized gateway and backend images on an arbitrary-uid offline boot (#36136)

.github/ci-coverage-allowlist.yml

@@ -136,13 +136,6 @@ test_paths:
       - tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
 
 dockerfiles:
-  - reason: >-
-      The componentized images the microservices chart deploys are built by no job; wiring both into
-      the scan workflow costs a full image build each and is deferred to a change that prices the
-      whole set
-    paths:
-      - backend/Dockerfile
-      - gateway/Dockerfile
   - reason: >-
       The dashboard container is a static Next.js export served by nginx, and the dashboard build
       and lint workflows already exercise that output, so building the image adds no signal about it

.github/workflows/image-scan.yml

@@ -12,6 +12,11 @@ on:
       - docker/Dockerfile.non_root
       - migrations/Dockerfile
       - migrations/run.py
+      - gateway/Dockerfile
+      - gateway/main.py
+      - backend/Dockerfile
+      - backend/main.py
+      - docker/component_entrypoint.sh
       - litellm-proxy-extras/**
       - tests/proxy_migration_tests/**
       - uv.lock
@@ -147,3 +152,63 @@ jobs:
         run: |
           python -m pip install "pytest==9.0.3"
           python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+
+  gateway-image:
+    name: gateway-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build gateway image
+        run: docker build -f gateway/Dockerfile -t litellm-gateway-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the gateway serves offline as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-gateway-scan:${{ github.sha }}
+          LITELLM_COMPONENT_PORT: "4000"
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
+
+  backend-image:
+    name: backend-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build backend image
+        run: docker build -f backend/Dockerfile -t litellm-backend-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the backend serves offline as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-backend-scan:${{ github.sha }}
+          LITELLM_COMPONENT_PORT: "4001"
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v

tests/proxy_migration_tests/test_component_image_serves_offline.py

@@ -0,0 +1,187 @@
+"""Image-level regression net for the prisma bake in the componentized images.
+
+The gateway and backend serve requests; they never shell out to the Prisma CLI
+(``PrismaManager.setup_database`` is reachable only from ``proxy_cli.py``, which
+uvicorn'ing ``gateway.main:app`` bypasses). What they do need is the generated
+client's baked query engine, and prisma-python resolves those baked paths
+eagerly, with an existence check that propagates EACCES rather than skipping the
+candidate. An engine baked under a build-time ``HOME`` is therefore unreadable to
+any other runtime uid, and the process dies during startup before
+``PRISMA_QUERY_ENGINE_BINARY`` is ever consulted.
+
+That is what an OpenShift ``restricted-v2`` namespace produces: the image
+``USER`` is ignored and an arbitrary uid in GID 0 is assigned instead. The
+symptom is not a degraded proxy, it is a proxy that does not serve at all.
+
+Booting the image the way that deployment does, and requiring it to answer a
+request with a live database connection, is what catches the whole class:
+a boot as the default uid, or one that reaches the internet, passes even when
+the bake is unusable everywhere it actually ships.
+
+Gated on LITELLM_IMAGE (the tag of the image to exercise) so it is skipped in
+the normal unit-test run and exercised only where an image has been built (the
+image-scan workflow). Requires a working docker CLI.
+"""
+
+import json
+import shutil
+import subprocess
+import time
+import uuid
+
+import os
+import pytest
+
+IMAGE = os.getenv("LITELLM_IMAGE")
+POSTGRES_IMAGE = os.getenv("LITELLM_TEST_POSTGRES_IMAGE", "postgres:16-alpine")
+CURL_IMAGE = os.getenv("LITELLM_TEST_CURL_IMAGE", "curlimages/curl:8.11.1")
+COMPONENT_PORT = os.getenv("LITELLM_COMPONENT_PORT", "4000")
+NON_ROOT_UID = "12345:0"
+STARTUP_TIMEOUT_SECONDS = int(os.getenv("LITELLM_COMPONENT_STARTUP_TIMEOUT", "180"))
+
+pytestmark = [
+    pytest.mark.skipif(IMAGE is None, reason="requires a built image (set LITELLM_IMAGE)"),
+    pytest.mark.skipif(shutil.which("docker") is None, reason="requires the docker CLI"),
+]
+
+
+def _docker(*args: str, check: bool = True) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        ["docker", *args], capture_output=True, text=True, check=check
+    )
+
+
+@pytest.fixture()
+def offline_stack():
+    """A component container and a fresh Postgres on a network with no egress.
+
+    NON_ROOT_UID is an arbitrary uid in GID 0, the shape OpenShift restricted-v2
+    assigns. Postgres and curl are pulled while egress still exists, because the
+    ``--internal`` network below has none: that is what makes a prisma engine
+    download (binaries.prisma.sh / npm) fail rather than mask a bake that is not
+    self-contained.
+
+    The container runs with DISABLE_SCHEMA_UPDATE, since applying the schema is
+    the migration job's responsibility in this topology and needs the Prisma CLI
+    these images deliberately omit, and with LITELLM_LOCAL_MODEL_COST_MAP, or the
+    proxy spends the whole startup budget timing out on a cost-map fetch over the
+    network it does not have.
+
+    Yields (network_name, component_container). Both are torn down afterwards.
+    """
+    run_id = f"componentserve-{uuid.uuid4().hex[:8]}"
+    network = f"{run_id}-net"
+    pg = f"{run_id}-pg"
+    component = f"{run_id}-app"
+
+    _docker("pull", "--quiet", POSTGRES_IMAGE)
+    _docker("pull", "--quiet", CURL_IMAGE)
+    _docker("network", "create", "--internal", network)
+    try:
+        _docker(
+            "run", "-d", "--name", pg, "--network", network,
+            "-e", "POSTGRES_PASSWORD=pw", "-e", "POSTGRES_DB=litellm",
+            POSTGRES_IMAGE,
+        )
+        _wait_until_postgres_ready(pg)
+        assert IMAGE is not None
+        _docker(
+            "run", "-d", "--name", component, "--network", network,
+            "--user", NON_ROOT_UID,
+            "-e", f"DATABASE_URL=postgresql://postgres:pw@{pg}:5432/litellm",
+            "-e", "LITELLM_MASTER_KEY=sk-component-serve-test",
+            "-e", "DISABLE_SCHEMA_UPDATE=true",
+            "-e", "LITELLM_LOCAL_MODEL_COST_MAP=True",
+            IMAGE,
+        )
+        yield network, component
+    finally:
+        _docker("logs", component, check=False)
+        _docker("rm", "-f", component, check=False)
+        _docker("rm", "-f", pg, check=False)
+        _docker("network", "rm", network, check=False)
+
+
+def _wait_until_postgres_ready(pg: str, attempts: int = 60) -> None:
+    for _ in range(attempts):
+        running = _docker(
+            "ps", "--filter", f"name={pg}", "--filter", "status=running",
+            "--format", "{{.Names}}", check=False,
+        ).stdout
+        if pg not in running:
+            logs = _docker("logs", pg, check=False)
+            pytest.fail(f"postgres container is not running:\n{logs.stdout}\n{logs.stderr}")
+        ready = _docker(
+            "exec", pg, "pg_isready", "-U", "postgres", "-d", "litellm", check=False
+        )
+        if ready.returncode == 0:
+            return
+        time.sleep(1)
+    pytest.fail(f"postgres never became ready after {attempts}s")
+
+
+def _container_logs(container: str) -> str:
+    logs = _docker("logs", container, check=False)
+    return f"stdout:\n{logs.stdout}\nstderr:\n{logs.stderr}"
+
+
+def _is_running(container: str) -> bool:
+    return bool(
+        _docker(
+            "ps", "--filter", f"name={container}", "--filter", "status=running",
+            "--format", "{{.Names}}", check=False,
+        ).stdout.strip()
+    )
+
+
+def _readiness(network: str, component: str) -> subprocess.CompletedProcess:
+    """Ask the component for its readiness, from a peer on the same egress-less network."""
+    return _docker(
+        "run", "--rm", "--network", network, CURL_IMAGE,
+        "--silent", "--max-time", "10",
+        f"http://{component}:{COMPONENT_PORT}/health/readiness",
+        check=False,
+    )
+
+
+def test_component_serves_offline_as_non_root_uid(offline_stack):
+    """The component answers a request with a live DB connection, offline, as an arbitrary uid.
+
+    On the pre-fix image this never gets a response: the engine baked under
+    /home/nonroot (mode 0700, owned by uid 65532) raises
+    ``PermissionError: .../query-engine-linux-...`` out of pathlib and uvicorn
+    reports ``Application startup failed. Exiting.``. A bake at the fixed,
+    world-readable /opt/prisma is what lets any uid start the client.
+
+    `db: connected` is the load-bearing part of the assertion: it means the
+    query engine binary was found, executed, and reached Postgres. A liveness
+    probe alone would pass on an image whose engine never resolved.
+    """
+    network, component = offline_stack
+
+    deadline = time.time() + STARTUP_TIMEOUT_SECONDS
+    probe = None
+    while time.time() < deadline:
+        if not _is_running(component):
+            pytest.fail(
+                f"the component exited during startup as uid {NON_ROOT_UID} with no egress. "
+                "The prisma bake is not readable to a uid other than the one that built it, "
+                "so the proxy does not serve at all.\n"
+                f"{_container_logs(component)}"
+            )
+        probe = _readiness(network, component)
+        if probe.returncode == 0 and probe.stdout.strip():
+            break
+        time.sleep(2)
+
+    assert probe is not None and probe.returncode == 0 and probe.stdout.strip(), (
+        f"/health/readiness never answered within {STARTUP_TIMEOUT_SECONDS}s as uid "
+        f"{NON_ROOT_UID} with no egress.\n{_container_logs(component)}"
+    )
+
+    payload = json.loads(probe.stdout)
+    assert payload.get("db") == "connected", (
+        f"the component answered but its database is {payload.get('db')!r}, so the baked "
+        f"query engine did not resolve as uid {NON_ROOT_UID}.\nresponse: {probe.stdout}\n"
+        f"{_container_logs(component)}"
+    )