fix(ui): boot the UI image as an arbitrary uid by anchoring nginx writes under /tmp (#37982)

* fix(ui): boot the UI image as an arbitrary uid by anchoring nginx writes under /tmp

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(ui): type the arbitrary-uid image test fixture

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.github/workflows/image-scan.yml

@@ -23,6 +23,8 @@ on:
       - tests/proxy_migration_tests/**
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
+      - ui/Dockerfile
+      - ui/nginx.conf
       - .github/workflows/image-scan.yml
   schedule:
     - cron: "41 6 * * *"
@@ -185,6 +187,35 @@ jobs:
           python -m pip install "pytest==9.0.3"
           python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
 
+  ui-image:
+    name: ui-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build UI image
+        run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
+        env:
+          LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
+
   backend-image:
     name: backend-image
     runs-on: ubuntu-latest

helm/litellm/values.yaml

@@ -428,9 +428,11 @@ ui:
     maxUnavailable: ""
   podAnnotations: {}
   # Same shape as the gateway blocks of the same name. The nginx runtime
-  # writes its pid, cache, and proxy temp files under the image's root
-  # filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs
-  # emptyDir volumes mounted over those paths.
+  # writes its pid, cache, and proxy temp files under /tmp, so it boots as
+  # any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem:
+  # true` here needs an emptyDir volume mounted over /tmp. Images before
+  # the /tmp move instead need emptyDirs over /var/cache/nginx and /run to
+  # run as a non-root uid at all.
   podLabels: {}
   podSecurityContext: {}
   securityContext: {}

tests/proxy_migration_tests/test_ui_image_serves_offline.py

@@ -0,0 +1,132 @@
+"""Image-level regression net for arbitrary-uid boot of the UI image.
+
+OpenShift ``restricted-v2`` ignores the image ``USER`` and assigns an
+arbitrary uid in GID 0. The stock nginx base expects to start as root, so
+its cache (``/var/cache/nginx``) and pid (``/run``) paths are root-owned
+755 and the master process dies at startup with
+``mkdir() "/var/cache/nginx/client_temp" failed (13: Permission denied)``.
+The fix anchors everything nginx writes under ``/tmp`` in ``ui/nginx.conf``.
+
+Booting the image the way that deployment does, with a read-only root
+filesystem and ``/tmp`` as the only writable mount, is what catches the
+whole class: a boot as the default (root) uid passes even on the broken
+config.
+
+Gated on LITELLM_IMAGE so it is skipped in the normal unit-test run and
+exercised only where an image has been built (the image-scan workflow).
+Requires a working docker CLI.
+"""
+
+import os
+import shutil
+import subprocess
+import time
+import uuid
+from collections.abc import Iterator
+
+import pytest
+
+IMAGE = os.getenv("LITELLM_IMAGE")
+CURL_IMAGE = os.getenv("LITELLM_TEST_CURL_IMAGE", "curlimages/curl:8.11.1")
+UI_PORT = os.getenv("LITELLM_UI_PORT", "3000")
+ARBITRARY_UID = "1001200000:0"
+STARTUP_TIMEOUT_SECONDS = int(os.getenv("LITELLM_UI_STARTUP_TIMEOUT", "60"))
+
+pytestmark = [
+    pytest.mark.skipif(IMAGE is None, reason="requires a built image (set LITELLM_IMAGE)"),
+    pytest.mark.skipif(shutil.which("docker") is None, reason="requires the docker CLI"),
+]
+
+
+def _docker(*args: str, check: bool = True) -> "subprocess.CompletedProcess[str]":
+    return subprocess.run(["docker", *args], capture_output=True, text=True, check=check)
+
+
+@pytest.fixture()
+def ui_container() -> Iterator[tuple[str, str]]:
+    """The UI container as an arbitrary uid in GID 0 on a network with no egress.
+
+    ``--read-only`` with a tmpfs on ``/tmp`` mirrors the strictest supported
+    deployment: ``readOnlyRootFilesystem: true`` with an emptyDir on ``/tmp``.
+    A config that writes anywhere else fails here exactly like it does on
+    OpenShift.
+    """
+    run_id = f"uiserve-{uuid.uuid4().hex[:8]}"
+    network = f"{run_id}-net"
+    container = f"{run_id}-ui"
+
+    _docker("pull", "--quiet", CURL_IMAGE)
+    _docker("network", "create", "--internal", network)
+    try:
+        assert IMAGE is not None
+        _docker(
+            "run", "-d", "--name", container, "--network", network,
+            "--user", ARBITRARY_UID,
+            "--read-only", "--tmpfs", "/tmp",
+            IMAGE,
+        )
+        yield network, container
+    finally:
+        _docker("logs", container, check=False)
+        _docker("rm", "-f", container, check=False)
+        _docker("network", "rm", network, check=False)
+
+
+def _container_logs(container: str) -> str:
+    logs = _docker("logs", container, check=False)
+    return f"stdout:\n{logs.stdout}\nstderr:\n{logs.stderr}"
+
+
+def _is_running(container: str) -> bool:
+    return bool(
+        _docker(
+            "ps", "--filter", f"name={container}", "--filter", "status=running",
+            "--format", "{{.Names}}", check=False,
+        ).stdout.strip()
+    )
+
+
+def _probe(network: str, container: str, path: str) -> "subprocess.CompletedProcess[str]":
+    return _docker(
+        "run", "--rm", "--network", network, CURL_IMAGE,
+        "--silent", "--show-error", "--max-time", "10",
+        "--output", "/dev/null", "--write-out", "%{http_code}",
+        f"http://{container}:{UI_PORT}{path}",
+        check=False,
+    )
+
+
+def test_ui_serves_as_arbitrary_uid_read_only(ui_container: tuple[str, str]) -> None:
+    """nginx boots and serves as an arbitrary uid with a read-only root fs.
+
+    On the pre-fix config nginx exits during startup with
+    ``mkdir() "/var/cache/nginx/client_temp" failed (13: Permission denied)``
+    and the running-check below fails; it never reaches the probes.
+    """
+    network, container = ui_container
+
+    deadline = time.time() + STARTUP_TIMEOUT_SECONDS
+    healthz = None
+    while time.time() < deadline:
+        if not _is_running(container):
+            pytest.fail(
+                f"the UI container exited during startup as uid {ARBITRARY_UID} with a "
+                f"read-only root filesystem. nginx writes outside /tmp.\n"
+                f"{_container_logs(container)}"
+            )
+        healthz = _probe(network, container, "/healthz")
+        if healthz.returncode == 0 and healthz.stdout.strip() == "200":
+            break
+        time.sleep(2)
+
+    assert healthz is not None and healthz.stdout.strip() == "200", (
+        f"/healthz never answered 200 within {STARTUP_TIMEOUT_SECONDS}s as uid "
+        f"{ARBITRARY_UID}.\n{_container_logs(container)}"
+    )
+
+    for path in ("/", "/ui", "/ui/login"):
+        page = _probe(network, container, path)
+        assert page.stdout.strip() == "200", (
+            f"GET {path} returned {page.stdout.strip()!r} as uid {ARBITRARY_UID}.\n"
+            f"{_container_logs(container)}"
+        )

ui/nginx.conf

@@ -1,7 +1,20 @@
 worker_processes auto;
+
+# Anchor everything nginx writes under /tmp so the image boots as an
+# arbitrary uid (OpenShift restricted-v2 assigns one in gid 0; the stock
+# nginx image's /var/cache/nginx and /run are root-owned 755) and works
+# with readOnlyRootFilesystem when /tmp is an emptyDir.
+pid /tmp/nginx.pid;
+
 events { worker_connections 1024; }
 
 http {
+  client_body_temp_path /tmp/nginx-client-temp;
+  proxy_temp_path       /tmp/nginx-proxy-temp;
+  fastcgi_temp_path     /tmp/nginx-fastcgi-temp;
+  uwsgi_temp_path       /tmp/nginx-uwsgi-temp;
+  scgi_temp_path        /tmp/nginx-scgi-temp;
+
   include       /etc/nginx/mime.types;
   default_type  application/octet-stream;
   sendfile      on;