litellm/.github/workflows/image-scan.yml

name: Image Scan

on:
  pull_request:
    branches:
      - main
      - litellm_oss_branch
      - "litellm_**"
    paths:
      - Dockerfile
      - docker/Dockerfile.non_root
      - migrations/Dockerfile
      - migrations/run.py
      - gateway/Dockerfile
      - gateway/main.py
      - backend/Dockerfile
      - backend/main.py
      - deploy/lens/**
      - litellm/proxy/lens/**
      - tests/e2e/migrations/lens_compose_smoke.sh
      - docker/component_entrypoint.sh
      - docker/entrypoint.sh
      - litellm/proxy/prisma_migration.py
      - litellm-proxy-extras/**
      - tests/proxy_migration_tests/**
      - uv.lock
      - ui/litellm-dashboard/package-lock.json
      - ui/Dockerfile
      - ui/nginx.conf
      - .github/workflows/image-scan.yml
      - .grype.yaml
  schedule:
    - cron: "41 6 * * *"
  workflow_dispatch:

permissions: {}

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  lens-worker-image:
    name: lens-worker-image (${{ matrix.arch }})
    runs-on: ${{ matrix.runner }}
    if: >-
      github.event_name != 'pull_request' ||
      github.event.pull_request.head.repo.full_name == github.repository
    timeout-minutes: 15
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        include:
          - arch: amd64
            runner: ubuntu-latest
            grype_sha256: edda0968d8827daab01d32b3cd7de192ae0915005e7bbfcfef9e68e79bc43343
          - arch: arm64
            runner: ubuntu-24.04-arm
            grype_sha256: 553e4c36d9d61349830ba6034d43b8700a7f10576d3e2f4981c0fd2b96086465
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false
      - name: Build the release worker
        env:
          RELEASE_TAG: sha-${{ github.sha }}
        run: docker build --build-arg LITELLM_RELEASE_TAG="${RELEASE_TAG}" -f deploy/lens/Dockerfile -t lens-worker-scan .
      - name: Verify the standalone worker on a read-only filesystem
        env:
          RELEASE_TAG: sha-${{ github.sha }}
        run: |
          docker run --rm --network none --read-only --cap-drop ALL \
            --tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
            -e EXPECTED_RELEASE_TAG="${RELEASE_TAG}" --entrypoint python lens-worker-scan -c '
          import os
          import lens.worker
          from lens.release import release_tag
          from lens.trace_store import trace_store
          assert os.getuid() == 65532
          assert release_tag() == os.environ["EXPECTED_RELEASE_TAG"]
          with trace_store() as store:
              assert store.count() == 0
          '
      - name: Reject a dependency whose hash has changed
        run: |
          docker build --target builder -f deploy/lens/Dockerfile -t lens-worker-deps .
          sed -E 's/sha256:[0-9a-f]{64}/sha256:0000000000000000000000000000000000000000000000000000000000000000/g' \
            deploy/lens/requirements.lock > "$RUNNER_TEMP/tampered.lock"
          if docker run --rm -v "$RUNNER_TEMP/tampered.lock:/tmp/tampered.lock:ro" \
            --entrypoint uv lens-worker-deps pip sync --python /app/.venv/bin/python \
            --require-hashes --only-binary :all: --reinstall --no-cache /tmp/tampered.lock \
            > "$RUNNER_TEMP/hash-check.log" 2>&1; then
            echo "::error::Dependency hash mismatch was accepted"
            exit 1
          fi
          cat "$RUNNER_TEMP/hash-check.log"
          grep -qi 'hash mismatch' "$RUNNER_TEMP/hash-check.log"
      - name: Download Grype v0.114.0
        env:
          ARCH: ${{ matrix.arch }}
          GRYPE_SHA256: ${{ matrix.grype_sha256 }}
        run: |
          curl -fsSL --retry 3 -o "$RUNNER_TEMP/grype.tar.gz" \
            "https://github.com/anchore/grype/releases/download/v0.114.0/grype_0.114.0_linux_${ARCH}.tar.gz"
          echo "${GRYPE_SHA256}  $RUNNER_TEMP/grype.tar.gz" | sha256sum -c -
          tar xzf "$RUNNER_TEMP/grype.tar.gz" -C "$RUNNER_TEMP" grype
          chmod +x "$RUNNER_TEMP/grype"
      - name: Scan the worker for fixable HIGH/CRITICAL CVEs
        env:
          GRYPE_MATCH_PYTHON_USING_CPES: "true"
        run: |
          "$RUNNER_TEMP/grype" lens-worker-scan \
            --config .grype.yaml --only-fixed --fail-on high --output table

  image-scan:
    name: image-scan
    runs-on: ubuntu-latest
    if: >-
      github.event_name != 'pull_request' ||
      github.event.pull_request.head.repo.full_name == github.repository
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false

      - name: Download Grype v0.114.0
        run: |
          curl -fsSL --retry 3 -o "$RUNNER_TEMP/grype.tar.gz" \
            https://github.com/anchore/grype/releases/download/v0.114.0/grype_0.114.0_linux_amd64.tar.gz
          echo "edda0968d8827daab01d32b3cd7de192ae0915005e7bbfcfef9e68e79bc43343  $RUNNER_TEMP/grype.tar.gz" | sha256sum -c -
          tar xzf "$RUNNER_TEMP/grype.tar.gz" -C "$RUNNER_TEMP" grype
          chmod +x "$RUNNER_TEMP/grype"

      # Dockerfile.non_root is the rootless variant we ship. The other
      # Dockerfiles share the same wolfi base and apk set, so OS-layer coverage
      # is the same; matrix-scan if those variants ever diverge.
      - name: Build runtime image
        run: docker build -f docker/Dockerfile.non_root -t litellm-image-scan:${{ github.sha }} .

      # The prisma bake must migrate a fresh DB with no egress as an arbitrary
      # non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyRootFilesystem).
      # `docker run` as the default uid with network hides a broken bake because
      # the migration entrypoint exits 0 even when it applied nothing; asserting
      # the schema was created is what catches it.
      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Verify offline migration as a non-root uid
        env:
          LITELLM_IMAGE: litellm-image-scan:${{ github.sha }}
        run: |
          python -m pip install "pytest==9.0.3"
          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v

      # Scans the whole shipped artifact: OS/apk plus every language package
      # baked into the image, including ones no lockfile declares (e.g. prisma's
      # vendored node engine) that osv-scan cannot see. osv-scan stays the fast
      # source-level gate; this is the customer's-eye-view backstop. Credential-
      # free OSS, run as a pinned, checksum-verified binary; no GitHub Action
      # dependency and no vendor SaaS callout.
      - name: Scan image for fixable HIGH/CRITICAL CVEs
        env:
          GRYPE_MATCH_PYTHON_USING_CPES: "true"
        run: |
          "$RUNNER_TEMP/grype" litellm-image-scan:${{ github.sha }} \
            --config .grype.yaml \
            --only-fixed \
            --fail-on high \
            --output table

  runtime-image:
    name: runtime-image
    runs-on: ubuntu-latest
    if: >-
      github.event_name != 'pull_request' ||
      github.event.pull_request.head.repo.full_name == github.repository
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false

      - name: Build runtime image
        run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Verify offline migration as a non-root uid
        env:
          LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
        run: |
          python -m pip install "pytest==9.0.3"
          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v

      - name: Verify the bundled Lens Compose installation and restart
        env:
          LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
        run: bash tests/e2e/migrations/lens_compose_smoke.sh

  migrations-image:
    name: migrations-image
    runs-on: ubuntu-latest
    if: >-
      github.event_name != 'pull_request' ||
      github.event.pull_request.head.repo.full_name == github.repository
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false

      - name: Build migrations image
        run: docker build -f migrations/Dockerfile -t litellm-migrations-scan:${{ github.sha }} .

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Verify offline migration as a non-root uid
        env:
          LITELLM_IMAGE: litellm-migrations-scan:${{ github.sha }}
          LITELLM_MIGRATION_INTERPRETER: python3
          LITELLM_MIGRATION_SCRIPT: /app/run.py
        run: |
          python -m pip install "pytest==9.0.3"
          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v

  gateway-image:
    name: gateway-image
    runs-on: ubuntu-latest
    if: >-
      github.event_name != 'pull_request' ||
      github.event.pull_request.head.repo.full_name == github.repository
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false

      - name: Build gateway image
        run: docker build -f gateway/Dockerfile -t litellm-gateway-scan:${{ github.sha }} .

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Verify the gateway serves offline as a non-root uid
        env:
          LITELLM_IMAGE: litellm-gateway-scan:${{ github.sha }}
          LITELLM_COMPONENT_PORT: "4000"
        run: |
          python -m pip install "pytest==9.0.3"
          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v

  ui-image:
    name: ui-image
    runs-on: ubuntu-latest
    if: >-
      github.event_name != 'pull_request' ||
      github.event.pull_request.head.repo.full_name == github.repository
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false

      - name: Build UI image
        run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
        env:
          LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
        run: |
          python -m pip install "pytest==9.0.3"
          python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v

  backend-image:
    name: backend-image
    runs-on: ubuntu-latest
    if: >-
      github.event_name != 'pull_request' ||
      github.event.pull_request.head.repo.full_name == github.repository
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false

      - name: Build backend image
        run: docker build -f backend/Dockerfile -t litellm-backend-scan:${{ github.sha }} .

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Verify the backend serves offline as a non-root uid
        env:
          LITELLM_IMAGE: litellm-backend-scan:${{ github.sha }}
          LITELLM_COMPONENT_PORT: "4001"
        run: |
          python -m pip install "pytest==9.0.3"
          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v