litellm/.github/workflows/codspeed.yml

name: CodSpeed Benchmarks

on:
  push:
    branches:
      - main
    paths:
      - "litellm/**"
      - "tests/benchmarks/**"
      - "pyproject.toml"
      - "uv.lock"
      - ".github/workflows/codspeed.yml"
      - ".github/actions/setup-uv-with-retries/**"
      - ".github/actions/cache-cargo-build/**"
      - ".github/scripts/uv_sync_with_retries.sh"
  pull_request:
    branches:
      - main
    paths:
      - "litellm/**"
      - "tests/benchmarks/**"
      - "pyproject.toml"
      - "uv.lock"
      - ".github/workflows/codspeed.yml"
      - ".github/actions/setup-uv-with-retries/**"
      - ".github/actions/cache-cargo-build/**"
      - ".github/scripts/uv_sync_with_retries.sh"
  # Allow CodSpeed to trigger backtest performance analysis
  # in order to generate initial data
  workflow_dispatch:

permissions:
  contents: read
  id-token: write

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  benchmarks:
    runs-on: ubuntu-24.04
    timeout-minutes: 60

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Set up uv
        uses: ./.github/actions/setup-uv-with-retries
        with:
          version: "0.10.9"

      - name: Cache the Rust build
        uses: ./.github/actions/cache-cargo-build
        with:
          profile: release

      # Build the wheel and resolve every dependency outside the CodSpeed
      # runner: the same maturin build took 42 minutes inside `codspeed run`
      # versus under 3 minutes as a plain step (LIT-6183)
      - name: Build the release wheel
        run: uv build --wheel --out-dir dist

      - name: Install the wheel into the benchmark environment
        run: |
          UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12
          uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl

      - name: Collect benchmarks
        env:
          PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
          LITELLM_REQUIRE_INSTALLED_WHEEL: "1"
        run: >
          "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
          --import-mode=importlib
          -p pytest_codspeed.plugin
          tests/benchmarks/
          --codspeed
          --collect-only -q

      - name: Run benchmarks
        uses: CodSpeedHQ/action@821a40cd6c80e3cfe387caf55668287addd808da # v5.5.0
        with:
          mode: simulation
          run: >
            env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1
            "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
            --import-mode=importlib
            -p pytest_codspeed.plugin
            tests/benchmarks/
            --codspeed