litellm/.github/workflows/codspeed.yml

name: CodSpeed Benchmarks

on:
  push:
    branches:
      - main
      - litellm_internal_staging
    paths:
      - "litellm/**"
      - "tests/benchmarks/**"
      - "pyproject.toml"
      - "uv.lock"
      - ".github/workflows/codspeed.yml"
      - ".github/actions/setup-uv-with-retries/**"
  pull_request:
    branches:
      - main
      - litellm_internal_staging
    paths:
      - "litellm/**"
      - "tests/benchmarks/**"
      - "pyproject.toml"
      - "uv.lock"
      - ".github/workflows/codspeed.yml"
      - ".github/actions/setup-uv-with-retries/**"
  # Allow CodSpeed to trigger backtest performance analysis
  # in order to generate initial data
  workflow_dispatch:

permissions:
  contents: read
  id-token: write

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  benchmarks:
    runs-on: ubuntu-24.04
    timeout-minutes: 60

    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Set up uv
        uses: ./.github/actions/setup-uv-with-retries
        with:
          version: "0.10.9"

      - name: Run benchmarks
        uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
        with:
          mode: simulation
          run: >
            env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
            uv run --frozen --no-default-groups
            --with pytest==8.3.5
            --with pytest-codspeed==4.3.0
            --with "mcp>=1.26.0,<2.0"
            --with "a2a-sdk>=1.1.0,<2.0"
            pytest
            -p pytest_codspeed.plugin
            tests/benchmarks/
            --codspeed