name: CodSpeed Benchmarks

on:
  push:
    branches:
      - main
      - litellm_internal_staging
    paths:
      - "litellm/**"
      - "tests/benchmarks/**"
      - "pyproject.toml"
      - "uv.lock"
      - ".github/workflows/codspeed.yml"
      - ".github/actions/setup-uv-with-retries/**"
  pull_request:
    branches:
      - main
      - litellm_internal_staging
    paths:
      - "litellm/**"
      - "tests/benchmarks/**"
      - "pyproject.toml"
      - "uv.lock"
      - ".github/workflows/codspeed.yml"
      - ".github/actions/setup-uv-with-retries/**"
  # Allow CodSpeed to trigger backtest performance analysis
  # in order to generate initial data
  workflow_dispatch:

permissions:
  contents: read
  id-token: write

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  benchmarks:
    runs-on: ubuntu-24.04
    timeout-minutes: 30

    steps:
      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
        with:
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Set up uv
        uses: ./.github/actions/setup-uv-with-retries
        with:
          version: "0.10.9"

      - name: Cache uv dependencies
        timeout-minutes: 5
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          path: |
            ~/.cache/uv
            .venv
          key: ${{ runner.os }}-uv-benchmarks-${{ hashFiles('uv.lock', '.github/workflows/codspeed.yml') }}
          restore-keys: |
            ${{ runner.os }}-uv-benchmarks-

      - name: Cache cargo build of the Rust bridge
        timeout-minutes: 5
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            litellm-rust/target
          key: ${{ runner.os }}-cargo-benchmarks-${{ hashFiles('litellm-rust/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-benchmarks-

      # Everything the CodSpeed step runs is wrapped in valgrind, which costs
      # ~16x wall time, so installing there turned a 2 minute maturin build of
      # the Rust bridge into a 40 minute one. Keep that step down to pytest.
      - name: Install benchmark dependencies
        timeout-minutes: 15
        run: |
          .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups
          uv pip install \
            pytest==8.3.5 \
            pytest-codspeed==4.3.0 \
            "mcp>=1.26.0,<2.0" \
            "a2a-sdk>=1.1.0,<2.0"

      - name: Run benchmarks
        timeout-minutes: 10
        uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
        with:
          mode: simulation
          run: >
            env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
            uv run --no-sync
            pytest
            -p pytest_codspeed.plugin
            tests/benchmarks/
            --codspeed