mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
105 lines
3.1 KiB
YAML
105 lines
3.1 KiB
YAML
name: CodSpeed Benchmarks
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
- litellm_internal_staging
|
|
paths:
|
|
- "litellm/**"
|
|
- "tests/benchmarks/**"
|
|
- "pyproject.toml"
|
|
- "uv.lock"
|
|
- ".github/workflows/codspeed.yml"
|
|
- ".github/actions/setup-uv-with-retries/**"
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- litellm_internal_staging
|
|
paths:
|
|
- "litellm/**"
|
|
- "tests/benchmarks/**"
|
|
- "pyproject.toml"
|
|
- "uv.lock"
|
|
- ".github/workflows/codspeed.yml"
|
|
- ".github/actions/setup-uv-with-retries/**"
|
|
# Allow CodSpeed to trigger backtest performance analysis
|
|
# in order to generate initial data
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
benchmarks:
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Set up uv
|
|
uses: ./.github/actions/setup-uv-with-retries
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Cache uv dependencies
|
|
timeout-minutes: 5
|
|
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
|
with:
|
|
path: |
|
|
~/.cache/uv
|
|
.venv
|
|
key: ${{ runner.os }}-uv-benchmarks-${{ hashFiles('uv.lock', '.github/workflows/codspeed.yml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-uv-benchmarks-
|
|
|
|
- name: Cache cargo build of the Rust bridge
|
|
timeout-minutes: 5
|
|
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
|
with:
|
|
path: |
|
|
~/.cargo/registry
|
|
~/.cargo/git
|
|
litellm-rust/target
|
|
key: ${{ runner.os }}-cargo-benchmarks-${{ hashFiles('litellm-rust/Cargo.lock') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-cargo-benchmarks-
|
|
|
|
# Everything the CodSpeed step runs is wrapped in valgrind, which costs
|
|
# ~16x wall time, so installing there turned a 2 minute maturin build of
|
|
# the Rust bridge into a 40 minute one. Keep that step down to pytest.
|
|
- name: Install benchmark dependencies
|
|
timeout-minutes: 15
|
|
run: |
|
|
.github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups
|
|
uv pip install \
|
|
pytest==8.3.5 \
|
|
pytest-codspeed==4.3.0 \
|
|
"mcp>=1.26.0,<2.0" \
|
|
"a2a-sdk>=1.1.0,<2.0"
|
|
|
|
- name: Run benchmarks
|
|
timeout-minutes: 10
|
|
uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
|
|
with:
|
|
mode: simulation
|
|
run: >
|
|
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
|
|
uv run --no-sync
|
|
pytest
|
|
-p pytest_codspeed.plugin
|
|
tests/benchmarks/
|
|
--codspeed
|