mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-25 05:54:53 +00:00
868cd06ea9
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
uv already retries network failures, so the wrapper only repeated deterministic uv sync failures five times. Cargo network settings it exported already live in .cargo/config.toml
96 lines
2.9 KiB
YAML
96 lines
2.9 KiB
YAML
name: CodSpeed Benchmarks
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
paths:
|
|
- "litellm/**"
|
|
- "tests/benchmarks/**"
|
|
- "pyproject.toml"
|
|
- "uv.lock"
|
|
- ".github/workflows/codspeed.yml"
|
|
- ".github/actions/setup-uv-with-retries/**"
|
|
- ".github/actions/cache-cargo-build/**"
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
paths:
|
|
- "litellm/**"
|
|
- "tests/benchmarks/**"
|
|
- "pyproject.toml"
|
|
- "uv.lock"
|
|
- ".github/workflows/codspeed.yml"
|
|
- ".github/actions/setup-uv-with-retries/**"
|
|
- ".github/actions/cache-cargo-build/**"
|
|
# Allow CodSpeed to trigger backtest performance analysis
|
|
# in order to generate initial data
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
benchmarks:
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 60
|
|
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Set up uv
|
|
uses: ./.github/actions/setup-uv-with-retries
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Cache the Rust build
|
|
uses: ./.github/actions/cache-cargo-build
|
|
with:
|
|
profile: release
|
|
|
|
# Build the wheel and resolve every dependency outside the CodSpeed
|
|
# runner: the same maturin build took 42 minutes inside `codspeed run`
|
|
# versus under 3 minutes as a plain step (LIT-6183)
|
|
- name: Build the release wheel
|
|
run: uv build --wheel --out-dir dist
|
|
|
|
- name: Install the wheel into the benchmark environment
|
|
run: |
|
|
UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" uv sync --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12
|
|
uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl
|
|
|
|
- name: Collect benchmarks
|
|
env:
|
|
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
|
|
LITELLM_REQUIRE_INSTALLED_WHEEL: "1"
|
|
run: >
|
|
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
|
|
--import-mode=importlib
|
|
-p pytest_codspeed.plugin
|
|
tests/benchmarks/
|
|
--codspeed
|
|
--collect-only -q
|
|
|
|
- name: Run benchmarks
|
|
uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
|
|
with:
|
|
mode: simulation
|
|
run: >
|
|
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1
|
|
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
|
|
--import-mode=importlib
|
|
-p pytest_codspeed.plugin
|
|
tests/benchmarks/
|
|
--codspeed
|