mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
1eb7122465
Guard the per-request CPU cost of the chat completion, MCP tool and A2A message transforms against regressions on every commit. All benchmarks are pure in-process work with no network I/O so they stay deterministic under CodSpeed's simulation mode, and they import under the base dependency set the benchmark job installs. Inference covers the full SDK overhead via mock_response (simple, multi-turn, tools, streaming) plus convert_to_model_response_object as a deterministic anchor. MCP covers the client-side tool translation and the proxy server-side tool-name prefix round-trip. A2A covers the client request/response transforms and the proxy server-ingress message conversion. Adds the mcp and a2a-sdk packages to the benchmark run since those transform modules need them, and broadens the workflow triggers to litellm_internal_staging so the internal branch flow is benchmarked too.
58 lines
1.5 KiB
YAML
58 lines
1.5 KiB
YAML
name: CodSpeed Benchmarks
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
- litellm_internal_staging
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- litellm_internal_staging
|
|
# Allow CodSpeed to trigger backtest performance analysis
|
|
# in order to generate initial data
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
benchmarks:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Set up uv
|
|
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Run benchmarks
|
|
uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
|
|
with:
|
|
mode: simulation
|
|
run: >
|
|
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
|
|
uv run --frozen --no-default-groups
|
|
--with pytest==8.3.5
|
|
--with pytest-codspeed==4.3.0
|
|
--with "mcp>=1.26.0,<2.0"
|
|
--with "a2a-sdk>=1.1.0,<2.0"
|
|
pytest
|
|
-p pytest_codspeed.plugin
|
|
tests/benchmarks/
|
|
--codspeed
|