mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-11 03:38:38 +00:00
169af2f883
* ci: split slow unit shards and build the Rust bridge once per run * ci: key the Rust bridge cache on source files only * ci: keep the unit setup ceiling unchanged with the shared Rust bridge * ci: fall back to the Cargo cache when the Rust bridge artifact is missing * ci: keep reruns on enterprise-routing for the prompt caching flake --------- Co-authored-by: yuneng <yuneng@berri.ai>
278 lines
10 KiB
YAML
278 lines
10 KiB
YAML
name: _Unit Test Base (Reusable)
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
test-path:
|
|
description: >-
|
|
Space-separated pytest paths to run. A path that no longer exists is
|
|
dropped with a warning instead of being passed to pytest, because one
|
|
missing path makes pytest-xdist collect nothing and report exit 5, which
|
|
the step treats as a drained shard. Options are passed through as
|
|
written, so use the `--flag=value` form: a bare `--ignore path` would
|
|
have its path existence-checked like any other token.
|
|
required: true
|
|
type: string
|
|
workers:
|
|
description: "Number of pytest-xdist workers"
|
|
required: false
|
|
type: number
|
|
default: 2
|
|
reruns:
|
|
description: "Number of reruns for flaky tests"
|
|
required: false
|
|
type: number
|
|
default: 2
|
|
timeout-minutes:
|
|
description: >-
|
|
Timeout for the test step alone. Setup (checkout, dependency install,
|
|
Prisma client generation) gets its own allowance on top, so a slow
|
|
runner or a cold binary download can never cancel passing tests.
|
|
required: false
|
|
type: number
|
|
default: 20
|
|
job-timeout-minutes:
|
|
description: >-
|
|
Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for
|
|
the per-step ceilings on the setup steps below, and 5 for the runner
|
|
overhead the job clock charges but no step owns (job init, step
|
|
transitions, post-job cleanup). That headroom is what makes the test
|
|
budget a floor rather than a hope, since setup cannot overrun into it
|
|
without failing its own step first. GitHub expressions have no
|
|
arithmetic, so the sum is passed in rather than computed.
|
|
required: false
|
|
type: number
|
|
default: 60
|
|
test-timeout-seconds:
|
|
description: >-
|
|
Per-test ceiling enforced by pytest-timeout, covering fixture setup and
|
|
teardown as well as the test body. A test that hangs fails with a
|
|
traceback of where it was stuck instead of idling the shard until
|
|
`timeout-minutes` cancels it. Timed-out tests are excluded from reruns
|
|
because pytest-timeout arms its timer once per test and
|
|
pytest-rerunfailures reruns inside that same window, so a rerun of a
|
|
timed-out test would run with no timer at all.
|
|
required: false
|
|
type: number
|
|
default: 120
|
|
max-failures:
|
|
description: "Stop after this many failures"
|
|
required: false
|
|
type: number
|
|
default: 10
|
|
dist:
|
|
description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)"
|
|
required: false
|
|
type: string
|
|
default: "loadscope"
|
|
artifact-name:
|
|
description: "Unique name for the coverage artifact (must be unique per run)"
|
|
required: true
|
|
type: string
|
|
rust-bridge-artifact:
|
|
description: "Prebuilt editable Rust bridge artifact"
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
UV_PYTHON: "3.12"
|
|
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
|
|
|
jobs:
|
|
run:
|
|
name: Run tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: ${{ inputs.job-timeout-minutes }}
|
|
permissions:
|
|
contents: read
|
|
pull-requests: read
|
|
outputs:
|
|
decision: ${{ steps.changes.outputs.decision }}
|
|
has-coverage: ${{ steps.tests.outputs.has-coverage }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
timeout-minutes: 3
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Detect relevant changes
|
|
id: changes
|
|
timeout-minutes: 2
|
|
uses: ./.github/actions/detect-changes
|
|
|
|
- name: Set up Python
|
|
if: steps.changes.outputs.decision != 'skip'
|
|
timeout-minutes: 3
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: ${{ env.UV_PYTHON }}
|
|
|
|
- name: Set up uv
|
|
if: steps.changes.outputs.decision != 'skip'
|
|
timeout-minutes: 3
|
|
uses: ./.github/actions/setup-uv-with-retries
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Cache uv dependencies
|
|
if: steps.changes.outputs.decision != 'skip'
|
|
timeout-minutes: 5
|
|
uses: ./.github/actions/cache-uv-downloads
|
|
|
|
- name: Set up the Rust build
|
|
if: steps.changes.outputs.decision != 'skip'
|
|
timeout-minutes: 5
|
|
uses: ./.github/actions/rust-bridge
|
|
with:
|
|
artifact: ${{ inputs.rust-bridge-artifact }}
|
|
|
|
- name: Install dependencies
|
|
if: steps.changes.outputs.decision != 'skip'
|
|
timeout-minutes: 8
|
|
env:
|
|
RUST_BRIDGE_ARTIFACT: ${{ inputs.rust-bridge-artifact }}
|
|
run: |
|
|
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
|
|
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
|
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime
|
|
else
|
|
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --no-install-project
|
|
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
|
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
|
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
|
|
fi
|
|
uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
|
|
|
|
- name: Cache Prisma binaries
|
|
if: steps.changes.outputs.decision != 'skip'
|
|
timeout-minutes: 3
|
|
uses: ./.github/actions/cache-prisma-binaries
|
|
|
|
- name: Generate Prisma client
|
|
if: steps.changes.outputs.decision != 'skip'
|
|
timeout-minutes: 3
|
|
run: |
|
|
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
|
|
|
- name: Run tests
|
|
id: tests
|
|
if: steps.changes.outputs.decision != 'skip'
|
|
timeout-minutes: ${{ inputs.timeout-minutes }}
|
|
env:
|
|
TEST_PATH: ${{ inputs.test-path }}
|
|
MAX_FAILURES: ${{ inputs.max-failures }}
|
|
WORKERS: ${{ inputs.workers }}
|
|
RERUNS: ${{ inputs.reruns }}
|
|
TEST_TIMEOUT_SECONDS: ${{ inputs.test-timeout-seconds }}
|
|
DIST: ${{ inputs.dist }}
|
|
COVERAGE_CORE: sysmon
|
|
run: |
|
|
echo "has-coverage=false" >> "$GITHUB_OUTPUT"
|
|
selection="${TEST_PATH}"
|
|
if [ -z "${selection// /}" ]; then
|
|
echo "shard selection is empty; nothing to run"
|
|
exit 0
|
|
fi
|
|
pytest_args=()
|
|
existing_paths=0
|
|
for token in ${selection}; do
|
|
case "${token}" in
|
|
-*) pytest_args+=("${token}") ;;
|
|
*)
|
|
if [ -e "${token%%::*}" ]; then
|
|
pytest_args+=("${token}")
|
|
existing_paths=$((existing_paths + 1))
|
|
else
|
|
echo "::warning::${token} does not exist; drop it from this shard's test-path"
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
if [ "${existing_paths}" -eq 0 ]; then
|
|
echo "No path in the selection exists (${selection}); nothing to run"
|
|
exit 0
|
|
fi
|
|
xdist_args=()
|
|
if [ "${WORKERS}" != "0" ]; then
|
|
xdist_args=(-n "${WORKERS}" --dist="${DIST}")
|
|
fi
|
|
set +e
|
|
uv run --no-sync pytest "${pytest_args[@]}" \
|
|
--tb=short -vv \
|
|
--maxfail="${MAX_FAILURES}" \
|
|
"${xdist_args[@]}" \
|
|
--reruns "${RERUNS}" \
|
|
--reruns-delay 1 \
|
|
--timeout="${TEST_TIMEOUT_SECONDS}" \
|
|
--rerun-except "from pytest-timeout" \
|
|
--durations=20 \
|
|
--cov=./litellm --cov=./enterprise/litellm_enterprise \
|
|
--cov-report=xml:coverage.xml \
|
|
--cov-config=pyproject.toml
|
|
status=$?
|
|
set -e
|
|
if [ -f coverage.xml ]; then
|
|
echo "has-coverage=true" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
if [ "$status" -eq 5 ]; then
|
|
echo "pytest collected no tests from ${selection}; passing"
|
|
exit 0
|
|
fi
|
|
exit "$status"
|
|
|
|
- name: Save coverage report
|
|
if: always() && steps.changes.outputs.decision != 'skip'
|
|
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
|
with:
|
|
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: coverage.xml
|
|
retention-days: 1
|
|
|
|
upload-coverage:
|
|
name: Upload coverage to Codecov
|
|
needs: run
|
|
if: always() && needs.run.outputs.decision != 'skip' && needs.run.outputs.has-coverage == 'true'
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
id-token: write
|
|
pull-requests: write
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Download coverage report
|
|
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
|
|
with:
|
|
pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
|
|
path: coverage-reports
|
|
merge-multiple: true
|
|
|
|
- name: Upload to Codecov
|
|
id: codecov-upload
|
|
continue-on-error: true
|
|
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
|
|
with:
|
|
use_oidc: true
|
|
directory: coverage-reports
|
|
root_dir: ${{ github.workspace }}
|
|
flags: ${{ inputs.artifact-name }}
|
|
fail_ci_if_error: false
|
|
|
|
- name: Upload to Codecov (retry)
|
|
if: steps.codecov-upload.outcome == 'failure'
|
|
continue-on-error: true
|
|
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
|
|
with:
|
|
use_oidc: true
|
|
directory: coverage-reports
|
|
root_dir: ${{ github.workspace }}
|
|
flags: ${{ inputs.artifact-name }}
|
|
fail_ci_if_error: false
|