mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-11 03:38:38 +00:00
[Infra] Remove redundant matrix unit test workflow
The individual test-unit-*.yml workflows fully cover all test paths that test-litellm-matrix.yml was running. Remove the matrix workflow to eliminate duplicate CI runs on every PR. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yuneng Jiang
parent
7250cba3db
commit
fb404ec5dd
2 changed files with 2 additions and 216 deletions
214
.github/workflows/test-litellm-matrix.yml
vendored
214
.github/workflows/test-litellm-matrix.yml
vendored
|
|
@ -1,214 +0,0 @@
|
|||
name: LiteLLM Unit Tests (Matrix)
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Cancel in-progress runs for the same PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20 # Increased from 15 to 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-group:
|
||||
# tests/test_litellm split by subdirectory (~560 files total)
|
||||
# Vertex AI tests separated for better isolation (prevent auth/env pollution)
|
||||
- name: "llms-vertex"
|
||||
path: "tests/test_litellm/llms/vertex_ai"
|
||||
workers: 1
|
||||
reruns: 2
|
||||
- name: "llms-other"
|
||||
path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
|
||||
workers: 2
|
||||
reruns: 2
|
||||
# tests/test_litellm/proxy split by subdirectory (~180 files total)
|
||||
- name: "proxy-guardrails"
|
||||
path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
|
||||
workers: 2
|
||||
reruns: 2
|
||||
- name: "proxy-core"
|
||||
path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
|
||||
workers: 2
|
||||
reruns: 2
|
||||
- name: "proxy-misc"
|
||||
path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
|
||||
workers: 2
|
||||
reruns: 2
|
||||
- name: "integrations"
|
||||
path: "tests/test_litellm/integrations"
|
||||
workers: 2
|
||||
reruns: 3 # Integration tests tend to be flakier
|
||||
- name: "core-utils"
|
||||
path: "tests/test_litellm/litellm_core_utils"
|
||||
workers: 2
|
||||
reruns: 1
|
||||
- name: "other-1"
|
||||
# responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
|
||||
path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
|
||||
workers: 2
|
||||
reruns: 2
|
||||
- name: "other-2"
|
||||
# enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
|
||||
path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
|
||||
workers: 2
|
||||
reruns: 2
|
||||
- name: "other-3"
|
||||
# remaining dirs ≈ 8.0k lines
|
||||
path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
|
||||
workers: 2
|
||||
reruns: 2
|
||||
- name: "root"
|
||||
path: "tests/test_litellm/test_*.py"
|
||||
workers: 2
|
||||
reruns: 2
|
||||
# tests/proxy_unit_tests split alphabetically (~48 files total)
|
||||
- name: "proxy-unit-a1"
|
||||
# test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
|
||||
path: "tests/proxy_unit_tests/test_[a-j]*.py"
|
||||
workers: 2
|
||||
reruns: 1
|
||||
- name: "proxy-unit-a2"
|
||||
# test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
|
||||
path: "tests/proxy_unit_tests/test_[k-o]*.py"
|
||||
workers: 2
|
||||
reruns: 1
|
||||
- name: "proxy-unit-b1"
|
||||
# lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
|
||||
path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
|
||||
workers: 2
|
||||
reruns: 1
|
||||
- name: "proxy-unit-b2"
|
||||
# proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
|
||||
path: "tests/proxy_unit_tests/test_proxy_server.py"
|
||||
workers: 2
|
||||
reruns: 1
|
||||
- name: "proxy-unit-b3"
|
||||
# proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
|
||||
path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
|
||||
workers: 2
|
||||
reruns: 1
|
||||
- name: "proxy-unit-b4"
|
||||
# proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
|
||||
path: "tests/proxy_unit_tests/test_proxy_utils.py"
|
||||
workers: 2
|
||||
reruns: 1
|
||||
- name: "proxy-unit-b5"
|
||||
# proxy_token_counter (1279) - runs independently from utils
|
||||
path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
|
||||
workers: 2
|
||||
reruns: 1
|
||||
- name: "proxy-unit-b6"
|
||||
# test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
|
||||
path: "tests/proxy_unit_tests/test_[r-t]*.py"
|
||||
workers: 2
|
||||
reruns: 1
|
||||
- name: "proxy-unit-b7"
|
||||
# test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
|
||||
path: "tests/proxy_unit_tests/test_[u-z]*.py"
|
||||
workers: 2
|
||||
reruns: 1
|
||||
|
||||
name: test (${{ matrix.test-group.name }})
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install Poetry
|
||||
run: pip install 'poetry==2.3.2'
|
||||
|
||||
- name: Cache Poetry dependencies
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/pypoetry
|
||||
~/.cache/pip
|
||||
.venv
|
||||
key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-poetry-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
poetry config virtualenvs.in-project true
|
||||
poetry install --with dev,proxy-dev --extras "proxy semantic-router"
|
||||
# pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
|
||||
poetry run pip install google-genai==1.22.0 \
|
||||
google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
|
||||
|
||||
- name: Setup litellm-enterprise
|
||||
run: |
|
||||
poetry run pip install --force-reinstall --no-deps -e enterprise/
|
||||
|
||||
- name: Generate Prisma client
|
||||
env:
|
||||
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
|
||||
run: |
|
||||
poetry run pip install nodejs-wheel-binaries==24.13.1
|
||||
poetry run prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Run tests - ${{ matrix.test-group.name }}
|
||||
run: |
|
||||
poetry run pytest ${{ matrix.test-group.path }} \
|
||||
--tb=short -vv \
|
||||
--maxfail=10 \
|
||||
-n ${{ matrix.test-group.workers }} \
|
||||
--reruns ${{ matrix.test-group.reruns }} \
|
||||
--reruns-delay 1 \
|
||||
--dist=loadscope \
|
||||
--durations=20 \
|
||||
--cov=litellm \
|
||||
--cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \
|
||||
--cov-config=pyproject.toml
|
||||
|
||||
- name: Save coverage report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: coverage-${{ matrix.test-group.name }}
|
||||
path: coverage-${{ matrix.test-group.name }}.xml
|
||||
retention-days: 1
|
||||
|
||||
upload-coverage:
|
||||
name: Upload coverage to Codecov
|
||||
needs: test
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # Required for OIDC tokenless upload
|
||||
pull-requests: write # Required for Codecov PR comments
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
||||
- name: Download all coverage reports
|
||||
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
|
||||
with:
|
||||
pattern: coverage-*
|
||||
path: coverage-reports
|
||||
merge-multiple: true
|
||||
|
||||
- name: Upload to Codecov
|
||||
uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4
|
||||
with:
|
||||
use_oidc: true
|
||||
directory: coverage-reports
|
||||
root_dir: ${{ github.workspace }}
|
||||
fail_ci_if_error: false
|
||||
4
.github/workflows/test-litellm.yml
vendored
4
.github/workflows/test-litellm.yml
vendored
|
|
@ -1,7 +1,7 @@
|
|||
name: LiteLLM Mock Tests (folder - tests/test_litellm)
|
||||
|
||||
# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs
|
||||
# the same tests in parallel across 10 jobs for faster CI times.
|
||||
# DEPRECATED: This workflow is replaced by the individual test-unit-*.yml
|
||||
# workflows which run the same tests with better readability.
|
||||
# Kept for manual debugging only.
|
||||
on:
|
||||
workflow_dispatch: # Manual trigger only
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue