From 3dda798c8a24f255219a2808eccbd2f66434f681 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 15 Sep 2026 19:28:50 -0700 Subject: [PATCH] ci(e2e): consolidate cache contracts in filtered CircleCI job --- .circleci/config.yml | 17 ++++- .circleci/scripts/classify_changes.sh | 11 +++- .circleci/scripts/path_filter.sh | 4 +- .github/workflows/test-provider-cache.yml | 63 ------------------- .../test_litellm/test_circleci_path_filter.py | 14 +++++ 5 files changed, 40 insertions(+), 69 deletions(-) delete mode 100644 .github/workflows/test-provider-cache.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index bb4ad0f4019..e2102a9ae91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ commands: parameters: category: type: enum - enum: ["backend", "client"] + enum: ["backend", "client", "provider-harness"] default: "backend" steps: - run: @@ -2918,19 +2918,30 @@ jobs: provider_replay_harness: docker: - *python312_image + - image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f working_directory: ~/project resource_class: medium + environment: + E2E_CACHE_TEST_REDIS_URL: redis://127.0.0.1:6379/0 + E2E_PROVIDER_CACHE: "0" + E2E_FIXTURE_MODE: live steps: + - checkout + - skip_if_unrelated_changes: + category: provider-harness - setup_litellm_test_deps + - wait_for_service: + url: tcp://localhost:6379 - run: - name: Test provider replay harness + name: Test provider capture and replay harness command: | mkdir -p test-results/provider-replay-harness uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \ --junitxml=test-results/provider-replay-harness/junit.xml \ tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \ tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \ - tests/code_coverage_tests/test_provider_replay_harness.py + tests/code_coverage_tests/test_provider_replay_harness.py \ + tests/code_coverage_tests/test_provider_cache.py - store_test_results: path: test-results/provider-replay-harness diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index 7aa0c3544ee..9dc7b76b23f 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -1,13 +1,19 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: classify_changes.sh }" +category="${1:?usage: classify_changes.sh }" has_client=false has_backend=false has_ci=false +has_provider_harness=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue + case "$file" in + tests/e2e/*/*.py) : ;; + tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock) + has_provider_harness=true ;; + esac case "$file" in ui/* | tests/e2e/ui/*) has_client=true ;; docs/* | *.md | *.mdx) : ;; @@ -17,6 +23,9 @@ while IFS= read -r file || [ -n "$file" ]; do done case "$category" in + provider-harness) + [ "$has_provider_harness" = true ] && echo run || echo skip + ;; backend) [ "$has_backend" = true ] && echo run || echo skip ;; diff --git a/.circleci/scripts/path_filter.sh b/.circleci/scripts/path_filter.sh index dcf64a24399..cdadde732bd 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.circleci/scripts/path_filter.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: path_filter.sh }" +category="${1:?usage: path_filter.sh }" here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_full() { @@ -36,5 +36,5 @@ if [ "$decision" = run ]; then run_full "$category-relevant changes detected" fi -echo "path-filter[$category]: only unrelated (docs/client) changes detected; halting job as successful" +echo "path-filter[$category]: only unrelated changes detected; halting job as successful" circleci-agent step halt diff --git a/.github/workflows/test-provider-cache.yml b/.github/workflows/test-provider-cache.yml deleted file mode 100644 index 0507f90cf89..00000000000 --- a/.github/workflows/test-provider-cache.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Provider cache contracts - -on: - pull_request: - paths: - - 'tests/e2e/**' - - 'tests/code_coverage_tests/test_provider_cache.py' - - 'tests/code_coverage_tests/test_provider_replay_harness.py' - - '.github/workflows/test-provider-cache.yml' - - 'pyproject.toml' - - 'uv.lock' - workflow_dispatch: - -permissions: - contents: read - -jobs: - provider-cache: - runs-on: ubuntu-latest - timeout-minutes: 20 - services: - redis: - image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 5s - --health-timeout 3s - --health-retries 10 - env: - PYTHONDONTWRITEBYTECODE: '1' - PYTHONPATH: tests/e2e - E2E_CACHE_TEST_REDIS_URL: redis://127.0.0.1:6379/0 - E2E_PROVIDER_CACHE: '0' - E2E_FIXTURE_MODE: live - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: '3.13' - - uses: ./.github/actions/setup-uv-with-retries - with: - version: '0.10.9' - - uses: ./.github/actions/cache-cargo-build - - name: Install locked dependencies - run: .github/scripts/uv_sync_with_retries.sh --frozen --group proxy-dev --group e2e-dev - - name: Verify cache and existing edge contracts - run: >- - uv run --no-sync pytest -c /dev/null -p no:cacheprovider - tests/code_coverage_tests/test_provider_cache.py - tests/code_coverage_tests/test_provider_replay_harness.py - tests/e2e/test_provider_edge.py - -q --junitxml=provider-cache-results.xml - - name: Save test results - if: always() - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: provider-cache-results - path: provider-cache-results.xml - if-no-files-found: warn diff --git a/tests/test_litellm/test_circleci_path_filter.py b/tests/test_litellm/test_circleci_path_filter.py index b427a1a3bd8..af0e932400f 100644 --- a/tests/test_litellm/test_circleci_path_filter.py +++ b/tests/test_litellm/test_circleci_path_filter.py @@ -49,6 +49,20 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"] @pytest.mark.parametrize( "category,changed,expected", [ + ("provider-harness", ["tests/e2e/provider_cache.py"], "run"), + ("provider-harness", ["tests/e2e/conftest.py"], "run"), + ("provider-harness", ["tests/e2e/e2e_http.py"], "run"), + ("provider-harness", ["tests/code_coverage_tests/test_provider_cache.py"], "run"), + ("provider-harness", ["tests/code_coverage_tests/test_provider_replay_harness.py"], "run"), + ("provider-harness", [".circleci/config.yml"], "run"), + ("provider-harness", [".circleci/scripts/classify_changes.sh"], "run"), + ("provider-harness", ["pyproject.toml"], "run"), + ("provider-harness", ["uv.lock"], "run"), + ("provider-harness", ["tests/e2e/PROVIDER_CACHE.md"], "skip"), + ("provider-harness", ["tests/e2e/ui/test_example.py"], "skip"), + ("provider-harness", ["tests/e2e/quota_management/test_quota.py"], "skip"), + ("provider-harness", ["litellm/main.py"], "skip"), + ("provider-harness", ["ui/litellm-dashboard/src/App.tsx"], "skip"), # docs-only: skip everything ("backend", DOCS, "skip"), ("client", DOCS, "skip"),