diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c241af5853..6b5d7a67189 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2915,6 +2915,25 @@ jobs: exit 1 fi + provider_replay_harness: + docker: + - *python312_image + working_directory: ~/project + resource_class: medium + steps: + - setup_litellm_test_deps + - run: + name: Test provider replay harness + command: | + mkdir -p test-results/provider-replay-harness + uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \ + --junitxml=test-results/provider-replay-harness/junit.xml \ + tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \ + tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \ + tests/code_coverage_tests/test_provider_replay_harness.py + - store_test_results: + path: test-results/provider-replay-harness + integration_contracts: parameters: suite: @@ -2967,6 +2986,8 @@ workflows: only: - main - /litellm_.*/ + - provider_replay_harness: + filters: *main_branches - base_sdk_install: filters: *main_branches - local_testing_part1: diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index e07593cdfb6..987f66773f2 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -83,14 +83,6 @@ jobs: - name: test_e2e_changed_gate run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py - - name: test_provider_replay_harness - run: | - pwd - uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \ - tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \ - tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \ - tests/code_coverage_tests/test_provider_replay_harness.py - - name: router_code_coverage run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 5beaf68df79..53a05931ca7 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -247,4 +247,4 @@ The semantic header set is `content-type`, `accept`, `anthropic-version`, `anthr Excluded transport and telemetry headers are `host`, `content-length`, `connection`, `accept-encoding`, `user-agent`, `traceparent`, `tracestate`, `x-request-id`, `x-client-request-id` and `x-stainless-*`. Inbound transfer-encoding is unsupported; send JSON with content-length framing. The destination represents host identity and the relay carries original body bytes. Replay does not verify credentials, SDK timeout/retry behavior, transport performance, model availability or stateful remote IDs. Live relay uses original request bytes and header values, never the stored identity -Strict replay harness regression tests live in `tests/code_coverage_tests/test_provider_replay_harness.py`. The `test_provider_replay_harness` code-quality step runs them alongside the existing legacy harness files with `--noconftest -o pythonpath=tests/e2e`; they need only synthetic HTTP providers and temporary fixture storage +Strict replay harness regression tests live in `tests/code_coverage_tests/test_provider_replay_harness.py`. The CircleCI `provider_replay_harness` job runs them alongside the existing legacy harness files with `--noconftest -o pythonpath=tests/e2e`; they need only synthetic HTTP providers and temporary fixture storage