diff --git a/.circleci/config.yml b/.circleci/config.yml index 161d6c4d793..1485f517164 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3100,9 +3100,7 @@ jobs: tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \ tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \ tests/code_coverage_tests/test_provider_replay_harness.py \ - tests/code_coverage_tests/test_provider_cache.py \ - tests/code_coverage_tests/test_e2e_metadata.py \ - tests/code_coverage_tests/test_e2e_junit_report.py + tests/code_coverage_tests/test_provider_cache.py - store_test_results: path: test-results/provider-replay-harness diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index ce5f2320b3c..01bc8290199 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -19,7 +19,7 @@ while IFS= read -r file || [ -n "$file" ]; do esac case "$file" in tests/e2e/*/*.py) : ;; - tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/code_coverage_tests/test_e2e_metadata.py | tests/code_coverage_tests/test_e2e_junit_report.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock) + tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock) has_provider_harness=true ;; esac case "$file" in diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index 987f66773f2..12e3a900771 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -83,6 +83,11 @@ jobs: - name: test_e2e_changed_gate run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py + - name: test_e2e_metadata + env: + PYTHONPATH: tests/e2e + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py + - name: router_code_coverage run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py diff --git a/tests/code_coverage_tests/test_e2e_junit_report.py b/tests/code_coverage_tests/test_e2e_junit_report.py index 9fd9c60a52d..4d9faaf21b6 100644 --- a/tests/code_coverage_tests/test_e2e_junit_report.py +++ b/tests/code_coverage_tests/test_e2e_junit_report.py @@ -1,6 +1,6 @@ """The JUnit report itself, written by a real pytest run. -No proxy. test_e2e_metadata.py pins the functions that build the properties; +No proxy. test_e2e_metadata.py pins the recorder's edge cases; this pins what reaches the XML once pytest, its junitxml plugin, pytest-rerunfailures and xdist are all in the loop. Each case writes a throwaway suite into a tmp dir and runs it in a child interpreter with tests/e2e's diff --git a/tests/code_coverage_tests/test_e2e_metadata.py b/tests/code_coverage_tests/test_e2e_metadata.py index fe433b91b3d..af74d56f90c 100644 --- a/tests/code_coverage_tests/test_e2e_metadata.py +++ b/tests/code_coverage_tests/test_e2e_metadata.py @@ -1,10 +1,11 @@ -"""The e2e step log: what `@step` records, and how it attaches to a JUnit item. +"""The e2e step recorder's edge cases: dedupe, the cap, nesting, context managers. Harness logic, so it lives here rather than under tests/e2e, which holds only tests that drive a live proxy. The harness modules are imported off -``-o pythonpath=tests/e2e``, the way CI's provider_replay_harness job runs this -file. test_e2e_junit_report.py pins what reaches the XML through the real -conftest. +``PYTHONPATH=tests/e2e``, the way the Code Quality workflow's +test_e2e_metadata step runs this file. Call order, the failing test's last step, +the per-test reset and the JUnit attach are pinned end to end in +test_e2e_junit_report.py. """ from __future__ import annotations @@ -16,8 +17,7 @@ from contextlib import contextmanager from pathlib import Path import pytest -from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step, step_properties -from junit_properties import attach_result_properties, attach_step_properties +from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step @pytest.fixture(autouse=True) @@ -29,11 +29,6 @@ def empty_step_log() -> Generator[None]: STEPS.reset() -def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item: - """The Item pytest collected for test ``name`` in this file, as pytest built it.""" - return next(item for item in request.session.items if item.path == request.path and item.name == name) - - class TestStepRecording: """`@step`-decorated harness helpers append to the running test's story as they execute. @@ -43,19 +38,6 @@ class TestStepRecording: live test. """ - def test_steps_land_in_call_order(self) -> None: - @step("register deployment") - def register() -> str: - return "model-id" - - @step("generate virtual key") - def generate() -> str: - return "sk-x" - - _ = register() - _ = generate() - assert STEPS.taken() == ("register deployment", "generate virtual key") - def test_a_decorated_helper_still_returns_exactly_what_it_did(self) -> None: """`@step` records, it does not intercept: arguments, return value and `__name__` all survive it, so decorating a live harness method cannot @@ -68,24 +50,6 @@ class TestStepRecording: assert chat("sk-x", model="gpt-5.5") == "sk-x:gpt-5.5" assert chat.__name__ == "chat" - def test_a_helper_that_raises_leaves_its_own_label_last(self) -> None: - """The whole point of the field. The label is recorded BEFORE the call, so - a test that dies inside a helper keeps a partial story whose last element - names the helper it died in.""" - - @step("generate virtual key") - def generate() -> str: - return "sk-x" - - @step("POST /chat/completions") - def chat() -> None: - raise RuntimeError("502 from upstream") - - _ = generate() - with pytest.raises(RuntimeError, match="502 from upstream"): - chat() - assert STEPS.taken() == ("generate virtual key", "POST /chat/completions") - def test_a_poll_loop_is_one_step_in_the_story_not_fifty(self) -> None: @step("poll /spend/logs for the request id") def poll() -> None: @@ -126,22 +90,6 @@ class TestStepRecording: STEPS.record(" ") assert STEPS.taken() == ("POST /chat/completions",) - def test_reset_empties_the_log_so_one_test_never_inherits_another_s(self) -> None: - STEPS.record("register deployment") - STEPS.reset() - assert STEPS.taken() == () - assert step_properties() == () - - def test_steps_serialize_as_repeated_properties_in_order(self) -> None: - """Repeated rather than joined on a delimiter: the labels are free text, so - no separator can be reserved, and a repeated property has none to corrupt.""" - STEPS.record('attach guardrail, comma & "quoted" ') - STEPS.record("POST /chat/completions") - assert step_properties() == ( - ("step", 'attach guardrail, comma & "quoted" '), - ("step", "POST /chat/completions"), - ) - def test_a_decorated_helper_warns_at_its_caller_with_step_frames(self) -> None: """`stacklevel` counts frames, and the wrapper is one of them: a cleanup helper that warns about its caller would otherwise report every warning at @@ -307,35 +255,3 @@ class TestContextManagerSteps: with pytest.raises(TypeError, match="cannot wrap the generator function"): _ = step("poll /spend/logs")(rows) - - -class TestAttachStepProperties: - def test_steps_are_appended_after_the_collected_properties(self, request: pytest.FixtureRequest) -> None: - """Order inside `` is list order, so the story reads after the - fixed prefix the collection hook already attached.""" - test = type(self).test_steps_are_appended_after_the_collected_properties - item = collected_item(request, test.__name__) - attach_result_properties(item) - STEPS.record("register deployment") - STEPS.record("POST /chat/completions") - attach_step_properties(item) - assert [name for name, _ in item.user_properties] == ["package", "covers", "source", "step", "step"] - assert [value for name, value in item.user_properties if name == "step"] == [ - "register deployment", - "POST /chat/completions", - ] - - def test_a_rerun_replaces_the_story_rather_than_appending_a_second_one( - self, request: pytest.FixtureRequest - ) -> None: - """The suite runs with `--reruns 1`. Without this the retry's steps would - queue up behind the first attempt's and the report would read as one test - that did everything twice.""" - test = type(self).test_a_rerun_replaces_the_story_rather_than_appending_a_second_one - item = collected_item(request, test.__name__) - STEPS.record("attempt one died here") - attach_step_properties(item) - STEPS.reset() - STEPS.record("attempt two got further") - attach_step_properties(item) - assert [value for name, value in item.user_properties if name == "step"] == ["attempt two got further"] diff --git a/tests/test_litellm/test_circleci_path_filter.py b/tests/test_litellm/test_circleci_path_filter.py index b72d85cbafd..84e2327057d 100644 --- a/tests/test_litellm/test_circleci_path_filter.py +++ b/tests/test_litellm/test_circleci_path_filter.py @@ -64,9 +64,6 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"] ("provider-harness", ["tests/e2e/e2e_http.py"], "run"), ("provider-harness", ["tests/code_coverage_tests/test_provider_cache.py"], "run"), ("provider-harness", ["tests/code_coverage_tests/test_provider_replay_harness.py"], "run"), - ("provider-harness", ["tests/code_coverage_tests/test_e2e_metadata.py"], "run"), - ("provider-harness", ["tests/code_coverage_tests/test_e2e_junit_report.py"], "run"), - ("provider-harness", ["tests/e2e/e2e_metadata.py"], "run"), ("provider-harness", [".circleci/config.yml"], "run"), ("provider-harness", [".circleci/scripts/classify_changes.sh"], "run"), ("provider-harness", ["pyproject.toml"], "run"),