ci(e2e): run the step-log tests in the Code Quality workflow, drop overlapping unit tests

CircleCI does not gate every PR, so the two step-log harness test files move
from its provider_replay_harness job to a test_e2e_metadata step in the Code
Quality GitHub Actions workflow. The CircleCI config, its path classifier and
the classifier's test go back to their base versions.

Six unit tests in test_e2e_metadata.py asserted what test_e2e_junit_report.py
already pins through a real pytest run: call order, a raising helper's label
last, the per-test reset, repeated step properties and both attach cases.
They are removed; mutating each of those behaviours still fails the suite.
This commit is contained in:
ryan-crabbe-berri 2026-09-26 17:03:16 -07:00
parent a49940ccf3
commit b43e45d99d
6 changed files with 14 additions and 98 deletions

View file

@ -3100,9 +3100,7 @@ jobs:
tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \
tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \
tests/code_coverage_tests/test_provider_replay_harness.py \
tests/code_coverage_tests/test_provider_cache.py \
tests/code_coverage_tests/test_e2e_metadata.py \
tests/code_coverage_tests/test_e2e_junit_report.py
tests/code_coverage_tests/test_provider_cache.py
- store_test_results:
path: test-results/provider-replay-harness

View file

@ -19,7 +19,7 @@ while IFS= read -r file || [ -n "$file" ]; do
esac
case "$file" in
tests/e2e/*/*.py) : ;;
tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/code_coverage_tests/test_e2e_metadata.py | tests/code_coverage_tests/test_e2e_junit_report.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
has_provider_harness=true ;;
esac
case "$file" in

View file

@ -83,6 +83,11 @@ jobs:
- name: test_e2e_changed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
- name: test_e2e_metadata
env:
PYTHONPATH: tests/e2e
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
- name: router_code_coverage
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py

View file

@ -1,6 +1,6 @@
"""The JUnit report itself, written by a real pytest run.
No proxy. test_e2e_metadata.py pins the functions that build the properties;
No proxy. test_e2e_metadata.py pins the recorder's edge cases;
this pins what reaches the XML once pytest, its junitxml plugin,
pytest-rerunfailures and xdist are all in the loop. Each case writes a throwaway
suite into a tmp dir and runs it in a child interpreter with tests/e2e's

View file

@ -1,10 +1,11 @@
"""The e2e step log: what `@step` records, and how it attaches to a JUnit item.
"""The e2e step recorder's edge cases: dedupe, the cap, nesting, context managers.
Harness logic, so it lives here rather than under tests/e2e, which holds only
tests that drive a live proxy. The harness modules are imported off
``-o pythonpath=tests/e2e``, the way CI's provider_replay_harness job runs this
file. test_e2e_junit_report.py pins what reaches the XML through the real
conftest.
``PYTHONPATH=tests/e2e``, the way the Code Quality workflow's
test_e2e_metadata step runs this file. Call order, the failing test's last step,
the per-test reset and the JUnit attach are pinned end to end in
test_e2e_junit_report.py.
"""
from __future__ import annotations
@ -16,8 +17,7 @@ from contextlib import contextmanager
from pathlib import Path
import pytest
from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step, step_properties
from junit_properties import attach_result_properties, attach_step_properties
from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step
@pytest.fixture(autouse=True)
@ -29,11 +29,6 @@ def empty_step_log() -> Generator[None]:
STEPS.reset()
def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item:
"""The Item pytest collected for test ``name`` in this file, as pytest built it."""
return next(item for item in request.session.items if item.path == request.path and item.name == name)
class TestStepRecording:
"""`@step`-decorated harness helpers append to the running test's story as
they execute.
@ -43,19 +38,6 @@ class TestStepRecording:
live test.
"""
def test_steps_land_in_call_order(self) -> None:
@step("register deployment")
def register() -> str:
return "model-id"
@step("generate virtual key")
def generate() -> str:
return "sk-x"
_ = register()
_ = generate()
assert STEPS.taken() == ("register deployment", "generate virtual key")
def test_a_decorated_helper_still_returns_exactly_what_it_did(self) -> None:
"""`@step` records, it does not intercept: arguments, return value and
`__name__` all survive it, so decorating a live harness method cannot
@ -68,24 +50,6 @@ class TestStepRecording:
assert chat("sk-x", model="gpt-5.5") == "sk-x:gpt-5.5"
assert chat.__name__ == "chat"
def test_a_helper_that_raises_leaves_its_own_label_last(self) -> None:
"""The whole point of the field. The label is recorded BEFORE the call, so
a test that dies inside a helper keeps a partial story whose last element
names the helper it died in."""
@step("generate virtual key")
def generate() -> str:
return "sk-x"
@step("POST /chat/completions")
def chat() -> None:
raise RuntimeError("502 from upstream")
_ = generate()
with pytest.raises(RuntimeError, match="502 from upstream"):
chat()
assert STEPS.taken() == ("generate virtual key", "POST /chat/completions")
def test_a_poll_loop_is_one_step_in_the_story_not_fifty(self) -> None:
@step("poll /spend/logs for the request id")
def poll() -> None:
@ -126,22 +90,6 @@ class TestStepRecording:
STEPS.record(" ")
assert STEPS.taken() == ("POST /chat/completions",)
def test_reset_empties_the_log_so_one_test_never_inherits_another_s(self) -> None:
STEPS.record("register deployment")
STEPS.reset()
assert STEPS.taken() == ()
assert step_properties() == ()
def test_steps_serialize_as_repeated_properties_in_order(self) -> None:
"""Repeated rather than joined on a delimiter: the labels are free text, so
no separator can be reserved, and a repeated property has none to corrupt."""
STEPS.record('attach guardrail, comma & "quoted" <tag>')
STEPS.record("POST /chat/completions")
assert step_properties() == (
("step", 'attach guardrail, comma & "quoted" <tag>'),
("step", "POST /chat/completions"),
)
def test_a_decorated_helper_warns_at_its_caller_with_step_frames(self) -> None:
"""`stacklevel` counts frames, and the wrapper is one of them: a cleanup
helper that warns about its caller would otherwise report every warning at
@ -307,35 +255,3 @@ class TestContextManagerSteps:
with pytest.raises(TypeError, match="cannot wrap the generator function"):
_ = step("poll /spend/logs")(rows)
class TestAttachStepProperties:
def test_steps_are_appended_after_the_collected_properties(self, request: pytest.FixtureRequest) -> None:
"""Order inside `<properties>` is list order, so the story reads after the
fixed prefix the collection hook already attached."""
test = type(self).test_steps_are_appended_after_the_collected_properties
item = collected_item(request, test.__name__)
attach_result_properties(item)
STEPS.record("register deployment")
STEPS.record("POST /chat/completions")
attach_step_properties(item)
assert [name for name, _ in item.user_properties] == ["package", "covers", "source", "step", "step"]
assert [value for name, value in item.user_properties if name == "step"] == [
"register deployment",
"POST /chat/completions",
]
def test_a_rerun_replaces_the_story_rather_than_appending_a_second_one(
self, request: pytest.FixtureRequest
) -> None:
"""The suite runs with `--reruns 1`. Without this the retry's steps would
queue up behind the first attempt's and the report would read as one test
that did everything twice."""
test = type(self).test_a_rerun_replaces_the_story_rather_than_appending_a_second_one
item = collected_item(request, test.__name__)
STEPS.record("attempt one died here")
attach_step_properties(item)
STEPS.reset()
STEPS.record("attempt two got further")
attach_step_properties(item)
assert [value for name, value in item.user_properties if name == "step"] == ["attempt two got further"]

View file

@ -64,9 +64,6 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"]
("provider-harness", ["tests/e2e/e2e_http.py"], "run"),
("provider-harness", ["tests/code_coverage_tests/test_provider_cache.py"], "run"),
("provider-harness", ["tests/code_coverage_tests/test_provider_replay_harness.py"], "run"),
("provider-harness", ["tests/code_coverage_tests/test_e2e_metadata.py"], "run"),
("provider-harness", ["tests/code_coverage_tests/test_e2e_junit_report.py"], "run"),
("provider-harness", ["tests/e2e/e2e_metadata.py"], "run"),
("provider-harness", [".circleci/config.yml"], "run"),
("provider-harness", [".circleci/scripts/classify_changes.sh"], "run"),
("provider-harness", ["pyproject.toml"], "run"),