diff --git a/.circleci/config.yml b/.circleci/config.yml index 1485f517164..161d6c4d793 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3100,7 +3100,9 @@ jobs: tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \ tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \ tests/code_coverage_tests/test_provider_replay_harness.py \ - tests/code_coverage_tests/test_provider_cache.py + tests/code_coverage_tests/test_provider_cache.py \ + tests/code_coverage_tests/test_e2e_metadata.py \ + tests/code_coverage_tests/test_e2e_junit_report.py - store_test_results: path: test-results/provider-replay-harness diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index 01bc8290199..ce5f2320b3c 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -19,7 +19,7 @@ while IFS= read -r file || [ -n "$file" ]; do esac case "$file" in tests/e2e/*/*.py) : ;; - tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock) + tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/code_coverage_tests/test_e2e_metadata.py | tests/code_coverage_tests/test_e2e_junit_report.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock) has_provider_harness=true ;; esac case "$file" in diff --git a/tests/e2e/test_junit_report.py b/tests/code_coverage_tests/test_e2e_junit_report.py similarity index 93% rename from tests/e2e/test_junit_report.py rename to tests/code_coverage_tests/test_e2e_junit_report.py index 508c4e9f417..9fd9c60a52d 100644 --- a/tests/e2e/test_junit_report.py +++ b/tests/code_coverage_tests/test_e2e_junit_report.py @@ -1,11 +1,11 @@ """The JUnit report itself, written by a real pytest run. -No proxy and no ``e2e`` marker. test_junit_properties.py pins the functions that -build the properties; this pins what reaches the XML once pytest, its junitxml -plugin, pytest-rerunfailures and xdist are all in the loop. Each case writes a -throwaway suite into a tmp dir and runs it in a child interpreter with THIS -directory's conftest.py loaded as a plugin, so the hooks under test are the ones -the live suite runs and the recorder is the real one, never a copy of either. +No proxy. test_e2e_metadata.py pins the functions that build the properties; +this pins what reaches the XML once pytest, its junitxml plugin, +pytest-rerunfailures and xdist are all in the loop. Each case writes a throwaway +suite into a tmp dir and runs it in a child interpreter with tests/e2e's +conftest.py loaded as a plugin, so the hooks under test are the ones the live +suite runs and the recorder is the real one, never a copy of either. The timing that makes the recorded half work is pytest's, which is why it is pinned here against the real thing: junitxml writes a testcase's properties from @@ -29,7 +29,7 @@ from xml.etree import ElementTree import pytest -SUITE_DIR: Final = Path(__file__).resolve().parent +SUITE_DIR: Final = Path(__file__).resolve().parents[1] / "e2e" CHILD_TIMEOUT_SECONDS: Final = 180 STORY_SUITE: Final = """ @@ -154,7 +154,7 @@ def write_suite(directory: Path, modules: Mapping[str, str]) -> None: """Lay a child suite out in ``directory``, with an ini file of its own. The ini pins the child's rootdir to the tmp dir wherever that lives, and its - ``pythonpath`` is what makes this directory's conftest.py, and the harness + ``pythonpath`` is what makes tests/e2e's conftest.py, and the harness modules the child suite imports, importable under ``-I``. """ _ = (directory / "pytest.ini").write_text(f"[pytest]\npythonpath = {shlex.quote(str(SUITE_DIR))}\n") @@ -165,7 +165,7 @@ def write_suite(directory: Path, modules: Mapping[str, str]) -> None: def run_child_pytest(suite: Path, *args: str) -> subprocess.CompletedProcess[str]: """Run pytest over ``suite`` in a fresh interpreter, hooked up like the live suite. - ``-p conftest`` registers this directory's conftest.py as a plugin, since a + ``-p conftest`` registers tests/e2e's conftest.py as a plugin, since a tmp dir outside tests/e2e would never pick it up by location. The parent's fixture-mode and addopts settings are dropped so a replay lane cannot leak into the child. diff --git a/tests/code_coverage_tests/test_e2e_metadata.py b/tests/code_coverage_tests/test_e2e_metadata.py new file mode 100644 index 00000000000..fe433b91b3d --- /dev/null +++ b/tests/code_coverage_tests/test_e2e_metadata.py @@ -0,0 +1,341 @@ +"""The e2e step log: what `@step` records, and how it attaches to a JUnit item. + +Harness logic, so it lives here rather than under tests/e2e, which holds only +tests that drive a live proxy. The harness modules are imported off +``-o pythonpath=tests/e2e``, the way CI's provider_replay_harness job runs this +file. test_e2e_junit_report.py pins what reaches the XML through the real +conftest. +""" + +from __future__ import annotations + +import threading +import warnings +from collections.abc import Generator +from contextlib import contextmanager +from pathlib import Path + +import pytest +from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step, step_properties +from junit_properties import attach_result_properties, attach_step_properties + + +@pytest.fixture(autouse=True) +def empty_step_log() -> Generator[None]: + """Each test starts from an empty log and leaves none behind, as conftest's + `pytest_runtest_setup` hook arranges for every live test.""" + STEPS.reset() + yield + STEPS.reset() + + +def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item: + """The Item pytest collected for test ``name`` in this file, as pytest built it.""" + return next(item for item in request.session.items if item.path == request.path and item.name == name) + + +class TestStepRecording: + """`@step`-decorated harness helpers append to the running test's story as + they execute. + + Each test here starts from an empty log because `empty_step_log` resets the + recorder first, the same reset conftest's `pytest_runtest_setup` gives every + live test. + """ + + def test_steps_land_in_call_order(self) -> None: + @step("register deployment") + def register() -> str: + return "model-id" + + @step("generate virtual key") + def generate() -> str: + return "sk-x" + + _ = register() + _ = generate() + assert STEPS.taken() == ("register deployment", "generate virtual key") + + def test_a_decorated_helper_still_returns_exactly_what_it_did(self) -> None: + """`@step` records, it does not intercept: arguments, return value and + `__name__` all survive it, so decorating a live harness method cannot + change what the test observes.""" + + @step("POST /chat/completions") + def chat(key: str, *, model: str) -> str: + return f"{key}:{model}" + + assert chat("sk-x", model="gpt-5.5") == "sk-x:gpt-5.5" + assert chat.__name__ == "chat" + + def test_a_helper_that_raises_leaves_its_own_label_last(self) -> None: + """The whole point of the field. The label is recorded BEFORE the call, so + a test that dies inside a helper keeps a partial story whose last element + names the helper it died in.""" + + @step("generate virtual key") + def generate() -> str: + return "sk-x" + + @step("POST /chat/completions") + def chat() -> None: + raise RuntimeError("502 from upstream") + + _ = generate() + with pytest.raises(RuntimeError, match="502 from upstream"): + chat() + assert STEPS.taken() == ("generate virtual key", "POST /chat/completions") + + def test_a_poll_loop_is_one_step_in_the_story_not_fifty(self) -> None: + @step("poll /spend/logs for the request id") + def poll() -> None: + return None + + for _ in range(20): + poll() + assert STEPS.taken() == ("poll /spend/logs for the request id",) + + def test_the_same_label_recorded_again_later_is_a_new_step(self) -> None: + """Only CONSECUTIVE duplicates collapse; a helper called again after + something else happened is a genuine second beat of the story.""" + STEPS.record("POST /chat/completions") + STEPS.record("poll /spend/logs") + STEPS.record("POST /chat/completions") + assert STEPS.taken() == ("POST /chat/completions", "poll /spend/logs", "POST /chat/completions") + + def test_a_full_log_keeps_the_latest_steps_so_the_last_is_where_the_test_died(self) -> None: + """A load test cannot bury the story in thousands of entries, and the cap + drops from the front: the step a test died on is the newest, so it is the + one that has to survive. The leading line says the story is partial.""" + for index in range(MAX_STEPS + 10): + STEPS.record(f"call {index}") + assert STEPS.taken() == ( + "(10 earlier steps not recorded)", + *(f"call {index}" for index in range(10, MAX_STEPS + 10)), + ) + + def test_reset_forgets_what_a_full_log_dropped(self) -> None: + for index in range(MAX_STEPS + 1): + STEPS.record(f"call {index}") + STEPS.reset() + STEPS.record("register deployment") + assert STEPS.taken() == ("register deployment",) + + def test_whitespace_is_normalized_and_an_empty_label_records_nothing(self) -> None: + STEPS.record(" POST /chat/completions\n ") + STEPS.record(" ") + assert STEPS.taken() == ("POST /chat/completions",) + + def test_reset_empties_the_log_so_one_test_never_inherits_another_s(self) -> None: + STEPS.record("register deployment") + STEPS.reset() + assert STEPS.taken() == () + assert step_properties() == () + + def test_steps_serialize_as_repeated_properties_in_order(self) -> None: + """Repeated rather than joined on a delimiter: the labels are free text, so + no separator can be reserved, and a repeated property has none to corrupt.""" + STEPS.record('attach guardrail, comma & "quoted" ') + STEPS.record("POST /chat/completions") + assert step_properties() == ( + ("step", 'attach guardrail, comma & "quoted" '), + ("step", "POST /chat/completions"), + ) + + def test_a_decorated_helper_warns_at_its_caller_with_step_frames(self) -> None: + """`stacklevel` counts frames, and the wrapper is one of them: a cleanup + helper that warns about its caller would otherwise report every warning at + e2e_metadata.py. Pins `STEP_FRAMES` to the frames the wrapper really adds.""" + + @step("delete team") + def delete_team() -> None: + warnings.warn("delete_team('t') failed", stacklevel=2 + STEP_FRAMES) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + delete_team() + assert [Path(warning.filename).name for warning in caught] == [Path(__file__).name] + + +class TestNestedSteps: + """Harness layers call each other, so a step's helper routinely calls other + decorated helpers. Only the outermost records.""" + + def test_a_step_called_inside_a_step_is_not_recorded(self) -> None: + """`ResourceManager.key` wraps `ProxyClient.generate_key`: one action, one + beat of the story, at the level the test called in at.""" + + @step("POST /key/generate") + def generate_key() -> str: + return "sk-x" + + @step("generate virtual key") + def key() -> str: + return generate_key() + + assert key() == "sk-x" + assert STEPS.taken() == ("generate virtual key",) + + def test_the_inner_step_records_again_once_the_outer_one_returns(self) -> None: + @step("POST /key/generate") + def generate_key() -> str: + return "sk-x" + + @step("generate virtual key") + def key() -> str: + return generate_key() + + _ = key() + _ = generate_key() + assert STEPS.taken() == ("generate virtual key", "POST /key/generate") + + def test_an_inner_step_that_raises_leaves_the_outer_label_last_and_unwinds(self) -> None: + """The helper the test called is where it died, and the nesting flag is + released on the way out, so the next top-level call still records.""" + + @step("POST /team/new") + def post_team() -> None: + raise RuntimeError("/team/new answered 500") + + @step("create team with a budget") + def create_team() -> None: + post_team() + + @step("POST /chat/completions") + def chat() -> None: + return None + + with pytest.raises(RuntimeError, match="answered 500"): + create_team() + chat() + assert STEPS.taken() == ("create team with a budget", "POST /chat/completions") + + def test_a_worker_thread_a_step_fans_out_to_records_its_own_steps(self) -> None: + """Nesting is per thread: a load helper that fans chats out to workers is + not inside a step on those workers, so their calls are still recorded.""" + + @step("POST /chat/completions") + def chat() -> None: + return None + + @step("fire concurrent chats") + def fan_out() -> None: + worker = threading.Thread(target=chat) + worker.start() + worker.join() + + fan_out() + assert STEPS.taken() == ("fire concurrent chats", "POST /chat/completions") + + +class TestContextManagerSteps: + """A `@contextmanager` helper's setup and cleanup run at `__enter__` and + `__exit__`, after the decorated call has returned. Both still count as part + of its step; the `with` body is the test's own code and records as usual.""" + + def test_setup_and_cleanup_stay_inside_the_step_and_the_body_records(self) -> None: + @step("run a SQL statement") + def execute() -> None: + return None + + @step("create a read-only database role") + @contextmanager + def restricted_user() -> Generator[str]: + execute() + try: + yield "reader" + finally: + execute() + + @step("POST /chat/completions") + def chat() -> None: + return None + + with restricted_user() as user: + assert user == "reader" + chat() + assert STEPS.taken() == ("create a read-only database role", "POST /chat/completions") + + def test_a_test_that_dies_in_the_with_body_keeps_its_last_step_last(self) -> None: + """The guarantee the field makes: the cleanup that runs on the way out of + the `with` must not append a step behind the one the test died on.""" + + @step("drop the role") + def drop_role() -> None: + return None + + @step("create a read-only database role") + @contextmanager + def restricted_user() -> Generator[None]: + try: + yield + finally: + drop_role() + + @step("POST /chat/completions") + def chat() -> None: + raise RuntimeError("502 from upstream") + + with pytest.raises(RuntimeError, match="502 from upstream"), restricted_user(): + chat() + assert STEPS.taken() == ("create a read-only database role", "POST /chat/completions") + + def test_the_wrapped_context_keeps_its_exception_handling(self) -> None: + """`__exit__` is forwarded, return value included, so a context that + suppresses an exception still does.""" + + @step("hold an advisory lock") + @contextmanager + def swallowing() -> Generator[None]: + try: + yield + except KeyError: + pass + + with swallowing(): + raise KeyError("suppressed by the context") + assert STEPS.taken() == ("hold an advisory lock",) + + def test_a_bare_generator_is_refused_where_the_decorator_runs(self) -> None: + """Its body runs only as the caller iterates, interleaved with the caller's + own steps, so no single point in the story is where it happened. Refused at + decoration, which for a harness module is import, so it lands as a + collection error rather than a story that quietly reads out of order.""" + + def rows() -> Generator[int]: + yield 1 + + with pytest.raises(TypeError, match="cannot wrap the generator function"): + _ = step("poll /spend/logs")(rows) + + +class TestAttachStepProperties: + def test_steps_are_appended_after_the_collected_properties(self, request: pytest.FixtureRequest) -> None: + """Order inside `` is list order, so the story reads after the + fixed prefix the collection hook already attached.""" + test = type(self).test_steps_are_appended_after_the_collected_properties + item = collected_item(request, test.__name__) + attach_result_properties(item) + STEPS.record("register deployment") + STEPS.record("POST /chat/completions") + attach_step_properties(item) + assert [name for name, _ in item.user_properties] == ["package", "covers", "source", "step", "step"] + assert [value for name, value in item.user_properties if name == "step"] == [ + "register deployment", + "POST /chat/completions", + ] + + def test_a_rerun_replaces_the_story_rather_than_appending_a_second_one( + self, request: pytest.FixtureRequest + ) -> None: + """The suite runs with `--reruns 1`. Without this the retry's steps would + queue up behind the first attempt's and the report would read as one test + that did everything twice.""" + test = type(self).test_a_rerun_replaces_the_story_rather_than_appending_a_second_one + item = collected_item(request, test.__name__) + STEPS.record("attempt one died here") + attach_step_properties(item) + STEPS.reset() + STEPS.record("attempt two got further") + attach_step_properties(item) + assert [value for name, value in item.user_properties if name == "step"] == ["attempt two got further"] diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 3bdc8af2f52..575512b13a2 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -134,9 +134,9 @@ The harness is fully typed with no error budget: `make lint-e2e-basedpyright` mu `@step("POST /chat/completions")` from `e2e_metadata.py` goes on HARNESS helpers - client methods, `ResourceManager.key`, poll loops - never on a test, and appends its label to the running test's `user_properties` in call order. The list IS the test's user story, and because the label is recorded BEFORE the wrapped call, a failing test's LAST step is where it died. Nothing about steps is hand-written: the call sequence cannot drift from what the test actually did. A new public harness method that performs an action (an HTTP call, a poll, a login, a CLI run) gets a `@step`; pure builders, parsers and `_private` helpers do not. Labels are static, lowercase, one beat of the story: a plain-English action ("create team with a budget") or, for a raw route call, the route itself ("POST /v1/messages") -Only the outermost step records. Harness layers call each other - `ResourceManager.key` goes through `ProxyClient.generate_key`, a domain client wraps the shared `ProxyClient` - so every layer carries its own label and the story still reads at the level the test called in at, one beat per action. On a `@contextmanager` helper `@step` goes ABOVE `@contextmanager`: the setup and cleanup around its `yield` count as part of the step, while the `with` body - the test's own code - records as usual, so cleanup never lands behind the step a test died on. A bare generator function is refused at import, since its body interleaves with the caller's. A decorated helper that warns about its caller uses `stacklevel=2 + STEP_FRAMES`, because the wrapper is a frame too. Nesting is tracked per thread, so a helper that fans work out to worker threads still records their steps. Consecutive duplicates collapse and the log caps at 50 entries, so a poll loop is one beat rather than fifty. The log is emptied first thing in every test's setup phase and attached after setup and again after call, so a test that errors in a fixture keeps the steps recorded before the crash. Teardown steps are left out on purpose: they are cleanup, and listing them would put a finalizer's step after the one a failing test died on +Only the outermost step records. Harness layers call each other - `ResourceManager.key` goes through `ProxyClient.generate_key`, a domain client wraps the shared `ProxyClient` - so every layer carries its own label and the story still reads at the level the test called in at, one beat per action. On a `@contextmanager` helper `@step` goes ABOVE `@contextmanager`: the setup and cleanup around its `yield` count as part of the step, while the `with` body - the test's own code - records as usual, so cleanup never lands behind the step a test died on. A bare generator function is refused at import, since its body interleaves with the caller's. A decorated helper that warns about its caller uses `stacklevel=2 + STEP_FRAMES`, because the wrapper is a frame too. Nesting is tracked per thread, so a helper that fans work out to worker threads still records their steps. Consecutive duplicates collapse, so a poll loop is one beat rather than fifty, and the log keeps the latest 50 steps behind a line counting the ones it dropped: the cap drops from the front because the last step is where a failing test died. The log is emptied first thing in every test's setup phase and attached after setup and again after call, so a test that errors in a fixture keeps the steps recorded before the crash. Teardown steps are left out on purpose: they are cleanup, and listing them would put a finalizer's step after the one a failing test died on -Steps ride out as repeated JUnit `` entries (`junit_properties.py`), one per step rather than one delimiter-joined value, since a free-text label has no separator that can be reserved. The results JSON downstream regroups them into a `steps` array. `test_junit_report.py` runs real pytest with `--junitxml`, in-process and under `-n 2`, and pins what reaches the XML +Steps ride out as repeated JUnit `` entries (`junit_properties.py`), one per step rather than one delimiter-joined value, since a free-text label has no separator that can be reserved. The results JSON downstream regroups them into a `steps` array. The harness tests for it sit outside the suite, in `tests/code_coverage_tests/test_e2e_metadata.py` and `test_e2e_junit_report.py`; the latter runs real pytest with `--junitxml` through this conftest, in-process and under `-n 2`, and pins what reaches the XML ## Coverage registry diff --git a/tests/e2e/e2e_metadata.py b/tests/e2e/e2e_metadata.py index 46911745871..edcbc5dc469 100644 --- a/tests/e2e/e2e_metadata.py +++ b/tests/e2e/e2e_metadata.py @@ -14,6 +14,7 @@ from __future__ import annotations import inspect import threading +from collections import deque from collections.abc import Callable, Generator from contextlib import AbstractContextManager, contextmanager from functools import wraps @@ -44,21 +45,25 @@ class _StepRecorder: def __init__(self) -> None: self._lock = threading.Lock() - self._steps: list[str] = [] + self._steps: deque[str] = deque(maxlen=MAX_STEPS) + self._dropped = 0 def reset(self) -> None: """Called first thing in every test's setup phase, so each test starts empty.""" with self._lock: self._steps.clear() + self._dropped = 0 def record(self, label: str) -> None: - """Append `label`, unless it repeats the previous step or the log is full. + """Append `label`, unless it repeats the previous step. A retrying helper (poll_cost_row) or a load test calling a decorated helper in a loop would otherwise emit thousands of entries per testcase: a consecutive repeat collapses, so a poll loop is one step in - the story rather than fifty, and the log stops growing at MAX_STEPS. + the story rather than fifty, and past MAX_STEPS the oldest step makes way. + It is the oldest that goes because the last step is the one that has to + survive: it is where a failing test died. """ cleaned = " ".join(label.split())[:MAX_STEP_CHARS] if not cleaned: @@ -66,13 +71,16 @@ class _StepRecorder: with self._lock: if self._steps and self._steps[-1] == cleaned: return - if len(self._steps) >= MAX_STEPS: - return + if len(self._steps) == MAX_STEPS: + self._dropped += 1 self._steps.append(cleaned) def taken(self) -> tuple[str, ...]: + """The story so far, led by a line counting the steps a full log dropped, + so a story that starts mid-test says so rather than reading as complete.""" with self._lock: - return tuple(self._steps) + dropped: Final = (f"({self._dropped} earlier steps not recorded)",) if self._dropped else () + return dropped + tuple(self._steps) STEPS: Final = _StepRecorder() diff --git a/tests/e2e/test_junit_properties.py b/tests/e2e/test_junit_properties.py index 9b660470309..02c1413c840 100644 --- a/tests/e2e/test_junit_properties.py +++ b/tests/e2e/test_junit_properties.py @@ -10,18 +10,12 @@ rollups and, for ``source``, the status page's per-test links to GitHub. from __future__ import annotations -import threading -import warnings -from collections.abc import Generator -from contextlib import contextmanager from pathlib import Path import pytest -from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step, step_properties from junit_properties import ( SUITE_ROOT, attach_result_properties, - attach_step_properties, dedupe_covers, package_from_nodeid, result_properties, @@ -135,298 +129,3 @@ class TestSuiteRoot: class TestDedupeCovers: def test_ids_are_unique_order_preserving_and_non_empty_strings(self) -> None: assert dedupe_covers([("A", "B"), ("B", ""), ("C", 7)]) == ("A", "B", "C") - - -class TestStepRecording: - """`@step`-decorated harness helpers append to the running test's story as - they execute. - - Each test here starts from an empty log because conftest's - `pytest_runtest_setup` hook resets the recorder first thing in every test's - setup -- the same reset the live suite relies on for per-test isolation. - """ - - def test_steps_land_in_call_order(self) -> None: - @step("register deployment") - def register() -> str: - return "model-id" - - @step("generate virtual key") - def generate() -> str: - return "sk-x" - - _ = register() - _ = generate() - assert STEPS.taken() == ("register deployment", "generate virtual key") - - def test_a_decorated_helper_still_returns_exactly_what_it_did(self) -> None: - """`@step` records, it does not intercept: arguments, return value and - `__name__` all survive it, so decorating a live harness method cannot - change what the test observes.""" - - @step("POST /chat/completions") - def chat(key: str, *, model: str) -> str: - return f"{key}:{model}" - - assert chat("sk-x", model="gpt-5.5") == "sk-x:gpt-5.5" - assert chat.__name__ == "chat" - - def test_a_helper_that_raises_leaves_its_own_label_last(self) -> None: - """The whole point of the field. The label is recorded BEFORE the call, so - a test that dies inside a helper keeps a partial story whose last element - names the helper it died in.""" - - @step("generate virtual key") - def generate() -> str: - return "sk-x" - - @step("POST /chat/completions") - def chat() -> None: - raise RuntimeError("502 from upstream") - - _ = generate() - with pytest.raises(RuntimeError, match="502 from upstream"): - chat() - assert STEPS.taken() == ("generate virtual key", "POST /chat/completions") - - def test_a_poll_loop_is_one_step_in_the_story_not_fifty(self) -> None: - @step("poll /spend/logs for the request id") - def poll() -> None: - return None - - for _ in range(20): - poll() - assert STEPS.taken() == ("poll /spend/logs for the request id",) - - def test_the_same_label_recorded_again_later_is_a_new_step(self) -> None: - """Only CONSECUTIVE duplicates collapse; a helper called again after - something else happened is a genuine second beat of the story.""" - STEPS.record("POST /chat/completions") - STEPS.record("poll /spend/logs") - STEPS.record("POST /chat/completions") - assert STEPS.taken() == ("POST /chat/completions", "poll /spend/logs", "POST /chat/completions") - - def test_the_log_is_capped_so_a_load_test_cannot_bury_the_story(self) -> None: - for index in range(MAX_STEPS * 2): - STEPS.record(f"call {index}") - taken = STEPS.taken() - assert len(taken) == MAX_STEPS - assert taken[0] == "call 0" - - def test_whitespace_is_normalized_and_an_empty_label_records_nothing(self) -> None: - STEPS.record(" POST /chat/completions\n ") - STEPS.record(" ") - assert STEPS.taken() == ("POST /chat/completions",) - - def test_reset_empties_the_log_so_one_test_never_inherits_another_s(self) -> None: - STEPS.record("register deployment") - STEPS.reset() - assert STEPS.taken() == () - assert step_properties() == () - - def test_steps_serialize_as_repeated_properties_in_order(self) -> None: - """Repeated rather than joined on a delimiter: the labels are free text, so - no separator can be reserved, and a repeated property has none to corrupt.""" - STEPS.record('attach guardrail, comma & "quoted" ') - STEPS.record("POST /chat/completions") - assert step_properties() == ( - ("step", 'attach guardrail, comma & "quoted" '), - ("step", "POST /chat/completions"), - ) - - def test_a_decorated_helper_warns_at_its_caller_with_step_frames(self) -> None: - """`stacklevel` counts frames, and the wrapper is one of them: a cleanup - helper that warns about its caller would otherwise report every warning at - e2e_metadata.py. Pins `STEP_FRAMES` to the frames the wrapper really adds.""" - - @step("delete team") - def delete_team() -> None: - warnings.warn("delete_team('t') failed", stacklevel=2 + STEP_FRAMES) - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - delete_team() - assert [Path(warning.filename).name for warning in caught] == [Path(__file__).name] - - -class TestNestedSteps: - """Harness layers call each other, so a step's helper routinely calls other - decorated helpers. Only the outermost records.""" - - def test_a_step_called_inside_a_step_is_not_recorded(self) -> None: - """`ResourceManager.key` wraps `ProxyClient.generate_key`: one action, one - beat of the story, at the level the test called in at.""" - - @step("POST /key/generate") - def generate_key() -> str: - return "sk-x" - - @step("generate virtual key") - def key() -> str: - return generate_key() - - assert key() == "sk-x" - assert STEPS.taken() == ("generate virtual key",) - - def test_the_inner_step_records_again_once_the_outer_one_returns(self) -> None: - @step("POST /key/generate") - def generate_key() -> str: - return "sk-x" - - @step("generate virtual key") - def key() -> str: - return generate_key() - - _ = key() - _ = generate_key() - assert STEPS.taken() == ("generate virtual key", "POST /key/generate") - - def test_an_inner_step_that_raises_leaves_the_outer_label_last_and_unwinds(self) -> None: - """The helper the test called is where it died, and the nesting flag is - released on the way out, so the next top-level call still records.""" - - @step("POST /team/new") - def post_team() -> None: - raise RuntimeError("/team/new answered 500") - - @step("create team with a budget") - def create_team() -> None: - post_team() - - @step("POST /chat/completions") - def chat() -> None: - return None - - with pytest.raises(RuntimeError, match="answered 500"): - create_team() - chat() - assert STEPS.taken() == ("create team with a budget", "POST /chat/completions") - - def test_a_worker_thread_a_step_fans_out_to_records_its_own_steps(self) -> None: - """Nesting is per thread: a load helper that fans chats out to workers is - not inside a step on those workers, so their calls are still recorded.""" - - @step("POST /chat/completions") - def chat() -> None: - return None - - @step("fire concurrent chats") - def fan_out() -> None: - worker = threading.Thread(target=chat) - worker.start() - worker.join() - - fan_out() - assert STEPS.taken() == ("fire concurrent chats", "POST /chat/completions") - - -class TestContextManagerSteps: - """A `@contextmanager` helper's setup and cleanup run at `__enter__` and - `__exit__`, after the decorated call has returned. Both still count as part - of its step; the `with` body is the test's own code and records as usual.""" - - def test_setup_and_cleanup_stay_inside_the_step_and_the_body_records(self) -> None: - @step("run a SQL statement") - def execute() -> None: - return None - - @step("create a read-only database role") - @contextmanager - def restricted_user() -> Generator[str]: - execute() - try: - yield "reader" - finally: - execute() - - @step("POST /chat/completions") - def chat() -> None: - return None - - with restricted_user() as user: - assert user == "reader" - chat() - assert STEPS.taken() == ("create a read-only database role", "POST /chat/completions") - - def test_a_test_that_dies_in_the_with_body_keeps_its_last_step_last(self) -> None: - """The guarantee the field makes: the cleanup that runs on the way out of - the `with` must not append a step behind the one the test died on.""" - - @step("drop the role") - def drop_role() -> None: - return None - - @step("create a read-only database role") - @contextmanager - def restricted_user() -> Generator[None]: - try: - yield - finally: - drop_role() - - @step("POST /chat/completions") - def chat() -> None: - raise RuntimeError("502 from upstream") - - with pytest.raises(RuntimeError, match="502 from upstream"), restricted_user(): - chat() - assert STEPS.taken() == ("create a read-only database role", "POST /chat/completions") - - def test_the_wrapped_context_keeps_its_exception_handling(self) -> None: - """`__exit__` is forwarded, return value included, so a context that - suppresses an exception still does.""" - - @step("hold an advisory lock") - @contextmanager - def swallowing() -> Generator[None]: - try: - yield - except KeyError: - pass - - with swallowing(): - raise KeyError("suppressed by the context") - assert STEPS.taken() == ("hold an advisory lock",) - - def test_a_bare_generator_is_refused_where_the_decorator_runs(self) -> None: - """Its body runs only as the caller iterates, interleaved with the caller's - own steps, so no single point in the story is where it happened. Refused at - decoration, which for a harness module is import, so it lands as a - collection error rather than a story that quietly reads out of order.""" - - def rows() -> Generator[int]: - yield 1 - - with pytest.raises(TypeError, match="cannot wrap the generator function"): - _ = step("poll /spend/logs")(rows) - - -class TestAttachStepProperties: - def test_steps_are_appended_after_the_collected_properties(self, request: pytest.FixtureRequest) -> None: - """Order inside `` is list order, so the story reads after the - fixed prefix the collection hook already attached.""" - test = type(self).test_steps_are_appended_after_the_collected_properties - item = collected_item(request, test.__name__) - STEPS.record("register deployment") - STEPS.record("POST /chat/completions") - attach_step_properties(item) - assert [name for name, _ in item.user_properties] == ["package", "covers", "source", "step", "step"] - assert [value for name, value in item.user_properties if name == "step"] == [ - "register deployment", - "POST /chat/completions", - ] - - def test_a_rerun_replaces_the_story_rather_than_appending_a_second_one( - self, request: pytest.FixtureRequest - ) -> None: - """The suite runs with `--reruns 1`. Without this the retry's steps would - queue up behind the first attempt's and the report would read as one test - that did everything twice.""" - test = type(self).test_a_rerun_replaces_the_story_rather_than_appending_a_second_one - item = collected_item(request, test.__name__) - STEPS.record("attempt one died here") - attach_step_properties(item) - STEPS.reset() - STEPS.record("attempt two got further") - attach_step_properties(item) - assert [value for name, value in item.user_properties if name == "step"] == ["attempt two got further"] diff --git a/tests/test_litellm/test_circleci_path_filter.py b/tests/test_litellm/test_circleci_path_filter.py index 84e2327057d..b72d85cbafd 100644 --- a/tests/test_litellm/test_circleci_path_filter.py +++ b/tests/test_litellm/test_circleci_path_filter.py @@ -64,6 +64,9 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"] ("provider-harness", ["tests/e2e/e2e_http.py"], "run"), ("provider-harness", ["tests/code_coverage_tests/test_provider_cache.py"], "run"), ("provider-harness", ["tests/code_coverage_tests/test_provider_replay_harness.py"], "run"), + ("provider-harness", ["tests/code_coverage_tests/test_e2e_metadata.py"], "run"), + ("provider-harness", ["tests/code_coverage_tests/test_e2e_junit_report.py"], "run"), + ("provider-harness", ["tests/e2e/e2e_metadata.py"], "run"), ("provider-harness", [".circleci/config.yml"], "run"), ("provider-harness", [".circleci/scripts/classify_changes.sh"], "run"), ("provider-harness", ["pyproject.toml"], "run"),