scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-26 01:12:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 99

Keep a full step log's newest steps, and test it outside tests/e2e

Browse source 
The log stopped recording at MAX_STEPS, so a test that ran past 50 distinct steps and then failed reported step 50 as its last, not the helper it died in. The cap now drops from the front: the newest 50 steps are kept, led by a line counting the ones dropped, so the story still ends where the test died and says when it is partial

tests/e2e holds only tests that drive a live proxy, so the new recorder tests move to tests/code_coverage_tests/test_e2e_metadata.py and the report test to test_e2e_junit_report.py. tests/e2e/test_junit_properties.py is back to what main has. Without the e2e conftest, an autouse fixture does the per-test reset its setup hook did, and the attach tests lay the fixed prefix down themselves. CircleCI's provider_replay_harness job runs both files, and classify_changes.sh triggers it when they change, the same as the other harness tests there
This commit is contained in:
ryan-crabbe-berri 2026-09-21 19:08:14 -07:00
parent d0e37d39c4
commit a49940ccf3
8 changed files with 373 additions and 320 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.circleci/config.yml
View file

@ -3100,7 +3100,9 @@ jobs:
tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \
tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \
tests/code_coverage_tests/test_provider_replay_harness.py \
tests/code_coverage_tests/test_provider_cache.py
tests/code_coverage_tests/test_provider_cache.py \
tests/code_coverage_tests/test_e2e_metadata.py \
tests/code_coverage_tests/test_e2e_junit_report.py
- store_test_results:
path: test-results/provider-replay-harness

2
.circleci/scripts/classify_changes.sh
View file

@ -19,7 +19,7 @@ while IFS= read -r file || [ -n "$file" ]; do
esac
case "$file" in
tests/e2e/*/*.py) : ;;
tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/code_coverage_tests/test_e2e_metadata.py | tests/code_coverage_tests/test_e2e_junit_report.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
has_provider_harness=true ;;
esac
case "$file" in

18
tests/e2e/test_junit_report.py → tests/code_coverage_tests/test_e2e_junit_report.py
View file

@ -1,11 +1,11 @@
"""The JUnit report itself, written by a real pytest run.
No proxy and no ``e2e`` marker. test_junit_properties.py pins the functions that
build the properties; this pins what reaches the XML once pytest, its junitxml
plugin, pytest-rerunfailures and xdist are all in the loop. Each case writes a
throwaway suite into a tmp dir and runs it in a child interpreter with THIS
directory's conftest.py loaded as a plugin, so the hooks under test are the ones
the live suite runs and the recorder is the real one, never a copy of either.
No proxy. test_e2e_metadata.py pins the functions that build the properties;
this pins what reaches the XML once pytest, its junitxml plugin,
pytest-rerunfailures and xdist are all in the loop. Each case writes a throwaway
suite into a tmp dir and runs it in a child interpreter with tests/e2e's
conftest.py loaded as a plugin, so the hooks under test are the ones the live
suite runs and the recorder is the real one, never a copy of either.
The timing that makes the recorded half work is pytest's, which is why it is
pinned here against the real thing: junitxml writes a testcase's properties from
@ -29,7 +29,7 @@ from xml.etree import ElementTree
import pytest
SUITE_DIR: Final = Path(__file__).resolve().parent
SUITE_DIR: Final = Path(__file__).resolve().parents[1] / "e2e"
CHILD_TIMEOUT_SECONDS: Final = 180
STORY_SUITE: Final = """
@ -154,7 +154,7 @@ def write_suite(directory: Path, modules: Mapping[str, str]) -> None:
"""Lay a child suite out in ``directory``, with an ini file of its own.
The ini pins the child's rootdir to the tmp dir wherever that lives, and its
``pythonpath`` is what makes this directory's conftest.py, and the harness
``pythonpath`` is what makes tests/e2e's conftest.py, and the harness
modules the child suite imports, importable under ``-I``.
"""
_ = (directory / "pytest.ini").write_text(f"[pytest]\npythonpath = {shlex.quote(str(SUITE_DIR))}\n")
@ -165,7 +165,7 @@ def write_suite(directory: Path, modules: Mapping[str, str]) -> None:
def run_child_pytest(suite: Path, *args: str) -> subprocess.CompletedProcess[str]:
"""Run pytest over ``suite`` in a fresh interpreter, hooked up like the live suite.
``-p conftest`` registers this directory's conftest.py as a plugin, since a
``-p conftest`` registers tests/e2e's conftest.py as a plugin, since a
tmp dir outside tests/e2e would never pick it up by location. The parent's
fixture-mode and addopts settings are dropped so a replay lane cannot leak
into the child.

341
tests/code_coverage_tests/test_e2e_metadata.py Normal file
View file

@ -0,0 +1,341 @@
"""The e2e step log: what `@step` records, and how it attaches to a JUnit item.
Harness logic, so it lives here rather than under tests/e2e, which holds only
tests that drive a live proxy. The harness modules are imported off
``-o pythonpath=tests/e2e``, the way CI's provider_replay_harness job runs this
file. test_e2e_junit_report.py pins what reaches the XML through the real
conftest.
"""
from __future__ import annotations
import threading
import warnings
from collections.abc import Generator
from contextlib import contextmanager
from pathlib import Path
import pytest
from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step, step_properties
from junit_properties import attach_result_properties, attach_step_properties
@pytest.fixture(autouse=True)
def empty_step_log() -> Generator[None]:
"""Each test starts from an empty log and leaves none behind, as conftest's
`pytest_runtest_setup` hook arranges for every live test."""
STEPS.reset()
yield
STEPS.reset()
def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item:
"""The Item pytest collected for test ``name`` in this file, as pytest built it."""
return next(item for item in request.session.items if item.path == request.path and item.name == name)
class TestStepRecording:
"""`@step`-decorated harness helpers append to the running test's story as
they execute.
Each test here starts from an empty log because `empty_step_log` resets the
recorder first, the same reset conftest's `pytest_runtest_setup` gives every
live test.
"""
def test_steps_land_in_call_order(self) -> None:
@step("register deployment")
def register() -> str:
return "model-id"
@step("generate virtual key")
def generate() -> str:
return "sk-x"
_ = register()
_ = generate()
assert STEPS.taken() == ("register deployment", "generate virtual key")
def test_a_decorated_helper_still_returns_exactly_what_it_did(self) -> None:
"""`@step` records, it does not intercept: arguments, return value and
`__name__` all survive it, so decorating a live harness method cannot
change what the test observes."""
@step("POST /chat/completions")
def chat(key: str, *, model: str) -> str:
return f"{key}:{model}"
assert chat("sk-x", model="gpt-5.5") == "sk-x:gpt-5.5"
assert chat.__name__ == "chat"
def test_a_helper_that_raises_leaves_its_own_label_last(self) -> None:
"""The whole point of the field. The label is recorded BEFORE the call, so
a test that dies inside a helper keeps a partial story whose last element
names the helper it died in."""
@step("generate virtual key")
def generate() -> str:
return "sk-x"
@step("POST /chat/completions")
def chat() -> None:
raise RuntimeError("502 from upstream")
_ = generate()
with pytest.raises(RuntimeError, match="502 from upstream"):
chat()
assert STEPS.taken() == ("generate virtual key", "POST /chat/completions")
def test_a_poll_loop_is_one_step_in_the_story_not_fifty(self) -> None:
@step("poll /spend/logs for the request id")
def poll() -> None:
return None
for _ in range(20):
poll()
assert STEPS.taken() == ("poll /spend/logs for the request id",)
def test_the_same_label_recorded_again_later_is_a_new_step(self) -> None:
"""Only CONSECUTIVE duplicates collapse; a helper called again after
something else happened is a genuine second beat of the story."""
STEPS.record("POST /chat/completions")
STEPS.record("poll /spend/logs")
STEPS.record("POST /chat/completions")
assert STEPS.taken() == ("POST /chat/completions", "poll /spend/logs", "POST /chat/completions")
def test_a_full_log_keeps_the_latest_steps_so_the_last_is_where_the_test_died(self) -> None:
"""A load test cannot bury the story in thousands of entries, and the cap
drops from the front: the step a test died on is the newest, so it is the
one that has to survive. The leading line says the story is partial."""
for index in range(MAX_STEPS + 10):
STEPS.record(f"call {index}")
assert STEPS.taken() == (
"(10 earlier steps not recorded)",
*(f"call {index}" for index in range(10, MAX_STEPS + 10)),
)
def test_reset_forgets_what_a_full_log_dropped(self) -> None:
for index in range(MAX_STEPS + 1):
STEPS.record(f"call {index}")
STEPS.reset()
STEPS.record("register deployment")
assert STEPS.taken() == ("register deployment",)
def test_whitespace_is_normalized_and_an_empty_label_records_nothing(self) -> None:
STEPS.record(" POST /chat/completions\n ")
STEPS.record(" ")
assert STEPS.taken() == ("POST /chat/completions",)
def test_reset_empties_the_log_so_one_test_never_inherits_another_s(self) -> None:
STEPS.record("register deployment")
STEPS.reset()
assert STEPS.taken() == ()
assert step_properties() == ()
def test_steps_serialize_as_repeated_properties_in_order(self) -> None:
"""Repeated rather than joined on a delimiter: the labels are free text, so
no separator can be reserved, and a repeated property has none to corrupt."""
STEPS.record('attach guardrail, comma & "quoted" <tag>')
STEPS.record("POST /chat/completions")
assert step_properties() == (
("step", 'attach guardrail, comma & "quoted" <tag>'),
("step", "POST /chat/completions"),
)
def test_a_decorated_helper_warns_at_its_caller_with_step_frames(self) -> None:
"""`stacklevel` counts frames, and the wrapper is one of them: a cleanup
helper that warns about its caller would otherwise report every warning at
e2e_metadata.py. Pins `STEP_FRAMES` to the frames the wrapper really adds."""
@step("delete team")
def delete_team() -> None:
warnings.warn("delete_team('t') failed", stacklevel=2 + STEP_FRAMES)
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
delete_team()
assert [Path(warning.filename).name for warning in caught] == [Path(__file__).name]
class TestNestedSteps:
"""Harness layers call each other, so a step's helper routinely calls other
decorated helpers. Only the outermost records."""
def test_a_step_called_inside_a_step_is_not_recorded(self) -> None:
"""`ResourceManager.key` wraps `ProxyClient.generate_key`: one action, one
beat of the story, at the level the test called in at."""
@step("POST /key/generate")
def generate_key() -> str:
return "sk-x"
@step("generate virtual key")
def key() -> str:
return generate_key()
assert key() == "sk-x"
assert STEPS.taken() == ("generate virtual key",)
def test_the_inner_step_records_again_once_the_outer_one_returns(self) -> None:
@step("POST /key/generate")
def generate_key() -> str:
return "sk-x"
@step("generate virtual key")
def key() -> str:
return generate_key()
_ = key()
_ = generate_key()
assert STEPS.taken() == ("generate virtual key", "POST /key/generate")
def test_an_inner_step_that_raises_leaves_the_outer_label_last_and_unwinds(self) -> None:
"""The helper the test called is where it died, and the nesting flag is
released on the way out, so the next top-level call still records."""
@step("POST /team/new")
def post_team() -> None:
raise RuntimeError("/team/new answered 500")
@step("create team with a budget")
def create_team() -> None:
post_team()
@step("POST /chat/completions")
def chat() -> None:
return None
with pytest.raises(RuntimeError, match="answered 500"):
create_team()
chat()
assert STEPS.taken() == ("create team with a budget", "POST /chat/completions")
def test_a_worker_thread_a_step_fans_out_to_records_its_own_steps(self) -> None:
"""Nesting is per thread: a load helper that fans chats out to workers is
not inside a step on those workers, so their calls are still recorded."""
@step("POST /chat/completions")
def chat() -> None:
return None
@step("fire concurrent chats")
def fan_out() -> None:
worker = threading.Thread(target=chat)
worker.start()
worker.join()
fan_out()
assert STEPS.taken() == ("fire concurrent chats", "POST /chat/completions")
class TestContextManagerSteps:
"""A `@contextmanager` helper's setup and cleanup run at `__enter__` and
`__exit__`, after the decorated call has returned. Both still count as part
of its step; the `with` body is the test's own code and records as usual."""
def test_setup_and_cleanup_stay_inside_the_step_and_the_body_records(self) -> None:
@step("run a SQL statement")
def execute() -> None:
return None
@step("create a read-only database role")
@contextmanager
def restricted_user() -> Generator[str]:
execute()
try:
yield "reader"
finally:
execute()
@step("POST /chat/completions")
def chat() -> None:
return None
with restricted_user() as user:
assert user == "reader"
chat()
assert STEPS.taken() == ("create a read-only database role", "POST /chat/completions")
def test_a_test_that_dies_in_the_with_body_keeps_its_last_step_last(self) -> None:
"""The guarantee the field makes: the cleanup that runs on the way out of
the `with` must not append a step behind the one the test died on."""
@step("drop the role")
def drop_role() -> None:
return None
@step("create a read-only database role")
@contextmanager
def restricted_user() -> Generator[None]:
try:
yield
finally:
drop_role()
@step("POST /chat/completions")
def chat() -> None:
raise RuntimeError("502 from upstream")
with pytest.raises(RuntimeError, match="502 from upstream"), restricted_user():
chat()
assert STEPS.taken() == ("create a read-only database role", "POST /chat/completions")
def test_the_wrapped_context_keeps_its_exception_handling(self) -> None:
"""`__exit__` is forwarded, return value included, so a context that
suppresses an exception still does."""
@step("hold an advisory lock")
@contextmanager
def swallowing() -> Generator[None]:
try:
yield
except KeyError:
pass
with swallowing():
raise KeyError("suppressed by the context")
assert STEPS.taken() == ("hold an advisory lock",)
def test_a_bare_generator_is_refused_where_the_decorator_runs(self) -> None:
"""Its body runs only as the caller iterates, interleaved with the caller's
own steps, so no single point in the story is where it happened. Refused at
decoration, which for a harness module is import, so it lands as a
collection error rather than a story that quietly reads out of order."""
def rows() -> Generator[int]:
yield 1
with pytest.raises(TypeError, match="cannot wrap the generator function"):
_ = step("poll /spend/logs")(rows)
class TestAttachStepProperties:
def test_steps_are_appended_after_the_collected_properties(self, request: pytest.FixtureRequest) -> None:
"""Order inside `<properties>` is list order, so the story reads after the
fixed prefix the collection hook already attached."""
test = type(self).test_steps_are_appended_after_the_collected_properties
item = collected_item(request, test.__name__)
attach_result_properties(item)
STEPS.record("register deployment")
STEPS.record("POST /chat/completions")
attach_step_properties(item)
assert [name for name, _ in item.user_properties] == ["package", "covers", "source", "step", "step"]
assert [value for name, value in item.user_properties if name == "step"] == [
"register deployment",
"POST /chat/completions",
]
def test_a_rerun_replaces_the_story_rather_than_appending_a_second_one(
self, request: pytest.FixtureRequest
) -> None:
"""The suite runs with `--reruns 1`. Without this the retry's steps would
queue up behind the first attempt's and the report would read as one test
that did everything twice."""
test = type(self).test_a_rerun_replaces_the_story_rather_than_appending_a_second_one
item = collected_item(request, test.__name__)
STEPS.record("attempt one died here")
attach_step_properties(item)
STEPS.reset()
STEPS.record("attempt two got further")
attach_step_properties(item)
assert [value for name, value in item.user_properties if name == "step"] == ["attempt two got further"]

4
tests/e2e/AGENTS.md
View file

@ -134,9 +134,9 @@ The harness is fully typed with no error budget: `make lint-e2e-basedpyright` mu
`@step("POST /chat/completions")` from `e2e_metadata.py` goes on HARNESS helpers - client methods, `ResourceManager.key`, poll loops - never on a test, and appends its label to the running test's `user_properties` in call order. The list IS the test's user story, and because the label is recorded BEFORE the wrapped call, a failing test's LAST step is where it died. Nothing about steps is hand-written: the call sequence cannot drift from what the test actually did. A new public harness method that performs an action (an HTTP call, a poll, a login, a CLI run) gets a `@step`; pure builders, parsers and `_private` helpers do not. Labels are static, lowercase, one beat of the story: a plain-English action ("create team with a budget") or, for a raw route call, the route itself ("POST /v1/messages")
Only the outermost step records. Harness layers call each other - `ResourceManager.key` goes through `ProxyClient.generate_key`, a domain client wraps the shared `ProxyClient` - so every layer carries its own label and the story still reads at the level the test called in at, one beat per action. On a `@contextmanager` helper `@step` goes ABOVE `@contextmanager`: the setup and cleanup around its `yield` count as part of the step, while the `with` body - the test's own code - records as usual, so cleanup never lands behind the step a test died on. A bare generator function is refused at import, since its body interleaves with the caller's. A decorated helper that warns about its caller uses `stacklevel=2 + STEP_FRAMES`, because the wrapper is a frame too. Nesting is tracked per thread, so a helper that fans work out to worker threads still records their steps. Consecutive duplicates collapse and the log caps at 50 entries, so a poll loop is one beat rather than fifty. The log is emptied first thing in every test's setup phase and attached after setup and again after call, so a test that errors in a fixture keeps the steps recorded before the crash. Teardown steps are left out on purpose: they are cleanup, and listing them would put a finalizer's step after the one a failing test died on
Only the outermost step records. Harness layers call each other - `ResourceManager.key` goes through `ProxyClient.generate_key`, a domain client wraps the shared `ProxyClient` - so every layer carries its own label and the story still reads at the level the test called in at, one beat per action. On a `@contextmanager` helper `@step` goes ABOVE `@contextmanager`: the setup and cleanup around its `yield` count as part of the step, while the `with` body - the test's own code - records as usual, so cleanup never lands behind the step a test died on. A bare generator function is refused at import, since its body interleaves with the caller's. A decorated helper that warns about its caller uses `stacklevel=2 + STEP_FRAMES`, because the wrapper is a frame too. Nesting is tracked per thread, so a helper that fans work out to worker threads still records their steps. Consecutive duplicates collapse, so a poll loop is one beat rather than fifty, and the log keeps the latest 50 steps behind a line counting the ones it dropped: the cap drops from the front because the last step is where a failing test died. The log is emptied first thing in every test's setup phase and attached after setup and again after call, so a test that errors in a fixture keeps the steps recorded before the crash. Teardown steps are left out on purpose: they are cleanup, and listing them would put a finalizer's step after the one a failing test died on
Steps ride out as repeated JUnit `<property name="step">` entries (`junit_properties.py`), one per step rather than one delimiter-joined value, since a free-text label has no separator that can be reserved. The results JSON downstream regroups them into a `steps` array. `test_junit_report.py` runs real pytest with `--junitxml`, in-process and under `-n 2`, and pins what reaches the XML
Steps ride out as repeated JUnit `<property name="step">` entries (`junit_properties.py`), one per step rather than one delimiter-joined value, since a free-text label has no separator that can be reserved. The results JSON downstream regroups them into a `steps` array. The harness tests for it sit outside the suite, in `tests/code_coverage_tests/test_e2e_metadata.py` and `test_e2e_junit_report.py`; the latter runs real pytest with `--junitxml` through this conftest, in-process and under `-n 2`, and pins what reaches the XML
## Coverage registry

20
tests/e2e/e2e_metadata.py
View file

@ -14,6 +14,7 @@ from __future__ import annotations
import inspect
import threading
from collections import deque
from collections.abc import Callable, Generator
from contextlib import AbstractContextManager, contextmanager
from functools import wraps
@ -44,21 +45,25 @@ class _StepRecorder:
def __init__(self) -> None:
self._lock = threading.Lock()
self._steps: list[str] = []
self._steps: deque[str] = deque(maxlen=MAX_STEPS)
self._dropped = 0
def reset(self) -> None:
"""Called first thing in every test's setup phase, so each test starts
empty."""
with self._lock:
self._steps.clear()
self._dropped = 0
def record(self, label: str) -> None:
"""Append `label`, unless it repeats the previous step or the log is full.
"""Append `label`, unless it repeats the previous step.
A retrying helper (poll_cost_row) or a load test calling a decorated
helper in a loop would otherwise emit thousands of <property> entries per
testcase: a consecutive repeat collapses, so a poll loop is one step in
the story rather than fifty, and the log stops growing at MAX_STEPS.
the story rather than fifty, and past MAX_STEPS the oldest step makes way.
It is the oldest that goes because the last step is the one that has to
survive: it is where a failing test died.
"""
cleaned = " ".join(label.split())[:MAX_STEP_CHARS]
if not cleaned:
@ -66,13 +71,16 @@ class _StepRecorder:
with self._lock:
if self._steps and self._steps[-1] == cleaned:
return
if len(self._steps) >= MAX_STEPS:
return
if len(self._steps) == MAX_STEPS:
self._dropped += 1
self._steps.append(cleaned)
def taken(self) -> tuple[str, ...]:
"""The story so far, led by a line counting the steps a full log dropped,
so a story that starts mid-test says so rather than reading as complete."""
with self._lock:
return tuple(self._steps)
dropped: Final = (f"({self._dropped} earlier steps not recorded)",) if self._dropped else ()
return dropped + tuple(self._steps)
STEPS: Final = _StepRecorder()

301
tests/e2e/test_junit_properties.py
View file

@ -10,18 +10,12 @@ rollups and, for ``source``, the status page's per-test links to GitHub.
from __future__ import annotations
import threading
import warnings
from collections.abc import Generator
from contextlib import contextmanager
from pathlib import Path
import pytest
from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step, step_properties
from junit_properties import (
SUITE_ROOT,
attach_result_properties,
attach_step_properties,
dedupe_covers,
package_from_nodeid,
result_properties,
@ -135,298 +129,3 @@ class TestSuiteRoot:
class TestDedupeCovers:
def test_ids_are_unique_order_preserving_and_non_empty_strings(self) -> None:
assert dedupe_covers([("A", "B"), ("B", ""), ("C", 7)]) == ("A", "B", "C")
class TestStepRecording:
"""`@step`-decorated harness helpers append to the running test's story as
they execute.
Each test here starts from an empty log because conftest's
`pytest_runtest_setup` hook resets the recorder first thing in every test's
setup -- the same reset the live suite relies on for per-test isolation.
"""
def test_steps_land_in_call_order(self) -> None:
@step("register deployment")
def register() -> str:
return "model-id"
@step("generate virtual key")
def generate() -> str:
return "sk-x"
_ = register()
_ = generate()
assert STEPS.taken() == ("register deployment", "generate virtual key")
def test_a_decorated_helper_still_returns_exactly_what_it_did(self) -> None:
"""`@step` records, it does not intercept: arguments, return value and
`__name__` all survive it, so decorating a live harness method cannot
change what the test observes."""
@step("POST /chat/completions")
def chat(key: str, *, model: str) -> str:
return f"{key}:{model}"
assert chat("sk-x", model="gpt-5.5") == "sk-x:gpt-5.5"
assert chat.__name__ == "chat"
def test_a_helper_that_raises_leaves_its_own_label_last(self) -> None:
"""The whole point of the field. The label is recorded BEFORE the call, so
a test that dies inside a helper keeps a partial story whose last element
names the helper it died in."""
@step("generate virtual key")
def generate() -> str:
return "sk-x"
@step("POST /chat/completions")
def chat() -> None:
raise RuntimeError("502 from upstream")
_ = generate()
with pytest.raises(RuntimeError, match="502 from upstream"):
chat()
assert STEPS.taken() == ("generate virtual key", "POST /chat/completions")
def test_a_poll_loop_is_one_step_in_the_story_not_fifty(self) -> None:
@step("poll /spend/logs for the request id")
def poll() -> None:
return None
for _ in range(20):
poll()
assert STEPS.taken() == ("poll /spend/logs for the request id",)
def test_the_same_label_recorded_again_later_is_a_new_step(self) -> None:
"""Only CONSECUTIVE duplicates collapse; a helper called again after
something else happened is a genuine second beat of the story."""
STEPS.record("POST /chat/completions")
STEPS.record("poll /spend/logs")
STEPS.record("POST /chat/completions")
assert STEPS.taken() == ("POST /chat/completions", "poll /spend/logs", "POST /chat/completions")
def test_the_log_is_capped_so_a_load_test_cannot_bury_the_story(self) -> None:
for index in range(MAX_STEPS * 2):
STEPS.record(f"call {index}")
taken = STEPS.taken()
assert len(taken) == MAX_STEPS
assert taken[0] == "call 0"
def test_whitespace_is_normalized_and_an_empty_label_records_nothing(self) -> None:
STEPS.record(" POST /chat/completions\n ")
STEPS.record(" ")
assert STEPS.taken() == ("POST /chat/completions",)
def test_reset_empties_the_log_so_one_test_never_inherits_another_s(self) -> None:
STEPS.record("register deployment")
STEPS.reset()
assert STEPS.taken() == ()
assert step_properties() == ()
def test_steps_serialize_as_repeated_properties_in_order(self) -> None:
"""Repeated rather than joined on a delimiter: the labels are free text, so
no separator can be reserved, and a repeated property has none to corrupt."""
STEPS.record('attach guardrail, comma & "quoted" <tag>')
STEPS.record("POST /chat/completions")
assert step_properties() == (
("step", 'attach guardrail, comma & "quoted" <tag>'),
("step", "POST /chat/completions"),
)
def test_a_decorated_helper_warns_at_its_caller_with_step_frames(self) -> None:
"""`stacklevel` counts frames, and the wrapper is one of them: a cleanup
helper that warns about its caller would otherwise report every warning at
e2e_metadata.py. Pins `STEP_FRAMES` to the frames the wrapper really adds."""
@step("delete team")
def delete_team() -> None:
warnings.warn("delete_team('t') failed", stacklevel=2 + STEP_FRAMES)
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
delete_team()
assert [Path(warning.filename).name for warning in caught] == [Path(__file__).name]
class TestNestedSteps:
"""Harness layers call each other, so a step's helper routinely calls other
decorated helpers. Only the outermost records."""
def test_a_step_called_inside_a_step_is_not_recorded(self) -> None:
"""`ResourceManager.key` wraps `ProxyClient.generate_key`: one action, one
beat of the story, at the level the test called in at."""
@step("POST /key/generate")
def generate_key() -> str:
return "sk-x"
@step("generate virtual key")
def key() -> str:
return generate_key()
assert key() == "sk-x"
assert STEPS.taken() == ("generate virtual key",)
def test_the_inner_step_records_again_once_the_outer_one_returns(self) -> None:
@step("POST /key/generate")
def generate_key() -> str:
return "sk-x"
@step("generate virtual key")
def key() -> str:
return generate_key()
_ = key()
_ = generate_key()
assert STEPS.taken() == ("generate virtual key", "POST /key/generate")
def test_an_inner_step_that_raises_leaves_the_outer_label_last_and_unwinds(self) -> None:
"""The helper the test called is where it died, and the nesting flag is
released on the way out, so the next top-level call still records."""
@step("POST /team/new")
def post_team() -> None:
raise RuntimeError("/team/new answered 500")
@step("create team with a budget")
def create_team() -> None:
post_team()
@step("POST /chat/completions")
def chat() -> None:
return None
with pytest.raises(RuntimeError, match="answered 500"):
create_team()
chat()
assert STEPS.taken() == ("create team with a budget", "POST /chat/completions")
def test_a_worker_thread_a_step_fans_out_to_records_its_own_steps(self) -> None:
"""Nesting is per thread: a load helper that fans chats out to workers is
not inside a step on those workers, so their calls are still recorded."""
@step("POST /chat/completions")
def chat() -> None:
return None
@step("fire concurrent chats")
def fan_out() -> None:
worker = threading.Thread(target=chat)
worker.start()
worker.join()
fan_out()
assert STEPS.taken() == ("fire concurrent chats", "POST /chat/completions")
class TestContextManagerSteps:
"""A `@contextmanager` helper's setup and cleanup run at `__enter__` and
`__exit__`, after the decorated call has returned. Both still count as part
of its step; the `with` body is the test's own code and records as usual."""
def test_setup_and_cleanup_stay_inside_the_step_and_the_body_records(self) -> None:
@step("run a SQL statement")
def execute() -> None:
return None
@step("create a read-only database role")
@contextmanager
def restricted_user() -> Generator[str]:
execute()
try:
yield "reader"
finally:
execute()
@step("POST /chat/completions")
def chat() -> None:
return None
with restricted_user() as user:
assert user == "reader"
chat()
assert STEPS.taken() == ("create a read-only database role", "POST /chat/completions")
def test_a_test_that_dies_in_the_with_body_keeps_its_last_step_last(self) -> None:
"""The guarantee the field makes: the cleanup that runs on the way out of
the `with` must not append a step behind the one the test died on."""
@step("drop the role")
def drop_role() -> None:
return None
@step("create a read-only database role")
@contextmanager
def restricted_user() -> Generator[None]:
try:
yield
finally:
drop_role()
@step("POST /chat/completions")
def chat() -> None:
raise RuntimeError("502 from upstream")
with pytest.raises(RuntimeError, match="502 from upstream"), restricted_user():
chat()
assert STEPS.taken() == ("create a read-only database role", "POST /chat/completions")
def test_the_wrapped_context_keeps_its_exception_handling(self) -> None:
"""`__exit__` is forwarded, return value included, so a context that
suppresses an exception still does."""
@step("hold an advisory lock")
@contextmanager
def swallowing() -> Generator[None]:
try:
yield
except KeyError:
pass
with swallowing():
raise KeyError("suppressed by the context")
assert STEPS.taken() == ("hold an advisory lock",)
def test_a_bare_generator_is_refused_where_the_decorator_runs(self) -> None:
"""Its body runs only as the caller iterates, interleaved with the caller's
own steps, so no single point in the story is where it happened. Refused at
decoration, which for a harness module is import, so it lands as a
collection error rather than a story that quietly reads out of order."""
def rows() -> Generator[int]:
yield 1
with pytest.raises(TypeError, match="cannot wrap the generator function"):
_ = step("poll /spend/logs")(rows)
class TestAttachStepProperties:
def test_steps_are_appended_after_the_collected_properties(self, request: pytest.FixtureRequest) -> None:
"""Order inside `<properties>` is list order, so the story reads after the
fixed prefix the collection hook already attached."""
test = type(self).test_steps_are_appended_after_the_collected_properties
item = collected_item(request, test.__name__)
STEPS.record("register deployment")
STEPS.record("POST /chat/completions")
attach_step_properties(item)
assert [name for name, _ in item.user_properties] == ["package", "covers", "source", "step", "step"]
assert [value for name, value in item.user_properties if name == "step"] == [
"register deployment",
"POST /chat/completions",
]
def test_a_rerun_replaces_the_story_rather_than_appending_a_second_one(
self, request: pytest.FixtureRequest
) -> None:
"""The suite runs with `--reruns 1`. Without this the retry's steps would
queue up behind the first attempt's and the report would read as one test
that did everything twice."""
test = type(self).test_a_rerun_replaces_the_story_rather_than_appending_a_second_one
item = collected_item(request, test.__name__)
STEPS.record("attempt one died here")
attach_step_properties(item)
STEPS.reset()
STEPS.record("attempt two got further")
attach_step_properties(item)
assert [value for name, value in item.user_properties if name == "step"] == ["attempt two got further"]

3
tests/test_litellm/test_circleci_path_filter.py
View file

@ -64,6 +64,9 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"]
("provider-harness", ["tests/e2e/e2e_http.py"], "run"),
("provider-harness", ["tests/code_coverage_tests/test_provider_cache.py"], "run"),
("provider-harness", ["tests/code_coverage_tests/test_provider_replay_harness.py"], "run"),
("provider-harness", ["tests/code_coverage_tests/test_e2e_metadata.py"], "run"),
("provider-harness", ["tests/code_coverage_tests/test_e2e_junit_report.py"], "run"),
("provider-harness", ["tests/e2e/e2e_metadata.py"], "run"),
("provider-harness", [".circleci/config.yml"], "run"),
("provider-harness", [".circleci/scripts/classify_changes.sh"], "run"),
("provider-harness", ["pyproject.toml"], "run"),