mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
ci(e2e): run the step-log tests in the Code Quality workflow, drop overlapping unit tests
CircleCI does not gate every PR, so the two step-log harness test files move from its provider_replay_harness job to a test_e2e_metadata step in the Code Quality GitHub Actions workflow. The CircleCI config, its path classifier and the classifier's test go back to their base versions. Six unit tests in test_e2e_metadata.py asserted what test_e2e_junit_report.py already pins through a real pytest run: call order, a raising helper's label last, the per-test reset, repeated step properties and both attach cases. They are removed; mutating each of those behaviours still fails the suite.
This commit is contained in:
parent
a49940ccf3
commit
b43e45d99d
6 changed files with 14 additions and 98 deletions
|
|
@ -3100,9 +3100,7 @@ jobs:
|
|||
tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \
|
||||
tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \
|
||||
tests/code_coverage_tests/test_provider_replay_harness.py \
|
||||
tests/code_coverage_tests/test_provider_cache.py \
|
||||
tests/code_coverage_tests/test_e2e_metadata.py \
|
||||
tests/code_coverage_tests/test_e2e_junit_report.py
|
||||
tests/code_coverage_tests/test_provider_cache.py
|
||||
- store_test_results:
|
||||
path: test-results/provider-replay-harness
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ while IFS= read -r file || [ -n "$file" ]; do
|
|||
esac
|
||||
case "$file" in
|
||||
tests/e2e/*/*.py) : ;;
|
||||
tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/code_coverage_tests/test_e2e_metadata.py | tests/code_coverage_tests/test_e2e_junit_report.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
|
||||
tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
|
||||
has_provider_harness=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
|
|
|
|||
5
.github/workflows/test-code-quality.yml
vendored
5
.github/workflows/test-code-quality.yml
vendored
|
|
@ -83,6 +83,11 @@ jobs:
|
|||
- name: test_e2e_changed_gate
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
|
||||
|
||||
- name: test_e2e_metadata
|
||||
env:
|
||||
PYTHONPATH: tests/e2e
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
|
||||
|
||||
- name: router_code_coverage
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""The JUnit report itself, written by a real pytest run.
|
||||
|
||||
No proxy. test_e2e_metadata.py pins the functions that build the properties;
|
||||
No proxy. test_e2e_metadata.py pins the recorder's edge cases;
|
||||
this pins what reaches the XML once pytest, its junitxml plugin,
|
||||
pytest-rerunfailures and xdist are all in the loop. Each case writes a throwaway
|
||||
suite into a tmp dir and runs it in a child interpreter with tests/e2e's
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
"""The e2e step log: what `@step` records, and how it attaches to a JUnit item.
|
||||
"""The e2e step recorder's edge cases: dedupe, the cap, nesting, context managers.
|
||||
|
||||
Harness logic, so it lives here rather than under tests/e2e, which holds only
|
||||
tests that drive a live proxy. The harness modules are imported off
|
||||
``-o pythonpath=tests/e2e``, the way CI's provider_replay_harness job runs this
|
||||
file. test_e2e_junit_report.py pins what reaches the XML through the real
|
||||
conftest.
|
||||
``PYTHONPATH=tests/e2e``, the way the Code Quality workflow's
|
||||
test_e2e_metadata step runs this file. Call order, the failing test's last step,
|
||||
the per-test reset and the JUnit attach are pinned end to end in
|
||||
test_e2e_junit_report.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -16,8 +17,7 @@ from contextlib import contextmanager
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step, step_properties
|
||||
from junit_properties import attach_result_properties, attach_step_properties
|
||||
from e2e_metadata import MAX_STEPS, STEP_FRAMES, STEPS, step
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -29,11 +29,6 @@ def empty_step_log() -> Generator[None]:
|
|||
STEPS.reset()
|
||||
|
||||
|
||||
def collected_item(request: pytest.FixtureRequest, name: str) -> pytest.Item:
|
||||
"""The Item pytest collected for test ``name`` in this file, as pytest built it."""
|
||||
return next(item for item in request.session.items if item.path == request.path and item.name == name)
|
||||
|
||||
|
||||
class TestStepRecording:
|
||||
"""`@step`-decorated harness helpers append to the running test's story as
|
||||
they execute.
|
||||
|
|
@ -43,19 +38,6 @@ class TestStepRecording:
|
|||
live test.
|
||||
"""
|
||||
|
||||
def test_steps_land_in_call_order(self) -> None:
|
||||
@step("register deployment")
|
||||
def register() -> str:
|
||||
return "model-id"
|
||||
|
||||
@step("generate virtual key")
|
||||
def generate() -> str:
|
||||
return "sk-x"
|
||||
|
||||
_ = register()
|
||||
_ = generate()
|
||||
assert STEPS.taken() == ("register deployment", "generate virtual key")
|
||||
|
||||
def test_a_decorated_helper_still_returns_exactly_what_it_did(self) -> None:
|
||||
"""`@step` records, it does not intercept: arguments, return value and
|
||||
`__name__` all survive it, so decorating a live harness method cannot
|
||||
|
|
@ -68,24 +50,6 @@ class TestStepRecording:
|
|||
assert chat("sk-x", model="gpt-5.5") == "sk-x:gpt-5.5"
|
||||
assert chat.__name__ == "chat"
|
||||
|
||||
def test_a_helper_that_raises_leaves_its_own_label_last(self) -> None:
|
||||
"""The whole point of the field. The label is recorded BEFORE the call, so
|
||||
a test that dies inside a helper keeps a partial story whose last element
|
||||
names the helper it died in."""
|
||||
|
||||
@step("generate virtual key")
|
||||
def generate() -> str:
|
||||
return "sk-x"
|
||||
|
||||
@step("POST /chat/completions")
|
||||
def chat() -> None:
|
||||
raise RuntimeError("502 from upstream")
|
||||
|
||||
_ = generate()
|
||||
with pytest.raises(RuntimeError, match="502 from upstream"):
|
||||
chat()
|
||||
assert STEPS.taken() == ("generate virtual key", "POST /chat/completions")
|
||||
|
||||
def test_a_poll_loop_is_one_step_in_the_story_not_fifty(self) -> None:
|
||||
@step("poll /spend/logs for the request id")
|
||||
def poll() -> None:
|
||||
|
|
@ -126,22 +90,6 @@ class TestStepRecording:
|
|||
STEPS.record(" ")
|
||||
assert STEPS.taken() == ("POST /chat/completions",)
|
||||
|
||||
def test_reset_empties_the_log_so_one_test_never_inherits_another_s(self) -> None:
|
||||
STEPS.record("register deployment")
|
||||
STEPS.reset()
|
||||
assert STEPS.taken() == ()
|
||||
assert step_properties() == ()
|
||||
|
||||
def test_steps_serialize_as_repeated_properties_in_order(self) -> None:
|
||||
"""Repeated rather than joined on a delimiter: the labels are free text, so
|
||||
no separator can be reserved, and a repeated property has none to corrupt."""
|
||||
STEPS.record('attach guardrail, comma & "quoted" <tag>')
|
||||
STEPS.record("POST /chat/completions")
|
||||
assert step_properties() == (
|
||||
("step", 'attach guardrail, comma & "quoted" <tag>'),
|
||||
("step", "POST /chat/completions"),
|
||||
)
|
||||
|
||||
def test_a_decorated_helper_warns_at_its_caller_with_step_frames(self) -> None:
|
||||
"""`stacklevel` counts frames, and the wrapper is one of them: a cleanup
|
||||
helper that warns about its caller would otherwise report every warning at
|
||||
|
|
@ -307,35 +255,3 @@ class TestContextManagerSteps:
|
|||
|
||||
with pytest.raises(TypeError, match="cannot wrap the generator function"):
|
||||
_ = step("poll /spend/logs")(rows)
|
||||
|
||||
|
||||
class TestAttachStepProperties:
|
||||
def test_steps_are_appended_after_the_collected_properties(self, request: pytest.FixtureRequest) -> None:
|
||||
"""Order inside `<properties>` is list order, so the story reads after the
|
||||
fixed prefix the collection hook already attached."""
|
||||
test = type(self).test_steps_are_appended_after_the_collected_properties
|
||||
item = collected_item(request, test.__name__)
|
||||
attach_result_properties(item)
|
||||
STEPS.record("register deployment")
|
||||
STEPS.record("POST /chat/completions")
|
||||
attach_step_properties(item)
|
||||
assert [name for name, _ in item.user_properties] == ["package", "covers", "source", "step", "step"]
|
||||
assert [value for name, value in item.user_properties if name == "step"] == [
|
||||
"register deployment",
|
||||
"POST /chat/completions",
|
||||
]
|
||||
|
||||
def test_a_rerun_replaces_the_story_rather_than_appending_a_second_one(
|
||||
self, request: pytest.FixtureRequest
|
||||
) -> None:
|
||||
"""The suite runs with `--reruns 1`. Without this the retry's steps would
|
||||
queue up behind the first attempt's and the report would read as one test
|
||||
that did everything twice."""
|
||||
test = type(self).test_a_rerun_replaces_the_story_rather_than_appending_a_second_one
|
||||
item = collected_item(request, test.__name__)
|
||||
STEPS.record("attempt one died here")
|
||||
attach_step_properties(item)
|
||||
STEPS.reset()
|
||||
STEPS.record("attempt two got further")
|
||||
attach_step_properties(item)
|
||||
assert [value for name, value in item.user_properties if name == "step"] == ["attempt two got further"]
|
||||
|
|
|
|||
|
|
@ -64,9 +64,6 @@ CI = [".github/workflows/test-litellm-ui-unit.yml"]
|
|||
("provider-harness", ["tests/e2e/e2e_http.py"], "run"),
|
||||
("provider-harness", ["tests/code_coverage_tests/test_provider_cache.py"], "run"),
|
||||
("provider-harness", ["tests/code_coverage_tests/test_provider_replay_harness.py"], "run"),
|
||||
("provider-harness", ["tests/code_coverage_tests/test_e2e_metadata.py"], "run"),
|
||||
("provider-harness", ["tests/code_coverage_tests/test_e2e_junit_report.py"], "run"),
|
||||
("provider-harness", ["tests/e2e/e2e_metadata.py"], "run"),
|
||||
("provider-harness", [".circleci/config.yml"], "run"),
|
||||
("provider-harness", [".circleci/scripts/classify_changes.sh"], "run"),
|
||||
("provider-harness", ["pyproject.toml"], "run"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue