From de2ba3fab1daab25ba2517ea1a93cf7120f996f5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:50:01 -0700 Subject: [PATCH 1/5] fix(make check): lint the test tree on tests-only changes like CI does CI's required lint job runs ruff with ruff-tests.toml over tests/ and the test-quality budget gate, but scripts/pre_commit_lint.sh only triggered make lint on litellm/ files, so a tests-only commit passed make check with a no-op note and then failed CI (a duplicate test name, ruff F811, did exactly that). When tests/ Python files are in scope and no litellm/ files are, run ruff with ruff-tests.toml over the changed test files and make lint-test-quality, with the matching partial-staging warning, summary line, and no-op condition. --- scripts/pre_commit_lint.sh | 25 +++++- tests/test_litellm/test_pre_commit_lint.py | 98 ++++++++++++++++++++-- 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index d38a0eee3de..ad1b4e793ab 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -12,6 +12,8 @@ # - litellm/ Python -> `make lint` (test-linting.yml's lint job) # - tests/e2e Python -> `make lint-e2e-basedpyright` (test-linting.yml's e2e type-check step) # + raw HTTP client ban (test-code-quality.yml's check_e2e_no_raw_requests) +# - tests/ Python -> ruff over ruff-tests.toml + `make lint-test-quality` (test-linting.yml's +# test-tree ruff and test-quality budget steps) # - dashboard -> prettier + eslint + lint budgets (test-litellm-ui-build.yml's frontend-lint) # - proxy/types -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift (check-ui-api-types.yml) # @@ -88,15 +90,18 @@ existing_files() { litellm_py_pattern='^litellm/.*\.py$' e2e_py_pattern='^tests/e2e/.*\.py$' +tests_py_pattern='^tests/.*\.py$' spec_pattern='^(litellm/(proxy|types)/.*|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|scss|md|mdx|yml|yaml|html)$' ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$' -# CI's lint job (test-linting.yml) only inspects litellm/, so a tests-only or -# scripts-only commit can't turn it red; scope the trigger there to skip the slow -# make lint when it couldn't catch anything. +# CI's lint job (test-linting.yml) is make lint: litellm/ checks plus two test-tree +# steps (ruff over ruff-tests.toml and the test-quality budget). Trigger the slow +# make lint on litellm/ files only; a tests-only commit runs just those two steps. litellm_py_files=$(scope_match "$litellm_py_pattern") e2e_py_files=$(scope_match "$e2e_py_pattern") +tests_py_changed=$(scope_match "$tests_py_pattern") +tests_py_files=$(printf '%s\n' "$tests_py_changed" | existing_files) # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it. fmt_files=$(printf '%s\n' "$litellm_py_files" | grep -v '^litellm/enterprise/' | existing_files) # check-ui-api-types.yml triggers on any file under litellm/proxy or litellm/types @@ -136,6 +141,7 @@ if [ -n "$staged" ]; then } warn_skipped "Python lint (make lint)" "$litellm_py_pattern" "$litellm_py_files" warn_skipped "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_pattern" "$e2e_py_files" + warn_skipped "test-tree lint (ruff-tests.toml + test-quality budget)" "$tests_py_pattern" "$tests_py_changed" warn_skipped "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_pattern" "$ui_prettier_changed" warn_skipped "dashboard API-type sync (npm run gen:api)" "$spec_pattern" "$spec_files" fi @@ -225,6 +231,16 @@ if [ -n "$e2e_py_files" ]; then || { echo "✗ Raw HTTP client import in tests/e2e. Route the call through tests/e2e/e2e_http.py, then re-run make check." >&2; status=1; } fi +if [ -n "$tests_py_changed" ] && [ -z "$litellm_py_files" ]; then + if [ -n "$tests_py_files" ]; then + echo "check: linting the test tree (ruff check --config ruff-tests.toml, scoped tests files)" + printf '%s\n' "$tests_py_files" | xargs uv run --no-sync ruff check --config ruff-tests.toml \ + || { echo "✗ Test-tree ruff failed. Fix the errors above, then re-run make check." >&2; status=1; } + fi + echo "check: checking the test-quality budget (make lint-test-quality)" + make lint-test-quality || { echo "✗ Test-quality budget failed. Fix the errors above, then re-run make check." >&2; status=1; } +fi + dashboard_checks() { echo "check: linting dashboard (prettier + eslint + lint budgets)" if [ ! -d ui/litellm-dashboard/node_modules ]; then @@ -313,10 +329,11 @@ summary_item() { echo "check: summary" summary_item "Python lint (make lint)" "$litellm_py_files" "no litellm/ Python files in scope" summary_item "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_files" "no tests/e2e Python files in scope" +summary_item "test-tree lint (ruff-tests.toml + test-quality budget)" "$tests_py_changed" "no tests/ Python files in scope" summary_item "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_changed$ui_eslint_changed" "no dashboard files in scope" summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/proxy, litellm/types, or generator files in scope" -if [ -z "$litellm_py_files$e2e_py_files$ui_prettier_changed$ui_eslint_changed$spec_files" ]; then +if [ -z "$litellm_py_files$e2e_py_files$tests_py_changed$ui_prettier_changed$ui_eslint_changed$spec_files" ]; then echo "check: NOTE - no gating lint check matches the files in scope, so nothing ran:" >&2 printf '%s\n' "$scope" | sed 's/^/ /' >&2 echo " A pass here is a no-op, not a lint verdict." >&2 diff --git a/tests/test_litellm/test_pre_commit_lint.py b/tests/test_litellm/test_pre_commit_lint.py index aa2260e89ea..22ae0e1bfaa 100644 --- a/tests/test_litellm/test_pre_commit_lint.py +++ b/tests/test_litellm/test_pre_commit_lint.py @@ -40,6 +40,9 @@ case "$*" in sleep 60 fi ;; + lint-test-quality) + [ "${STUB_FAIL:-}" = "test-quality" ] && exit 1 + ;; esac exit 0 """ @@ -69,6 +72,10 @@ case "$*" in *orjson*) [ -n "${STUB_BARRIER_DIR:-}" ] && barrier_sync genapi "python dashboard" ;; + "run --no-sync ruff check --config ruff-tests.toml"*) + [ -n "${STUB_ARGS_DIR:-}" ] && echo "$*" >> "$STUB_ARGS_DIR/ruff_tests.args" + [ "${STUB_FAIL:-}" = "tests-ruff" ] && exit 1 + ;; esac exit 0 """ @@ -153,6 +160,13 @@ def _set_base_ref(repo: Path) -> None: ) +def _stage_file(repo: Path, relative: str, body: str) -> None: + path = repo / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body) + subprocess.run(["git", "add", relative], cwd=repo, check=True) + + def test_nothing_staged_scopes_to_working_tree_diff_and_runs_checks(tmp_path: Path) -> None: repo, bin_dir = _sandbox(tmp_path) _commit_all(repo, "base") @@ -405,6 +419,7 @@ def test_run_ends_with_a_summary_of_ran_and_skipped_blocks(tmp_path: Path) -> No assert "ran: dashboard lint (prettier + eslint + lint budgets)" in proc.stdout assert "ran: dashboard API-type sync (npm run gen:api)" in proc.stdout assert "skipped: tests/e2e checks (basedpyright + raw HTTP client ban) (no tests/e2e Python files in scope)" in proc.stdout + assert "skipped: test-tree lint (ruff-tests.toml + test-quality budget) (no tests/ Python files in scope)" in proc.stdout assert "check: PASS" in proc.stdout assert "check: FAIL" not in proc.stdout @@ -412,20 +427,93 @@ def test_run_ends_with_a_summary_of_ran_and_skipped_blocks(tmp_path: Path) -> No def test_staged_files_matching_no_check_print_an_explicit_noop_note_and_nonempty_log(tmp_path: Path) -> None: repo, bin_dir = _sandbox(tmp_path) _commit_all(repo, "base") - tests_dir = repo / "tests" / "test_litellm" - tests_dir.mkdir(parents=True) - (tests_dir / "test_x.py").write_text("def test_x() -> None: ...\n") - subprocess.run(["git", "add", "tests"], cwd=repo, check=True) + _stage_file(repo, "scripts/tool.py", "def main() -> None: ...\n") proc = _run(repo, bin_dir, {}) assert proc.returncode == 0, proc.stdout + proc.stderr assert "no gating lint check matches the files in scope, so nothing ran" in proc.stdout - assert "tests/test_litellm/test_x.py" in proc.stdout + assert "scripts/tool.py" in proc.stdout assert "a no-op, not a lint verdict" in proc.stdout assert "check: PASS" in proc.stdout assert "linting Python" not in proc.stdout log = (repo / ".git" / "pre_commit_lint.log").read_text() assert "check: summary" in log assert "skipped: Python lint (make lint) (no litellm/ Python files in scope)" in log + assert "skipped: test-tree lint (ruff-tests.toml + test-quality budget) (no tests/ Python files in scope)" in log + + +def test_tests_only_change_runs_test_tree_ruff_on_staged_files_and_the_quality_gate(tmp_path: Path) -> None: + repo, bin_dir = _sandbox(tmp_path) + _commit_all(repo, "base") + args_dir = tmp_path / "args" + args_dir.mkdir() + _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") + _stage_file(repo, "tests/test_b.py", "def test_b() -> None: ...\n") + _stage_file(repo, "tests/fixtures/data.json", "{}\n") + proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)}) + assert proc.returncode == 0, proc.stdout + proc.stderr + ruff_args = (args_dir / "ruff_tests.args").read_text().splitlines() + assert ruff_args == ["run --no-sync ruff check --config ruff-tests.toml tests/test_a.py tests/test_b.py"] + assert "ran: test-tree lint (ruff-tests.toml + test-quality budget)" in proc.stdout + assert "no gating lint check matches" not in proc.stdout + assert "linting Python" not in proc.stdout + assert "check: PASS" in proc.stdout + + +@pytest.mark.parametrize( + ("fail", "message"), + [ + ("tests-ruff", "Test-tree ruff failed"), + ("test-quality", "Test-quality budget failed"), + ], +) +def test_a_failing_test_tree_check_fails_a_tests_only_run(tmp_path: Path, fail: str, message: str) -> None: + repo, bin_dir = _sandbox(tmp_path) + _commit_all(repo, "base") + _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") + proc = _run(repo, bin_dir, {"STUB_FAIL": fail}) + assert proc.returncode == 1 + assert message in proc.stdout + proc.stderr + assert "check: FAIL" in proc.stdout + + +def test_tests_changed_alongside_litellm_files_defer_to_make_lint(tmp_path: Path) -> None: + repo, bin_dir = _sandbox(tmp_path) + _commit_all(repo, "base") + args_dir = tmp_path / "args" + args_dir.mkdir() + _stage_file(repo, "litellm/foo.py", "x = 2\n") + _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") + proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir), "STUB_FAIL": "test-quality"}) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "linting Python" in proc.stdout + assert not (args_dir / "ruff_tests.args").exists() + assert "ran: test-tree lint (ruff-tests.toml + test-quality budget)" in proc.stdout + + +def test_deleted_test_file_still_runs_the_quality_gate_without_feeding_ruff_the_missing_file(tmp_path: Path) -> None: + repo, bin_dir = _sandbox(tmp_path) + _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") + _commit_all(repo, "base") + args_dir = tmp_path / "args" + args_dir.mkdir() + subprocess.run(["git", "rm", "-q", "tests/test_a.py"], cwd=repo, check=True) + proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir), "STUB_FAIL": "test-quality"}) + assert proc.returncode == 1 + assert "Test-quality budget failed" in proc.stdout + proc.stderr + assert not (args_dir / "ruff_tests.args").exists() + + +def test_partial_staging_warns_when_test_files_are_left_unstaged(tmp_path: Path) -> None: + repo, bin_dir = _sandbox(tmp_path) + _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") + _commit_all(repo, "base") + _stage_file(repo, "notes.md", "hi\n") + (repo / "tests" / "test_a.py").write_text("def test_a() -> None:\n assert True\n") + proc = _run(repo, bin_dir, {}) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "SKIPPED test-tree lint (ruff-tests.toml + test-quality budget)" in proc.stdout + assert "tests/test_a.py" in proc.stdout + assert "make lint-test-quality" not in proc.stdout def test_run_queues_through_the_machine_wide_gate_slot_lock(tmp_path: Path) -> None: From d22962248c1c659279e7389e6c2e1d1b640b400a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:07:25 -0700 Subject: [PATCH 2/5] fix(check): run CI's whole-tree test ruff and widen the test-tree trigger The scoped xargs list missed a ruff-tests.toml rule change and skipped ruff on deletions, so the block now runs test-linting.yml's exact command over tests/. ruff-tests.toml, test-quality-budget.json, and scripts/check_test_quality.py trigger the block too, and it sits after the background launches so the dashboard and gen:api jobs overlap it. --- scripts/pre_commit_lint.sh | 36 ++++---- tests/test_litellm/test_pre_commit_lint.py | 102 ++++++++++++++++----- 2 files changed, 96 insertions(+), 42 deletions(-) diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index ad1b4e793ab..ab84d2d518a 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -12,7 +12,8 @@ # - litellm/ Python -> `make lint` (test-linting.yml's lint job) # - tests/e2e Python -> `make lint-e2e-basedpyright` (test-linting.yml's e2e type-check step) # + raw HTTP client ban (test-code-quality.yml's check_e2e_no_raw_requests) -# - tests/ Python -> ruff over ruff-tests.toml + `make lint-test-quality` (test-linting.yml's +# - tests/ Python, ruff-tests.toml, test-quality-budget.json, scripts/check_test_quality.py +# -> ruff over ruff-tests.toml + `make lint-test-quality` (test-linting.yml's # test-tree ruff and test-quality budget steps) # - dashboard -> prettier + eslint + lint budgets (test-litellm-ui-build.yml's frontend-lint) # - proxy/types -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift (check-ui-api-types.yml) @@ -90,18 +91,17 @@ existing_files() { litellm_py_pattern='^litellm/.*\.py$' e2e_py_pattern='^tests/e2e/.*\.py$' -tests_py_pattern='^tests/.*\.py$' +test_tree_pattern='^(tests/.*\.py|ruff-tests\.toml|test-quality-budget\.json|scripts/check_test_quality\.py)$' spec_pattern='^(litellm/(proxy|types)/.*|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|scss|md|mdx|yml|yaml|html)$' ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$' # CI's lint job (test-linting.yml) is make lint: litellm/ checks plus two test-tree # steps (ruff over ruff-tests.toml and the test-quality budget). Trigger the slow -# make lint on litellm/ files only; a tests-only commit runs just those two steps. +# make lint on litellm/ files only; without them, the test-tree steps run on their own below. litellm_py_files=$(scope_match "$litellm_py_pattern") e2e_py_files=$(scope_match "$e2e_py_pattern") -tests_py_changed=$(scope_match "$tests_py_pattern") -tests_py_files=$(printf '%s\n' "$tests_py_changed" | existing_files) +test_tree_files=$(scope_match "$test_tree_pattern") # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it. fmt_files=$(printf '%s\n' "$litellm_py_files" | grep -v '^litellm/enterprise/' | existing_files) # check-ui-api-types.yml triggers on any file under litellm/proxy or litellm/types @@ -141,7 +141,7 @@ if [ -n "$staged" ]; then } warn_skipped "Python lint (make lint)" "$litellm_py_pattern" "$litellm_py_files" warn_skipped "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_pattern" "$e2e_py_files" - warn_skipped "test-tree lint (ruff-tests.toml + test-quality budget)" "$tests_py_pattern" "$tests_py_changed" + warn_skipped "test-tree lint (ruff-tests.toml + test-quality budget)" "$test_tree_pattern" "$test_tree_files" warn_skipped "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_pattern" "$ui_prettier_changed" warn_skipped "dashboard API-type sync (npm run gen:api)" "$spec_pattern" "$spec_files" fi @@ -231,16 +231,6 @@ if [ -n "$e2e_py_files" ]; then || { echo "✗ Raw HTTP client import in tests/e2e. Route the call through tests/e2e/e2e_http.py, then re-run make check." >&2; status=1; } fi -if [ -n "$tests_py_changed" ] && [ -z "$litellm_py_files" ]; then - if [ -n "$tests_py_files" ]; then - echo "check: linting the test tree (ruff check --config ruff-tests.toml, scoped tests files)" - printf '%s\n' "$tests_py_files" | xargs uv run --no-sync ruff check --config ruff-tests.toml \ - || { echo "✗ Test-tree ruff failed. Fix the errors above, then re-run make check." >&2; status=1; } - fi - echo "check: checking the test-quality budget (make lint-test-quality)" - make lint-test-quality || { echo "✗ Test-quality budget failed. Fix the errors above, then re-run make check." >&2; status=1; } -fi - dashboard_checks() { echo "check: linting dashboard (prettier + eslint + lint budgets)" if [ ! -d ui/litellm-dashboard/node_modules ]; then @@ -304,6 +294,15 @@ if [ -n "$spec_files" ]; then set +m fi +if [ -n "$test_tree_files" ] && [ -z "$litellm_py_files" ]; then + echo "check: linting the test tree (ruff check --config ruff-tests.toml tests)" + uv run --no-sync ruff check --config ruff-tests.toml tests \ + || { echo "✗ Test-tree ruff failed. Fix the errors above, then re-run make check." >&2; status=1; } + echo "check: checking the test-quality budget (make lint-test-quality)" + make lint-test-quality \ + || { echo "✗ Test-quality budget failed. Fix the errors above, then re-run make check." >&2; status=1; } +fi + if [ -n "${python_pid:-}" ]; then wait "$python_pid" || status=1 cat "$python_log"; rm -f "$python_log" @@ -329,11 +328,12 @@ summary_item() { echo "check: summary" summary_item "Python lint (make lint)" "$litellm_py_files" "no litellm/ Python files in scope" summary_item "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_files" "no tests/e2e Python files in scope" -summary_item "test-tree lint (ruff-tests.toml + test-quality budget)" "$tests_py_changed" "no tests/ Python files in scope" +summary_item "test-tree lint (ruff-tests.toml + test-quality budget)" "$test_tree_files" \ + "no tests/ Python files or test-tree lint inputs in scope" summary_item "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_changed$ui_eslint_changed" "no dashboard files in scope" summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/proxy, litellm/types, or generator files in scope" -if [ -z "$litellm_py_files$e2e_py_files$tests_py_changed$ui_prettier_changed$ui_eslint_changed$spec_files" ]; then +if [ -z "$litellm_py_files$e2e_py_files$test_tree_files$ui_prettier_changed$ui_eslint_changed$spec_files" ]; then echo "check: NOTE - no gating lint check matches the files in scope, so nothing ran:" >&2 printf '%s\n' "$scope" | sed 's/^/ /' >&2 echo " A pass here is a no-op, not a lint verdict." >&2 diff --git a/tests/test_litellm/test_pre_commit_lint.py b/tests/test_litellm/test_pre_commit_lint.py index 22ae0e1bfaa..e17abf2fa9f 100644 --- a/tests/test_litellm/test_pre_commit_lint.py +++ b/tests/test_litellm/test_pre_commit_lint.py @@ -11,6 +11,12 @@ import pytest ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "pre_commit_lint.sh" +WHOLE_TREE_RUFF = "run --no-sync ruff check --config ruff-tests.toml tests" +TEST_TREE_RAN = "ran: test-tree lint (ruff-tests.toml + test-quality budget)" +TEST_TREE_SKIPPED = ( + "skipped: test-tree lint (ruff-tests.toml + test-quality budget) " + "(no tests/ Python files or test-tree lint inputs in scope)" +) BARRIER_HELPER = """barrier_sync() { touch "$STUB_BARRIER_DIR/$1.started" @@ -30,6 +36,7 @@ BARRIER_HELPER = """barrier_sync() { MAKE_STUB = """#!/bin/sh . "$STUB_BIN/barrier.sh" +[ -n "${STUB_ARGS_DIR:-}" ] && echo "$*" >> "$STUB_ARGS_DIR/make.args" case "$*" in lint) [ "${STUB_FAIL:-}" = "make-lint" ] && exit 1 @@ -419,7 +426,7 @@ def test_run_ends_with_a_summary_of_ran_and_skipped_blocks(tmp_path: Path) -> No assert "ran: dashboard lint (prettier + eslint + lint budgets)" in proc.stdout assert "ran: dashboard API-type sync (npm run gen:api)" in proc.stdout assert "skipped: tests/e2e checks (basedpyright + raw HTTP client ban) (no tests/e2e Python files in scope)" in proc.stdout - assert "skipped: test-tree lint (ruff-tests.toml + test-quality budget) (no tests/ Python files in scope)" in proc.stdout + assert TEST_TREE_SKIPPED in proc.stdout assert "check: PASS" in proc.stdout assert "check: FAIL" not in proc.stdout @@ -438,41 +445,83 @@ def test_staged_files_matching_no_check_print_an_explicit_noop_note_and_nonempty log = (repo / ".git" / "pre_commit_lint.log").read_text() assert "check: summary" in log assert "skipped: Python lint (make lint) (no litellm/ Python files in scope)" in log - assert "skipped: test-tree lint (ruff-tests.toml + test-quality budget) (no tests/ Python files in scope)" in log + assert TEST_TREE_SKIPPED in log -def test_tests_only_change_runs_test_tree_ruff_on_staged_files_and_the_quality_gate(tmp_path: Path) -> None: +def _recorded(args_dir: Path, name: str) -> list[str]: + path = args_dir / name + return path.read_text().splitlines() if path.exists() else [] + + +def test_tests_only_change_runs_the_whole_test_tree_ruff_and_the_quality_gate(tmp_path: Path) -> None: repo, bin_dir = _sandbox(tmp_path) _commit_all(repo, "base") args_dir = tmp_path / "args" args_dir.mkdir() _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") - _stage_file(repo, "tests/test_b.py", "def test_b() -> None: ...\n") _stage_file(repo, "tests/fixtures/data.json", "{}\n") proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)}) assert proc.returncode == 0, proc.stdout + proc.stderr - ruff_args = (args_dir / "ruff_tests.args").read_text().splitlines() - assert ruff_args == ["run --no-sync ruff check --config ruff-tests.toml tests/test_a.py tests/test_b.py"] - assert "ran: test-tree lint (ruff-tests.toml + test-quality budget)" in proc.stdout + assert _recorded(args_dir, "ruff_tests.args") == [WHOLE_TREE_RUFF] + assert _recorded(args_dir, "make.args") == ["lint-test-quality"] + assert TEST_TREE_RAN in proc.stdout assert "no gating lint check matches" not in proc.stdout assert "linting Python" not in proc.stdout assert "check: PASS" in proc.stdout @pytest.mark.parametrize( - ("fail", "message"), - [ - ("tests-ruff", "Test-tree ruff failed"), - ("test-quality", "Test-quality budget failed"), - ], + "changed", + ["ruff-tests.toml", "test-quality-budget.json", "scripts/check_test_quality.py", "tests/e2e/test_x.py"], ) -def test_a_failing_test_tree_check_fails_a_tests_only_run(tmp_path: Path, fail: str, message: str) -> None: +def test_test_tree_lint_inputs_trigger_the_test_tree_checks(tmp_path: Path, changed: str) -> None: + repo, bin_dir = _sandbox(tmp_path) + _commit_all(repo, "base") + args_dir = tmp_path / "args" + args_dir.mkdir() + _stage_file(repo, changed, "x = 1\n") + proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)}) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert _recorded(args_dir, "ruff_tests.args") == [WHOLE_TREE_RUFF] + assert "lint-test-quality" in _recorded(args_dir, "make.args") + assert TEST_TREE_RAN in proc.stdout + + +def test_nothing_staged_tests_only_working_tree_change_runs_the_test_tree_checks(tmp_path: Path) -> None: + repo, bin_dir = _sandbox(tmp_path) + _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") + _commit_all(repo, "base") + _set_base_ref(repo) + args_dir = tmp_path / "args" + args_dir.mkdir() + (repo / "tests" / "test_a.py").write_text("def test_a() -> None:\n assert True\n") + proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)}) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "nothing staged; scoping to the working tree's diff" in proc.stdout + assert _recorded(args_dir, "ruff_tests.args") == [WHOLE_TREE_RUFF] + assert _recorded(args_dir, "make.args") == ["lint-test-quality"] + + +def test_a_failing_test_tree_ruff_fails_the_run_and_still_runs_the_quality_gate(tmp_path: Path) -> None: + repo, bin_dir = _sandbox(tmp_path) + _commit_all(repo, "base") + args_dir = tmp_path / "args" + args_dir.mkdir() + _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") + proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir), "STUB_FAIL": "tests-ruff"}) + assert proc.returncode == 1 + assert "Test-tree ruff failed" in proc.stdout + proc.stderr + assert "check: FAIL" in proc.stdout + assert _recorded(args_dir, "make.args") == ["lint-test-quality"] + + +def test_a_failing_quality_gate_fails_a_tests_only_run(tmp_path: Path) -> None: repo, bin_dir = _sandbox(tmp_path) _commit_all(repo, "base") _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") - proc = _run(repo, bin_dir, {"STUB_FAIL": fail}) + proc = _run(repo, bin_dir, {"STUB_FAIL": "test-quality"}) assert proc.returncode == 1 - assert message in proc.stdout + proc.stderr + assert "Test-quality budget failed" in proc.stdout + proc.stderr assert "check: FAIL" in proc.stdout @@ -486,34 +535,39 @@ def test_tests_changed_alongside_litellm_files_defer_to_make_lint(tmp_path: Path proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir), "STUB_FAIL": "test-quality"}) assert proc.returncode == 0, proc.stdout + proc.stderr assert "linting Python" in proc.stdout - assert not (args_dir / "ruff_tests.args").exists() - assert "ran: test-tree lint (ruff-tests.toml + test-quality budget)" in proc.stdout + assert _recorded(args_dir, "ruff_tests.args") == [] + assert _recorded(args_dir, "make.args") == ["lint"] + assert TEST_TREE_RAN in proc.stdout -def test_deleted_test_file_still_runs_the_quality_gate_without_feeding_ruff_the_missing_file(tmp_path: Path) -> None: +def test_deleted_test_file_still_runs_the_test_tree_checks(tmp_path: Path) -> None: repo, bin_dir = _sandbox(tmp_path) _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") _commit_all(repo, "base") args_dir = tmp_path / "args" args_dir.mkdir() subprocess.run(["git", "rm", "-q", "tests/test_a.py"], cwd=repo, check=True) - proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir), "STUB_FAIL": "test-quality"}) - assert proc.returncode == 1 - assert "Test-quality budget failed" in proc.stdout + proc.stderr - assert not (args_dir / "ruff_tests.args").exists() + proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)}) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert _recorded(args_dir, "ruff_tests.args") == [WHOLE_TREE_RUFF] + assert _recorded(args_dir, "make.args") == ["lint-test-quality"] + assert TEST_TREE_RAN in proc.stdout def test_partial_staging_warns_when_test_files_are_left_unstaged(tmp_path: Path) -> None: repo, bin_dir = _sandbox(tmp_path) _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") _commit_all(repo, "base") + args_dir = tmp_path / "args" + args_dir.mkdir() _stage_file(repo, "notes.md", "hi\n") (repo / "tests" / "test_a.py").write_text("def test_a() -> None:\n assert True\n") - proc = _run(repo, bin_dir, {}) + proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)}) assert proc.returncode == 0, proc.stdout + proc.stderr assert "SKIPPED test-tree lint (ruff-tests.toml + test-quality budget)" in proc.stdout assert "tests/test_a.py" in proc.stdout - assert "make lint-test-quality" not in proc.stdout + assert _recorded(args_dir, "ruff_tests.args") == [] + assert _recorded(args_dir, "make.args") == [] def test_run_queues_through_the_machine_wide_gate_slot_lock(tmp_path: Path) -> None: From e3366dddf44c7450a856ed472c2966798413ef36 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:26:16 -0700 Subject: [PATCH 3/5] fix(check): trigger the test-tree checks on the gate script and drop the scope comment --- scripts/pre_commit_lint.sh | 8 +++----- tests/test_litellm/test_pre_commit_lint.py | 8 +++++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index ab84d2d518a..f245803408c 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -12,7 +12,8 @@ # - litellm/ Python -> `make lint` (test-linting.yml's lint job) # - tests/e2e Python -> `make lint-e2e-basedpyright` (test-linting.yml's e2e type-check step) # + raw HTTP client ban (test-code-quality.yml's check_e2e_no_raw_requests) -# - tests/ Python, ruff-tests.toml, test-quality-budget.json, scripts/check_test_quality.py +# - tests/ Python, ruff-tests.toml, test-quality-budget.json, scripts/check_test_quality.py, +# scripts/test_quality_gate.py # -> ruff over ruff-tests.toml + `make lint-test-quality` (test-linting.yml's # test-tree ruff and test-quality budget steps) # - dashboard -> prettier + eslint + lint budgets (test-litellm-ui-build.yml's frontend-lint) @@ -91,14 +92,11 @@ existing_files() { litellm_py_pattern='^litellm/.*\.py$' e2e_py_pattern='^tests/e2e/.*\.py$' -test_tree_pattern='^(tests/.*\.py|ruff-tests\.toml|test-quality-budget\.json|scripts/check_test_quality\.py)$' +test_tree_pattern='^(tests/.*\.py|ruff-tests\.toml|test-quality-budget\.json|scripts/(check_test_quality|test_quality_gate)\.py)$' spec_pattern='^(litellm/(proxy|types)/.*|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|scss|md|mdx|yml|yaml|html)$' ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$' -# CI's lint job (test-linting.yml) is make lint: litellm/ checks plus two test-tree -# steps (ruff over ruff-tests.toml and the test-quality budget). Trigger the slow -# make lint on litellm/ files only; without them, the test-tree steps run on their own below. litellm_py_files=$(scope_match "$litellm_py_pattern") e2e_py_files=$(scope_match "$e2e_py_pattern") test_tree_files=$(scope_match "$test_tree_pattern") diff --git a/tests/test_litellm/test_pre_commit_lint.py b/tests/test_litellm/test_pre_commit_lint.py index e17abf2fa9f..b84cb8aa657 100644 --- a/tests/test_litellm/test_pre_commit_lint.py +++ b/tests/test_litellm/test_pre_commit_lint.py @@ -472,7 +472,13 @@ def test_tests_only_change_runs_the_whole_test_tree_ruff_and_the_quality_gate(tm @pytest.mark.parametrize( "changed", - ["ruff-tests.toml", "test-quality-budget.json", "scripts/check_test_quality.py", "tests/e2e/test_x.py"], + [ + "ruff-tests.toml", + "test-quality-budget.json", + "scripts/check_test_quality.py", + "scripts/test_quality_gate.py", + "tests/e2e/test_x.py", + ], ) def test_test_tree_lint_inputs_trigger_the_test_tree_checks(tmp_path: Path, changed: str) -> None: repo, bin_dir = _sandbox(tmp_path) From 2d2b5dabf27405fd413dfbfd71aef5a304775a75 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 01:44:16 -0700 Subject: [PATCH 4/5] fix(test-quality-gate): tear the base worktree down on SIGTERM and SIGHUP --- scripts/test_quality_gate.py | 22 ++++-- tests/test_litellm/test_test_quality_gate.py | 77 ++++++++++++++++++++ 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/scripts/test_quality_gate.py b/scripts/test_quality_gate.py index 7d34b194f1c..4f79c9488b1 100644 --- a/scripts/test_quality_gate.py +++ b/scripts/test_quality_gate.py @@ -34,13 +34,14 @@ import argparse import json import re import shutil +import signal import subprocess import sys import tempfile from collections import Counter from collections.abc import Mapping, Sequence from pathlib import Path -from types import MappingProxyType +from types import FrameType, MappingProxyType from typing import Final, NamedTuple REPO_ROOT: Final = Path(__file__).resolve().parent.parent @@ -48,6 +49,7 @@ CHECKER: Final = REPO_ROOT / "scripts" / "check_test_quality.py" BUDGET_PATH: Final = REPO_ROOT / "test-quality-budget.json" TARGET: Final = "tests" DEFAULT_BASE: Final = "origin/litellm_internal_staging" +TERMINATION_SIGNALS: Final = (signal.SIGTERM, signal.SIGHUP) _HUNK: Final = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@", re.MULTILINE) _FILE_HEADER: Final = re.compile(r"^\+\+\+ b/(.+)$", re.MULTILINE) @@ -116,22 +118,28 @@ def count_by_rule(violations: Sequence[Violation]) -> Mapping[str, int]: return MappingProxyType(dict(Counter(v.code for v in violations))) -def base_counts(ref: str) -> Mapping[str, int]: +def _exit_on_termination(signum: int, _frame: FrameType | None) -> None: + raise SystemExit(128 + signum) + + +def base_counts(ref: str, repo_root: Path = REPO_ROOT, checker: Path = CHECKER) -> Mapping[str, int]: """Rule counts at `ref`, measured with the *current* rule logic rather than whatever the checker looked like at that commit.""" + for termination in TERMINATION_SIGNALS: + signal.signal(termination, _exit_on_termination) parent: Final = Path(tempfile.mkdtemp(prefix="tq_base_")) worktree: Final = parent / "wt" try: - _run(["git", "worktree", "add", "--detach", str(worktree), ref]) + _run(["git", "worktree", "add", "--detach", str(worktree), ref], cwd=repo_root) (worktree / "scripts").mkdir(parents=True, exist_ok=True) - checker: Final = worktree / "scripts" / "check_test_quality.py" - shutil.copy(CHECKER, checker) - return count_by_rule(_check(worktree, checker)) + base_checker: Final = worktree / "scripts" / "check_test_quality.py" + shutil.copy(checker, base_checker) + return count_by_rule(_check(worktree, base_checker)) finally: # Teardown must never raise, or it masks the real error when the body failed. subprocess.run( ["git", "worktree", "remove", "--force", str(worktree)], - cwd=REPO_ROOT, capture_output=True, text=True, + cwd=repo_root, capture_output=True, text=True, ) shutil.rmtree(parent, ignore_errors=True) diff --git a/tests/test_litellm/test_test_quality_gate.py b/tests/test_litellm/test_test_quality_gate.py index 8cce6bc735a..0664153f671 100644 --- a/tests/test_litellm/test_test_quality_gate.py +++ b/tests/test_litellm/test_test_quality_gate.py @@ -9,7 +9,13 @@ file:line. """ import importlib.util +import os +import signal +import subprocess import sys +import time +from collections.abc import Callable +from contextlib import suppress from pathlib import Path _REPO_ROOT = Path(__file__).resolve().parents[2] @@ -23,6 +29,15 @@ _spec.loader.exec_module(gate) _BUDGET = {"TQ001": {"limit": 10}, "TQ003": {"limit": 5}} +_SCAN_BASE = ( + "import importlib.util, pathlib, sys\n" + "spec = importlib.util.spec_from_file_location('test_quality_gate', sys.argv[1])\n" + "gate = importlib.util.module_from_spec(spec)\n" + "sys.modules[spec.name] = gate\n" + "spec.loader.exec_module(gate)\n" + "gate.base_counts('HEAD', repo_root=pathlib.Path(sys.argv[2]), checker=pathlib.Path(sys.argv[3]))\n" +) + def test_a_rule_within_its_limit_is_not_a_breach(): assert gate.evaluate({"TQ001": 10}, {"TQ001": 10}, _BUDGET) == () @@ -146,3 +161,65 @@ def test_the_shipped_budget_covers_every_rule_the_checker_can_emit(): budget = json.loads((_REPO_ROOT / "test-quality-budget.json").read_text()) assert set(budget) == {"TQ001", "TQ002", "TQ003", "TQ004", "TQ005", "TQ006", "TQ007", "TQ008"} assert all(spec["limit"] >= 0 for spec in budget.values()) + + +def _git(cwd: Path, *args: str) -> str: + proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + return proc.stdout.strip() + + +def _committed_repo(tmp_path: Path) -> Path: + repo = tmp_path / "repo" + (repo / "tests").mkdir(parents=True) + (repo / "tests" / "test_seed.py").write_text("def test_seed():\n assert True\n") + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.email", "gate@example.com") + _git(repo, "config", "user.name", "gate") + _git(repo, "config", "commit.gpgsign", "false") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "seed") + return repo + + +def _wait_until(predicate: Callable[[], bool], timeout_seconds: float) -> bool: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.05) + return predicate() + + +def _reap(process: subprocess.Popen[bytes]) -> None: + with suppress(subprocess.TimeoutExpired): + process.wait(timeout=10) + if process.poll() is None: + process.kill() + process.wait(timeout=10) + + +def _registered_worktrees(repo: Path) -> int: + listing = _git(repo, "worktree", "list", "--porcelain") + return sum(line.startswith("worktree ") for line in listing.splitlines()) + + +def test_a_terminated_base_scan_still_removes_its_worktree(tmp_path: Path) -> None: + repo = _committed_repo(tmp_path) + scanning = tmp_path / "scanning" + slow_checker = tmp_path / "slow_checker.py" + slow_checker.write_text(f"import pathlib, time\npathlib.Path({str(scanning)!r}).touch()\ntime.sleep(30)\n") + temp_dir = tmp_path / "tmp" + temp_dir.mkdir() + scan = subprocess.Popen( + [sys.executable, "-c", _SCAN_BASE, str(_MODULE_PATH), str(repo), str(slow_checker)], + env={**os.environ, "TMPDIR": str(temp_dir)}, + ) + try: + assert _wait_until(scanning.exists, 30), "the base scan never reached the checker" + scan.send_signal(signal.SIGTERM) + assert scan.wait(timeout=30) == 128 + signal.SIGTERM + finally: + _reap(scan) + assert _registered_worktrees(repo) == 1 + assert list(temp_dir.iterdir()) == [] From 952f082e3ee80eb6bec0855a752ee3c101652744 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 02:07:14 -0700 Subject: [PATCH 5/5] fix(test-quality-gate): keep a termination signal the parent already ignores ignored The SIGTERM/SIGHUP teardown handlers were installed unconditionally, so a base scan started under nohup (SIGHUP inherited as SIG_IGN) would start dying on hangups it was told to ignore. Install them only where the disposition is still the default, and cover the ignored case with a regression test that hangs up a scan started with SIGHUP ignored and expects it to finish. --- scripts/test_quality_gate.py | 9 +++- tests/test_litellm/test_test_quality_gate.py | 56 ++++++++++++++++---- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/scripts/test_quality_gate.py b/scripts/test_quality_gate.py index 4f79c9488b1..c3316915b5e 100644 --- a/scripts/test_quality_gate.py +++ b/scripts/test_quality_gate.py @@ -122,11 +122,16 @@ def _exit_on_termination(signum: int, _frame: FrameType | None) -> None: raise SystemExit(128 + signum) +def _install_termination_handlers() -> None: + for termination in TERMINATION_SIGNALS: + if signal.getsignal(termination) == signal.SIG_DFL: + signal.signal(termination, _exit_on_termination) + + def base_counts(ref: str, repo_root: Path = REPO_ROOT, checker: Path = CHECKER) -> Mapping[str, int]: """Rule counts at `ref`, measured with the *current* rule logic rather than whatever the checker looked like at that commit.""" - for termination in TERMINATION_SIGNALS: - signal.signal(termination, _exit_on_termination) + _install_termination_handlers() parent: Final = Path(tempfile.mkdtemp(prefix="tq_base_")) worktree: Final = parent / "wt" try: diff --git a/tests/test_litellm/test_test_quality_gate.py b/tests/test_litellm/test_test_quality_gate.py index 0664153f671..b3511eabbfb 100644 --- a/tests/test_litellm/test_test_quality_gate.py +++ b/tests/test_litellm/test_test_quality_gate.py @@ -17,6 +17,7 @@ import time from collections.abc import Callable from contextlib import suppress from pathlib import Path +from typing import NamedTuple _REPO_ROOT = Path(__file__).resolve().parents[2] _MODULE_PATH = _REPO_ROOT / "scripts" / "test_quality_gate.py" @@ -37,6 +38,7 @@ _SCAN_BASE = ( "spec.loader.exec_module(gate)\n" "gate.base_counts('HEAD', repo_root=pathlib.Path(sys.argv[2]), checker=pathlib.Path(sys.argv[3]))\n" ) +_SCAN_BASE_WITH_SIGHUP_IGNORED = "import signal\nsignal.signal(signal.SIGHUP, signal.SIG_IGN)\n" + _SCAN_BASE def test_a_rule_within_its_limit_is_not_a_breach(): @@ -204,22 +206,56 @@ def _registered_worktrees(repo: Path) -> int: return sum(line.startswith("worktree ") for line in listing.splitlines()) -def test_a_terminated_base_scan_still_removes_its_worktree(tmp_path: Path) -> None: +class _StalledScan(NamedTuple): + process: subprocess.Popen[bytes] + repo: Path + release: Path + temp_dir: Path + + +def _base_scan_stalled_in_its_checker(tmp_path: Path, driver: str) -> _StalledScan: repo = _committed_repo(tmp_path) scanning = tmp_path / "scanning" + release = tmp_path / "release" slow_checker = tmp_path / "slow_checker.py" - slow_checker.write_text(f"import pathlib, time\npathlib.Path({str(scanning)!r}).touch()\ntime.sleep(30)\n") + slow_checker.write_text( + "import pathlib, time\n" + f"pathlib.Path({str(scanning)!r}).touch()\n" + f"while not pathlib.Path({str(release)!r}).exists():\n" + " time.sleep(0.05)\n" + ) temp_dir = tmp_path / "tmp" temp_dir.mkdir() scan = subprocess.Popen( - [sys.executable, "-c", _SCAN_BASE, str(_MODULE_PATH), str(repo), str(slow_checker)], + [sys.executable, "-c", driver, str(_MODULE_PATH), str(repo), str(slow_checker)], env={**os.environ, "TMPDIR": str(temp_dir)}, ) - try: - assert _wait_until(scanning.exists, 30), "the base scan never reached the checker" - scan.send_signal(signal.SIGTERM) - assert scan.wait(timeout=30) == 128 + signal.SIGTERM - finally: + if not _wait_until(scanning.exists, 30): _reap(scan) - assert _registered_worktrees(repo) == 1 - assert list(temp_dir.iterdir()) == [] + raise AssertionError("the base scan never reached the checker") + return _StalledScan(scan, repo, release, temp_dir) + + +def test_a_terminated_base_scan_still_removes_its_worktree(tmp_path: Path) -> None: + stalled = _base_scan_stalled_in_its_checker(tmp_path, _SCAN_BASE) + try: + stalled.process.send_signal(signal.SIGTERM) + assert stalled.process.wait(timeout=30) == 128 + signal.SIGTERM + finally: + _reap(stalled.process) + assert _registered_worktrees(stalled.repo) == 1 + assert list(stalled.temp_dir.iterdir()) == [] + + +def test_a_base_scan_keeps_ignoring_the_hangup_its_parent_ignored(tmp_path: Path) -> None: + stalled = _base_scan_stalled_in_its_checker(tmp_path, _SCAN_BASE_WITH_SIGHUP_IGNORED) + try: + stalled.process.send_signal(signal.SIGHUP) + time.sleep(1) + assert stalled.process.poll() is None, "a hangup the parent ignored killed the scan" + stalled.release.touch() + assert stalled.process.wait(timeout=30) == 0 + finally: + _reap(stalled.process) + assert _registered_worktrees(stalled.repo) == 1 + assert list(stalled.temp_dir.iterdir()) == []