mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-28 01:32:17 +00:00
635a718ba1
* ci: cut CircleCI wall time without loosening test isolation * fix(ci): parse integration split files that follow --results The CircleCI machine image ships Python 3.12.2, whose argparse leaves the files positional empty when it follows an option and another positional, so every extensions node exited with 'unrecognized arguments'. Reproduced on 3.12.2; parse_intermixed_args selects the files on 3.12.2, 3.12.13 and 3.13 * test(ci): resolve command references in the Rust toolchain guard The Windows rustup install moved into the install_windows_toolchain command, which the guard only recognized for install_rust. It now accepts any command that installs a pinned rustup and reads the Windows toolchain pin from it * ci: cache the Windows release cargo build from main windows_release_wheel rebuilt every dependency with fat LTO on each run. It now restores the release target and cargo registry saved by main's scheduled run, drops the workspace crates' fingerprints so they always rebuild from the checked-out source, and still runs the full LTO link * ci: run the Windows release wheel build on windows.xlarge The fat-LTO release build is the slowest job in the pipeline; more cores speed up the dependency compile ahead of the final link * ci: skip the Windows fingerprint cleanup when the cargo cache missed On a cold cache the release fingerprint directory does not exist, and the CircleCI PowerShell wrapper failed the step on the suppressed not-found error
645 lines
24 KiB
Python
645 lines
24 KiB
Python
import os
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import time
|
|
from collections.abc import Callable
|
|
from contextlib import suppress
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from tests._process_helpers import process_is_gone
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts" / "pre_commit_lint.sh"
|
|
WHOLE_TREE_RUFF = "run --no-sync ruff check --config ruff-tests.toml tests"
|
|
TEST_TREE_RAN = "ran: test-tree lint (ruff-tests.toml + test-quality budget)"
|
|
TEST_TREE_SKIPPED = (
|
|
"skipped: test-tree lint (ruff-tests.toml + test-quality budget) "
|
|
"(no tests/ Python files or test-tree lint inputs in scope)"
|
|
)
|
|
|
|
BARRIER_HELPER = """barrier_sync() {
|
|
touch "$STUB_BARRIER_DIR/$1.started"
|
|
for other in $2; do
|
|
tries=0
|
|
while [ ! -f "$STUB_BARRIER_DIR/$other.started" ]; do
|
|
tries=$((tries + 1))
|
|
if [ "$tries" -gt 100 ]; then
|
|
echo "barrier timeout: $1 never saw $other start" >&2
|
|
exit 1
|
|
fi
|
|
sleep 0.1
|
|
done
|
|
done
|
|
}
|
|
"""
|
|
|
|
MAKE_STUB = """#!/bin/sh
|
|
. "$STUB_BIN/barrier.sh"
|
|
[ -n "${STUB_ARGS_DIR:-}" ] && echo "$*" >> "$STUB_ARGS_DIR/make.args"
|
|
case "$*" in
|
|
lint)
|
|
[ "${STUB_FAIL:-}" = "make-lint" ] && exit 1
|
|
[ -n "${STUB_BARRIER_DIR:-}" ] && barrier_sync python "dashboard genapi"
|
|
if [ -n "${STUB_HANG_DIR:-}" ]; then
|
|
echo "$$" > "$STUB_HANG_DIR/make.pid"
|
|
touch "$STUB_HANG_DIR/make.started"
|
|
sleep 60
|
|
fi
|
|
;;
|
|
lint-test-quality)
|
|
[ "${STUB_FAIL:-}" = "test-quality" ] && exit 1
|
|
;;
|
|
esac
|
|
exit 0
|
|
"""
|
|
|
|
NPX_STUB = """#!/bin/sh
|
|
. "$STUB_BIN/barrier.sh"
|
|
case "$*" in
|
|
prettier*)
|
|
[ -n "${STUB_BARRIER_DIR:-}" ] && barrier_sync dashboard "python genapi"
|
|
;;
|
|
"eslint --no-warn-ignored"*)
|
|
[ "${STUB_FAIL:-}" = "eslint" ] && exit 1
|
|
;;
|
|
"eslint . -f json"*)
|
|
if [ -n "${STUB_HANG_DIR:-}" ]; then
|
|
touch "$STUB_HANG_DIR/eslint_report.started"
|
|
sleep 60
|
|
fi
|
|
;;
|
|
esac
|
|
exit 0
|
|
"""
|
|
|
|
UV_STUB = """#!/bin/sh
|
|
. "$STUB_BIN/barrier.sh"
|
|
case "$*" in
|
|
*orjson*)
|
|
[ -n "${STUB_BARRIER_DIR:-}" ] && barrier_sync genapi "python dashboard"
|
|
;;
|
|
"run --no-sync ruff check --config ruff-tests.toml"*)
|
|
[ -n "${STUB_ARGS_DIR:-}" ] && echo "$*" >> "$STUB_ARGS_DIR/ruff_tests.args"
|
|
[ "${STUB_FAIL:-}" = "tests-ruff" ] && exit 1
|
|
;;
|
|
esac
|
|
exit 0
|
|
"""
|
|
|
|
NPM_STUB = """#!/bin/sh
|
|
case "$*" in
|
|
"run gen:api")
|
|
[ "${STUB_FAIL:-}" = "gen-api" ] && exit 1
|
|
;;
|
|
esac
|
|
exit 0
|
|
"""
|
|
|
|
NODE_STUB = """#!/bin/sh
|
|
exit 0
|
|
"""
|
|
|
|
|
|
def _write_executable(path: Path, body: str) -> None:
|
|
path.write_text(body)
|
|
path.chmod(0o755)
|
|
|
|
|
|
def _sandbox(tmp_path: Path) -> tuple[Path, Path]:
|
|
repo = tmp_path / "repo"
|
|
(repo / "litellm" / "proxy").mkdir(parents=True)
|
|
(repo / "litellm" / "foo.py").write_text("x = 1\n")
|
|
(repo / "litellm" / "proxy" / "spec.py").write_text("y = 2\n")
|
|
dashboard = repo / "ui" / "litellm-dashboard"
|
|
(dashboard / "src").mkdir(parents=True)
|
|
(dashboard / "node_modules").mkdir()
|
|
(dashboard / "src" / "app.ts").write_text("export {}\n")
|
|
subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
|
|
subprocess.run(["git", "add", "."], cwd=repo, check=True)
|
|
|
|
bin_dir = tmp_path / "bin"
|
|
bin_dir.mkdir()
|
|
(bin_dir / "barrier.sh").write_text(BARRIER_HELPER)
|
|
_write_executable(bin_dir / "make", MAKE_STUB)
|
|
_write_executable(bin_dir / "npx", NPX_STUB)
|
|
_write_executable(bin_dir / "uv", UV_STUB)
|
|
_write_executable(bin_dir / "npm", NPM_STUB)
|
|
_write_executable(bin_dir / "node", NODE_STUB)
|
|
return repo, bin_dir
|
|
|
|
|
|
def _env(repo: Path, bin_dir: Path, extra_env: dict[str, str]) -> dict[str, str]:
|
|
return {
|
|
"PATH": os.pathsep.join([str(bin_dir), "/usr/bin", "/bin"]),
|
|
"HOME": str(repo.parent),
|
|
"STUB_BIN": str(bin_dir),
|
|
**extra_env,
|
|
}
|
|
|
|
|
|
def _run(repo: Path, bin_dir: Path, extra_env: dict[str, str]) -> subprocess.CompletedProcess[str]:
|
|
env = _env(repo, bin_dir, extra_env)
|
|
return subprocess.run(
|
|
[str(SCRIPT)],
|
|
cwd=repo,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
timeout=120,
|
|
)
|
|
|
|
|
|
def _commit_all(repo: Path, message: str) -> None:
|
|
subprocess.run(["git", "add", "."], cwd=repo, check=True)
|
|
subprocess.run(
|
|
["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", message],
|
|
cwd=repo,
|
|
check=True,
|
|
)
|
|
|
|
|
|
def _set_base_ref(repo: Path, branch: str = "release_branch") -> None:
|
|
remote = repo.parent / "remote.git"
|
|
subprocess.run(["git", "clone", "-q", "--bare", str(repo), str(remote)], check=True)
|
|
subprocess.run(["git", "update-ref", f"refs/heads/{branch}", "HEAD"], cwd=remote, check=True)
|
|
subprocess.run(["git", "symbolic-ref", "HEAD", f"refs/heads/{branch}"], cwd=remote, check=True)
|
|
subprocess.run(["git", "remote", "add", "origin", str(remote)], cwd=repo, check=True)
|
|
|
|
|
|
def _stage_file(repo: Path, relative: str, body: str) -> None:
|
|
path = repo / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(body)
|
|
subprocess.run(["git", "add", relative], cwd=repo, check=True)
|
|
|
|
|
|
@pytest.mark.parametrize("branch", ["release_branch", "main"])
|
|
def test_nothing_staged_scopes_to_working_tree_diff_and_runs_checks(tmp_path: Path, branch: str) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
_set_base_ref(repo, branch)
|
|
(repo / "litellm" / "foo.py").write_text("x = 2\n")
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "nothing staged; scoping to the working tree's diff" in proc.stdout
|
|
assert "litellm/foo.py" in proc.stdout
|
|
assert "linting Python" in proc.stdout
|
|
|
|
|
|
def test_nothing_staged_checks_committed_branch_changes(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
_set_base_ref(repo)
|
|
(repo / "litellm" / "foo.py").write_text("x = 2\n")
|
|
_commit_all(repo, "branch change")
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "nothing staged; scoping to the working tree's diff" in proc.stdout
|
|
assert "linting Python" in proc.stdout
|
|
|
|
|
|
def test_nothing_staged_includes_untracked_files_in_scope(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
_set_base_ref(repo)
|
|
(repo / "litellm" / "brand_new.py").write_text("z = 3\n")
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "litellm/brand_new.py" in proc.stdout
|
|
assert "linting Python" in proc.stdout
|
|
|
|
|
|
def test_nothing_staged_deletion_only_branch_triggers_checks(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
_set_base_ref(repo)
|
|
(repo / "litellm" / "foo.py").unlink()
|
|
_commit_all(repo, "delete module")
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "nothing to check" not in proc.stdout
|
|
assert "litellm/foo.py" in proc.stdout
|
|
assert "linting Python" in proc.stdout
|
|
assert "ruff format --check" not in proc.stdout
|
|
|
|
|
|
def test_staged_deletion_triggers_checks_without_feeding_missing_files_to_tools(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
subprocess.run(["git", "rm", "-q", "litellm/foo.py"], cwd=repo, check=True)
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "nothing staged" not in proc.stdout
|
|
assert "linting Python" in proc.stdout
|
|
assert "ruff format --check" not in proc.stdout
|
|
|
|
|
|
def test_deleted_dashboard_file_still_triggers_dashboard_lint(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
_set_base_ref(repo)
|
|
(repo / "ui" / "litellm-dashboard" / "src" / "app.ts").unlink()
|
|
_commit_all(repo, "delete dashboard file")
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "linting dashboard" in proc.stdout
|
|
|
|
|
|
def test_nothing_staged_and_no_changes_is_an_explicit_no_op(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
_set_base_ref(repo)
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "nothing to check" in proc.stdout
|
|
assert "check: PASS" in proc.stdout
|
|
assert "linting Python" not in proc.stdout
|
|
|
|
|
|
def test_nothing_staged_without_a_base_ref_fails_with_a_fetch_hint(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 1
|
|
assert "Cannot verify the base branch against origin" in proc.stdout
|
|
assert "explicit base ref" in proc.stdout
|
|
assert "check: FAIL" in proc.stdout
|
|
|
|
|
|
def test_partial_staging_warns_which_checks_were_skipped(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
(repo / "notes.md").write_text("hi\n")
|
|
subprocess.run(["git", "add", "notes.md"], cwd=repo, check=True)
|
|
(repo / "litellm" / "foo.py").write_text("x = 4\n")
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "SKIPPED Python lint (make lint)" in proc.stdout
|
|
assert "litellm/foo.py" in proc.stdout
|
|
assert "linting Python" not in proc.stdout
|
|
|
|
|
|
def test_python_dashboard_and_gen_api_blocks_run_concurrently_with_grouped_output(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
barrier_dir = tmp_path / "barrier"
|
|
barrier_dir.mkdir()
|
|
proc = _run(repo, bin_dir, {"STUB_BARRIER_DIR": str(barrier_dir)})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "barrier timeout" not in proc.stdout + proc.stderr
|
|
python_at = proc.stdout.index("linting Python")
|
|
dashboard_at = proc.stdout.index("linting dashboard")
|
|
gen_api_at = proc.stdout.index("API types")
|
|
assert python_at < dashboard_at < gen_api_at
|
|
|
|
|
|
def test_all_blocks_passing_exits_zero(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
|
|
|
|
def test_full_output_is_saved_to_a_log_file_in_the_git_dir(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
(repo / "scratch.txt").write_text("")
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
log_file = repo / ".git" / "pre_commit_lint.log"
|
|
log = log_file.read_text()
|
|
assert "linting Python" in log
|
|
assert "linting dashboard" in log
|
|
assert "API types" in log
|
|
assert "unstaged/untracked changes" in log
|
|
assert f"check: full log: {log_file}" in proc.stdout
|
|
assert "check: full log:" not in log
|
|
|
|
|
|
def test_unwritable_log_warns_and_falls_back_to_running_without_one(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
(repo / ".git" / "pre_commit_lint.log").mkdir()
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "linting Python" in proc.stdout
|
|
assert "output will not be saved" in proc.stderr
|
|
assert "check: full log:" not in proc.stdout
|
|
failing = _run(repo, bin_dir, {"STUB_FAIL": "make-lint"})
|
|
assert failing.returncode == 1
|
|
|
|
|
|
def test_failing_run_exit_code_survives_the_log_pipeline(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
proc = _run(repo, bin_dir, {"STUB_FAIL": "make-lint"})
|
|
assert proc.returncode == 1
|
|
log = (repo / ".git" / "pre_commit_lint.log").read_text()
|
|
assert "Python lint failed" in log
|
|
|
|
|
|
def _wait_until(predicate: Callable[[], bool], timeout_seconds: float) -> bool:
|
|
deadline = time.monotonic() + timeout_seconds
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return True
|
|
time.sleep(0.05)
|
|
return predicate()
|
|
|
|
|
|
def test_interrupt_kills_background_jobs_and_removes_logs(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
hang_dir = tmp_path / "hang"
|
|
hang_dir.mkdir()
|
|
tmp_dir = tmp_path / "tmpdir"
|
|
tmp_dir.mkdir()
|
|
extra = {"STUB_HANG_DIR": str(hang_dir), "TMPDIR": str(tmp_dir)}
|
|
proc = subprocess.Popen(
|
|
[str(SCRIPT)],
|
|
cwd=repo,
|
|
env=_env(repo, bin_dir, extra),
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
start_new_session=True,
|
|
)
|
|
try:
|
|
assert _wait_until((hang_dir / "make.started").exists, 10)
|
|
assert _wait_until((hang_dir / "eslint_report.started").exists, 10)
|
|
os.killpg(proc.pid, signal.SIGINT)
|
|
assert proc.wait(timeout=10) != 0
|
|
make_pid = int((hang_dir / "make.pid").read_text())
|
|
assert process_is_gone(make_pid, within_seconds=5)
|
|
assert _wait_until(lambda: not any(tmp_dir.iterdir()), 5), list(tmp_dir.iterdir())
|
|
finally:
|
|
with suppress(ProcessLookupError, PermissionError):
|
|
os.killpg(proc.pid, signal.SIGTERM)
|
|
|
|
|
|
def test_interrupt_spares_the_invoking_process(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
hang_dir = tmp_path / "hang"
|
|
hang_dir.mkdir()
|
|
marker = tmp_path / "invoker_survived"
|
|
proc = subprocess.Popen(
|
|
["bash", "-c", 'trap : INT; "$1"; echo "$?" > "$2"', "bash", str(SCRIPT), str(marker)],
|
|
cwd=repo,
|
|
env=_env(repo, bin_dir, {"STUB_HANG_DIR": str(hang_dir)}),
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
start_new_session=True,
|
|
)
|
|
try:
|
|
assert _wait_until((hang_dir / "make.started").exists, 10)
|
|
assert _wait_until((hang_dir / "eslint_report.started").exists, 10)
|
|
os.killpg(proc.pid, signal.SIGINT)
|
|
assert proc.wait(timeout=10) == 0
|
|
assert _wait_until(marker.exists, 5)
|
|
assert marker.read_text().strip() == "130"
|
|
finally:
|
|
with suppress(ProcessLookupError, PermissionError):
|
|
os.killpg(proc.pid, signal.SIGTERM)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("fail", "message"),
|
|
[
|
|
("make-lint", "Python lint failed"),
|
|
("eslint", "Dashboard lint failed"),
|
|
("gen-api", "npm run gen:api failed"),
|
|
],
|
|
)
|
|
def test_a_failing_block_fails_the_whole_run(tmp_path: Path, fail: str, message: str) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
proc = _run(repo, bin_dir, {"STUB_FAIL": fail})
|
|
assert proc.returncode == 1
|
|
assert message in proc.stdout + proc.stderr
|
|
|
|
|
|
def test_run_ends_with_a_summary_of_ran_and_skipped_blocks(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "check: summary" in proc.stdout
|
|
assert "ran: Python lint (make lint)" in proc.stdout
|
|
assert "ran: dashboard lint (prettier + eslint + lint budgets)" in proc.stdout
|
|
assert "ran: dashboard API-type sync (npm run gen:api)" in proc.stdout
|
|
assert "skipped: tests/e2e checks (basedpyright + raw HTTP client ban) (no tests/e2e Python files in scope)" in proc.stdout
|
|
assert TEST_TREE_SKIPPED in proc.stdout
|
|
assert "check: PASS" in proc.stdout
|
|
assert "check: FAIL" not in proc.stdout
|
|
|
|
|
|
def test_staged_files_matching_no_check_print_an_explicit_noop_note_and_nonempty_log(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
_stage_file(repo, "scripts/tool.py", "def main() -> None: ...\n")
|
|
proc = _run(repo, bin_dir, {})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "no gating lint check matches the files in scope, so nothing ran" in proc.stdout
|
|
assert "scripts/tool.py" in proc.stdout
|
|
assert "a no-op, not a lint verdict" in proc.stdout
|
|
assert "check: PASS" in proc.stdout
|
|
assert "linting Python" not in proc.stdout
|
|
log = (repo / ".git" / "pre_commit_lint.log").read_text()
|
|
assert "check: summary" in log
|
|
assert "skipped: Python lint (make lint) (no litellm/ Python files in scope)" in log
|
|
assert TEST_TREE_SKIPPED in log
|
|
|
|
|
|
def _recorded(args_dir: Path, name: str) -> list[str]:
|
|
path = args_dir / name
|
|
return path.read_text().splitlines() if path.exists() else []
|
|
|
|
|
|
def test_tests_only_change_runs_the_whole_test_tree_ruff_and_the_quality_gate(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
args_dir = tmp_path / "args"
|
|
args_dir.mkdir()
|
|
_stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n")
|
|
_stage_file(repo, "tests/fixtures/data.json", "{}\n")
|
|
proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert _recorded(args_dir, "ruff_tests.args") == [WHOLE_TREE_RUFF]
|
|
assert _recorded(args_dir, "make.args") == ["lint-test-quality"]
|
|
assert TEST_TREE_RAN in proc.stdout
|
|
assert "no gating lint check matches" not in proc.stdout
|
|
assert "linting Python" not in proc.stdout
|
|
assert "check: PASS" in proc.stdout
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"changed",
|
|
[
|
|
"ruff-tests.toml",
|
|
"test-quality-budget.json",
|
|
"scripts/check_test_quality.py",
|
|
"scripts/test_quality_gate.py",
|
|
"tests/e2e/test_x.py",
|
|
],
|
|
)
|
|
def test_test_tree_lint_inputs_trigger_the_test_tree_checks(tmp_path: Path, changed: str) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
args_dir = tmp_path / "args"
|
|
args_dir.mkdir()
|
|
_stage_file(repo, changed, "x = 1\n")
|
|
proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert _recorded(args_dir, "ruff_tests.args") == [WHOLE_TREE_RUFF]
|
|
assert "lint-test-quality" in _recorded(args_dir, "make.args")
|
|
assert TEST_TREE_RAN in proc.stdout
|
|
|
|
|
|
def test_nothing_staged_tests_only_working_tree_change_runs_the_test_tree_checks(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n")
|
|
_commit_all(repo, "base")
|
|
_set_base_ref(repo)
|
|
args_dir = tmp_path / "args"
|
|
args_dir.mkdir()
|
|
(repo / "tests" / "test_a.py").write_text("def test_a() -> None:\n assert True\n")
|
|
proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "nothing staged; scoping to the working tree's diff" in proc.stdout
|
|
assert _recorded(args_dir, "ruff_tests.args") == [WHOLE_TREE_RUFF]
|
|
assert _recorded(args_dir, "make.args") == ["lint-test-quality"]
|
|
|
|
|
|
def test_a_failing_test_tree_ruff_fails_the_run_and_still_runs_the_quality_gate(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
args_dir = tmp_path / "args"
|
|
args_dir.mkdir()
|
|
_stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n")
|
|
proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir), "STUB_FAIL": "tests-ruff"})
|
|
assert proc.returncode == 1
|
|
assert "Test-tree ruff failed" in proc.stdout + proc.stderr
|
|
assert "check: FAIL" in proc.stdout
|
|
assert _recorded(args_dir, "make.args") == ["lint-test-quality"]
|
|
|
|
|
|
def test_a_failing_quality_gate_fails_a_tests_only_run(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
_stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n")
|
|
proc = _run(repo, bin_dir, {"STUB_FAIL": "test-quality"})
|
|
assert proc.returncode == 1
|
|
assert "Test-quality budget failed" in proc.stdout + proc.stderr
|
|
assert "check: FAIL" in proc.stdout
|
|
|
|
|
|
def test_tests_changed_alongside_litellm_files_defer_to_make_lint(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
args_dir = tmp_path / "args"
|
|
args_dir.mkdir()
|
|
_stage_file(repo, "litellm/foo.py", "x = 2\n")
|
|
_stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n")
|
|
proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir), "STUB_FAIL": "test-quality"})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "linting Python" in proc.stdout
|
|
assert _recorded(args_dir, "ruff_tests.args") == []
|
|
assert _recorded(args_dir, "make.args") == ["lint"]
|
|
assert TEST_TREE_RAN in proc.stdout
|
|
|
|
|
|
def test_deleted_test_file_still_runs_the_test_tree_checks(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n")
|
|
_commit_all(repo, "base")
|
|
args_dir = tmp_path / "args"
|
|
args_dir.mkdir()
|
|
subprocess.run(["git", "rm", "-q", "tests/test_a.py"], cwd=repo, check=True)
|
|
proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert _recorded(args_dir, "ruff_tests.args") == [WHOLE_TREE_RUFF]
|
|
assert _recorded(args_dir, "make.args") == ["lint-test-quality"]
|
|
assert TEST_TREE_RAN in proc.stdout
|
|
|
|
|
|
def test_partial_staging_warns_when_test_files_are_left_unstaged(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n")
|
|
_commit_all(repo, "base")
|
|
args_dir = tmp_path / "args"
|
|
args_dir.mkdir()
|
|
_stage_file(repo, "notes.md", "hi\n")
|
|
(repo / "tests" / "test_a.py").write_text("def test_a() -> None:\n assert True\n")
|
|
proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "SKIPPED test-tree lint (ruff-tests.toml + test-quality budget)" in proc.stdout
|
|
assert "tests/test_a.py" in proc.stdout
|
|
assert _recorded(args_dir, "ruff_tests.args") == []
|
|
assert _recorded(args_dir, "make.args") == []
|
|
|
|
|
|
def test_run_queues_through_the_machine_wide_gate_slot_lock(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
lock_dir = tmp_path / "gate-locks"
|
|
proc = _run(repo, bin_dir, {"LITELLM_GATE_SLOT_DIR": str(lock_dir)})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert (lock_dir / "slot-0.lock").exists()
|
|
|
|
|
|
def test_run_under_a_held_slot_skips_reacquiring_the_gate_lock(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
lock_dir = tmp_path / "gate-locks"
|
|
proc = _run(
|
|
repo,
|
|
bin_dir,
|
|
{"LITELLM_GATE_SLOT_DIR": str(lock_dir), "LITELLM_GATE_SLOT_HELD": "1"},
|
|
)
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert not lock_dir.exists()
|
|
|
|
|
|
def test_hook_symlink_install_still_resolves_the_slot_lock_helper(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
scripts_dir = repo / "scripts"
|
|
scripts_dir.mkdir()
|
|
shutil.copy(SCRIPT, scripts_dir / "pre_commit_lint.sh")
|
|
shutil.copy(SCRIPT.parent / "gate_slot_lock.py", scripts_dir / "gate_slot_lock.py")
|
|
(repo / ".git" / "hooks" / "pre-commit").symlink_to(Path("../../scripts/pre_commit_lint.sh"))
|
|
lock_dir = tmp_path / "gate-locks"
|
|
proc = subprocess.run(
|
|
["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "hooked"],
|
|
cwd=repo,
|
|
capture_output=True,
|
|
text=True,
|
|
env=_env(repo, bin_dir, {"LITELLM_GATE_SLOT_DIR": str(lock_dir)}),
|
|
timeout=120,
|
|
)
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert (lock_dir / "slot-0.lock").exists()
|
|
|
|
|
|
def test_failing_run_ends_with_a_fail_verdict(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
proc = _run(repo, bin_dir, {"STUB_FAIL": "make-lint"})
|
|
assert proc.returncode == 1
|
|
assert "check: FAIL" in proc.stdout
|
|
assert "check: PASS" not in proc.stdout
|
|
|
|
|
|
|
|
def test_explicit_base_scopes_offline_without_a_remote(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
(repo / "litellm" / "foo.py").write_text("x = 2\n")
|
|
proc = _run(repo, bin_dir, {"BASE_REF": "HEAD"})
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "merge base with HEAD" in proc.stdout
|
|
assert "linting Python" in proc.stdout
|
|
|
|
|
|
def test_symlinked_hook_can_resolve_default_branch(tmp_path: Path) -> None:
|
|
repo, bin_dir = _sandbox(tmp_path)
|
|
_commit_all(repo, "base")
|
|
_set_base_ref(repo, "main")
|
|
hook = repo / ".git" / "hooks" / "pre-commit"
|
|
hook.symlink_to(SCRIPT)
|
|
proc = subprocess.run(
|
|
[str(hook)], cwd=repo, capture_output=True, text=True,
|
|
env=_env(repo, bin_dir, {}), timeout=120,
|
|
)
|
|
assert proc.returncode == 0, proc.stdout + proc.stderr
|
|
assert "no branch changes vs origin/main" in proc.stdout
|