From 21a2d828dffd70b0c868d2dba487d3c372239475 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:56:32 +0000 Subject: [PATCH] ci(test-unit): drop dead misc shard paths and skip missing paths with a warning (#42603) Eight directories the misc shard named moved to tests/unit on 2026-09-20, and one missing path makes pytest-xdist collect [0 items] for the whole shard, which the exit-5 tolerance turned into a green required check running nothing. The shared Run tests step now drops a path that does not exist with a ::warning:: and runs pytest over the rest, keeping option tokens verbatim. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .github/workflows/_test-unit-base.yml | 32 +++++--- .github/workflows/test-unit.yml | 8 -- .../test_unit_shard_missing_paths.py | 74 +++++++++++++++++++ 3 files changed, 97 insertions(+), 17 deletions(-) create mode 100644 tests/test_litellm/test_unit_shard_missing_paths.py diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index f4d4fb54ba6..8faddd11df8 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -4,7 +4,13 @@ on: workflow_call: inputs: test-path: - description: "Pytest path(s) to run" + description: >- + Space-separated pytest paths to run. A path that no longer exists is + dropped with a warning instead of being passed to pytest, because one + missing path makes pytest-xdist collect nothing and report exit 5, which + the step treats as a drained shard. Options are passed through as + written, so use the `--flag=value` form: a bare `--ignore path` would + have its path existence-checked like any other token. required: true type: string workers: @@ -165,14 +171,22 @@ jobs: DIST: ${{ inputs.dist }} COVERAGE_CORE: sysmon run: | - found_path=false - for path in ${TEST_PATH}; do - if [ -e "${path%%::*}" ]; then - found_path=true - break - fi + pytest_args=() + existing_paths=0 + for token in ${TEST_PATH:?}; do + case "${token}" in + -*) pytest_args+=("${token}") ;; + *) + if [ -e "${token%%::*}" ]; then + pytest_args+=("${token}") + existing_paths=$((existing_paths + 1)) + else + echo "::warning::${token} does not exist; drop it from this shard's test-path" + fi + ;; + esac done - if [ "$found_path" = false ]; then + if [ "${existing_paths}" -eq 0 ]; then echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run" exit 0 fi @@ -181,7 +195,7 @@ jobs: xdist_args=(-n "${WORKERS}" --dist="${DIST}") fi set +e - uv run --no-sync pytest ${TEST_PATH:?} \ + uv run --no-sync pytest "${pytest_args[@]}" \ --tb=short -vv \ --maxfail="${MAX_FAILURES}" \ "${xdist_args[@]}" \ diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 49e6d7040d4..ac60a9b5f05 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -107,26 +107,18 @@ jobs: tests/test_litellm/batches tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol - tests/test_litellm/anthropic_interface tests/test_litellm/chat_completions tests/test_litellm/completion_extras - tests/test_litellm/compression tests/test_litellm/containers tests/test_litellm/endpoints - tests/test_litellm/models - tests/test_litellm/repositories tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/messages tests/test_litellm/ocr tests/test_litellm/passthrough tests/test_litellm/rag - tests/test_litellm/realtime_api tests/test_litellm/rerank_api tests/test_litellm/rust_bridge - tests/test_litellm/sandbox - tests/test_litellm/skills - tests/test_litellm/test_router tests/test_litellm/vector_stores tests/test_litellm/videos tests/test_litellm/test_*.py diff --git a/tests/test_litellm/test_unit_shard_missing_paths.py b/tests/test_litellm/test_unit_shard_missing_paths.py new file mode 100644 index 00000000000..b91c2cff764 --- /dev/null +++ b/tests/test_litellm/test_unit_shard_missing_paths.py @@ -0,0 +1,74 @@ +import os +import subprocess +import sys +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest +import yaml + +_REPO_ROOT: Final = Path(__file__).resolve().parents[2] +_BASE_WORKFLOW: Final = _REPO_ROOT / ".github" / "workflows" / "_test-unit-base.yml" +_SHARD_ENV: Final = MappingProxyType( + {"MAX_FAILURES": "10", "RERUNS": "0", "DIST": "loadscope", "TEST_TIMEOUT_SECONDS": "60", "COVERAGE_CORE": "sysmon"} +) +_UV_SHIM: Final = f'#!/usr/bin/env bash\nshift 2\nexec "{sys.executable}" -m "$@"\n' +_PASSING_TEST: Final = "def test_passes():\n assert True\n" +_FAILING_TEST: Final = "def test_fails():\n assert False\n" + + +def _run_tests_script() -> str: + workflow: Final = yaml.safe_load(_BASE_WORKFLOW.read_text()) + return next(step["run"] for step in workflow["jobs"]["run"]["steps"] if step.get("name") == "Run tests") + + +def _run_shard(tmp_path: Path, test_path: str, workers: str) -> subprocess.CompletedProcess[str]: + shim_dir: Final = tmp_path / "bin" + shim_dir.mkdir() + (shim_dir / "uv").write_text(_UV_SHIM) + (shim_dir / "uv").chmod(0o755) + (tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = '-p no:cacheprovider'\n") + return subprocess.run( + ("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _run_tests_script()), + cwd=tmp_path, + env={ + **os.environ, + **_SHARD_ENV, + "PATH": f"{shim_dir}{os.pathsep}{os.environ['PATH']}", + "TEST_PATH": test_path, + "WORKERS": workers, + }, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + + +def _write_passing_test(tmp_path: Path) -> Path: + present: Final = tmp_path / "tests" / "present" + present.mkdir(parents=True) + (present / "test_present.py").write_text(_PASSING_TEST) + return present + + +@pytest.mark.parametrize("workers", ("0", "2"), ids=("serial", "xdist")) +def test_a_missing_path_is_dropped_and_the_existing_paths_still_run(tmp_path: Path, workers: str) -> None: + _write_passing_test(tmp_path) + + result: Final = _run_shard(tmp_path, "tests/gone tests/present", workers) + + assert result.returncode == 0, result.stdout + result.stderr + assert "1 passed" in result.stdout, result.stdout + assert "::warning::tests/gone does not exist" in result.stdout + + +def test_ignore_flags_survive_the_path_filter(tmp_path: Path) -> None: + present: Final = _write_passing_test(tmp_path) + (present / "test_ignored.py").write_text(_FAILING_TEST) + + result: Final = _run_shard(tmp_path, "tests/present --ignore=tests/present/test_ignored.py", "0") + + assert result.returncode == 0, result.stdout + result.stderr + assert "1 passed" in result.stdout, result.stdout