From f2dfadcfbc7435f5282a60076e550b756fe7cd91 Mon Sep 17 00:00:00 2001 From: Silu Panda <31051721+SiluPanda@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:32:27 -0700 Subject: [PATCH] test(ci): isolate workflow coverage audit fixtures Replace the stale unrun caching-file assumption with isolated workflow fixtures for file and node-ID selectors. Keep the unnamed-file negative check and clarify which live caching cases remain outside CI. Signed-off-by: Silu Panda <31051721+SiluPanda@users.noreply.github.com> --- .github/ci-coverage-allowlist.yml | 13 ++++++----- tests/test_litellm/test_assert_ci_coverage.py | 23 +++++++++++++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index 0da07038152..443971755a7 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -10,12 +10,13 @@ test_paths: paths: - tests/rust-python-harness - reason: >- - What is left of the caching suite in tests/local_testing that runs nowhere. Every job that - globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and - not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants), - and no job names these files the way redis_caching_unit_tests names test_dual_cache.py. - The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now - run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider + Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that + glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and + not caching and not cache"`) or keep only another keyword (langfuse, router, assistants). + Separately, test-redis-compat.yml selects two IAM cluster authentication tests in + test_caching.py by node ID. It does not run that file's other tests. + The gap was eight files and 118 tests when measured 2026-08-20; the five keyless files now + run in the caching-local shard, leaving live cases in these three. Measured 2026-08-21 with no provider credentials and no Redis: test_caching.py needs both (37 of 65 fail without them), test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live diff --git a/tests/test_litellm/test_assert_ci_coverage.py b/tests/test_litellm/test_assert_ci_coverage.py index d948c1a4155..aeb55bb8ab7 100644 --- a/tests/test_litellm/test_assert_ci_coverage.py +++ b/tests/test_litellm/test_assert_ci_coverage.py @@ -11,6 +11,9 @@ the question neither covers: whether the job that globs a file then deselects it import importlib.util import sys from pathlib import Path +from typing import Final + +import pytest _REPO_ROOT = Path(__file__).resolve().parents[2] _MODULE_PATH = _REPO_ROOT / ".github" / "scripts" / "assert_ci_coverage.py" @@ -299,8 +302,20 @@ def test_the_slice_check_credits_only_workflows_never_the_circleci_config(): ) -def test_a_file_no_workflow_names_is_still_reported_when_every_slice_drops_it(): - named = coverage._workflow_named_tokens() +@pytest.mark.parametrize("selector", ("test_selected.py", "test_selected.py::test_redis_auth")) +def test_a_workflow_does_not_credit_a_file_it_never_names( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, selector: str +) -> None: + workflows: Final = tmp_path / "workflows" + workflows.mkdir() + (workflows / "test.yml").write_text( + f"jobs:\n test:\n steps:\n - run: uv run pytest tests/local_testing/{selector}\n" + ) + monkeypatch.setattr(coverage, "WORKFLOW_DIR", workflows) + monkeypatch.setattr(coverage, "CIRCLECI_CONFIG", tmp_path / "circleci.yml") + + named: Final = coverage._workflow_named_tokens() + assert named == frozenset({"tests/local_testing/test_selected.py"}) assert not any( - coverage._token_covers(token, "tests/local_testing/test_caching.py") for token in named - ), "test_caching.py is allowlisted, not run; crediting it would hide a real gap" + coverage._token_covers(token, "tests/local_testing/test_unrun.py") for token in named + )