mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-28 01:32:17 +00:00
test(ci): isolate workflow coverage audit fixtures
Replace the stale unrun caching-file assumption with isolated workflow fixtures for file and node-ID selectors. Keep the unnamed-file negative check and clarify which live caching cases remain outside CI. Signed-off-by: Silu Panda <31051721+SiluPanda@users.noreply.github.com>
This commit is contained in:
parent
c8ff8c1237
commit
f2dfadcfbc
2 changed files with 26 additions and 10 deletions
13
.github/ci-coverage-allowlist.yml
vendored
13
.github/ci-coverage-allowlist.yml
vendored
|
|
@ -10,12 +10,13 @@ test_paths:
|
|||
paths:
|
||||
- tests/rust-python-harness
|
||||
- reason: >-
|
||||
What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
|
||||
globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
|
||||
not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
|
||||
and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
|
||||
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
|
||||
run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
|
||||
Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that
|
||||
glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and
|
||||
not caching and not cache"`) or keep only another keyword (langfuse, router, assistants).
|
||||
Separately, test-redis-compat.yml selects two IAM cluster authentication tests in
|
||||
test_caching.py by node ID. It does not run that file's other tests.
|
||||
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless files now
|
||||
run in the caching-local shard, leaving live cases in these three. Measured 2026-08-21 with no provider
|
||||
credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
|
||||
test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
|
||||
test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ the question neither covers: whether the job that globs a file then deselects it
|
|||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
_MODULE_PATH = _REPO_ROOT / ".github" / "scripts" / "assert_ci_coverage.py"
|
||||
|
|
@ -299,8 +302,20 @@ def test_the_slice_check_credits_only_workflows_never_the_circleci_config():
|
|||
)
|
||||
|
||||
|
||||
def test_a_file_no_workflow_names_is_still_reported_when_every_slice_drops_it():
|
||||
named = coverage._workflow_named_tokens()
|
||||
@pytest.mark.parametrize("selector", ("test_selected.py", "test_selected.py::test_redis_auth"))
|
||||
def test_a_workflow_does_not_credit_a_file_it_never_names(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, selector: str
|
||||
) -> None:
|
||||
workflows: Final = tmp_path / "workflows"
|
||||
workflows.mkdir()
|
||||
(workflows / "test.yml").write_text(
|
||||
f"jobs:\n test:\n steps:\n - run: uv run pytest tests/local_testing/{selector}\n"
|
||||
)
|
||||
monkeypatch.setattr(coverage, "WORKFLOW_DIR", workflows)
|
||||
monkeypatch.setattr(coverage, "CIRCLECI_CONFIG", tmp_path / "circleci.yml")
|
||||
|
||||
named: Final = coverage._workflow_named_tokens()
|
||||
assert named == frozenset({"tests/local_testing/test_selected.py"})
|
||||
assert not any(
|
||||
coverage._token_covers(token, "tests/local_testing/test_caching.py") for token in named
|
||||
), "test_caching.py is allowlisted, not run; crediting it would hide a real gap"
|
||||
coverage._token_covers(token, "tests/local_testing/test_unrun.py") for token in named
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue