litellm/tests/test_litellm/test_assert_ci_coverage.py

"""Tests for .github/scripts/assert_ci_coverage.py.

Three guards share one workflow parser. The census asks whether a test file is run at
all, so an ancestor path standing in for everything below it is a valid answer. The
shard guard asks whether a sharded tree, which has no catch-all bucket, names each
child outright, so that same ancestor path must NOT be an answer. The slice guard asks
the question neither covers: whether the job that globs a file then deselects it with
`-k`, which is how a file counts as covered while running nowhere.
"""

import importlib.util
import sys
from pathlib import Path
from typing import Final

import yaml

_REPO_ROOT = Path(__file__).resolve().parents[2]
_MODULE_PATH = _REPO_ROOT / ".github" / "scripts" / "assert_ci_coverage.py"
_spec = importlib.util.spec_from_file_location("assert_ci_coverage", _MODULE_PATH)
coverage = importlib.util.module_from_spec(_spec)
sys.modules[_spec.name] = coverage  # @dataclass(slots=True) rebuilds via sys.modules
_spec.loader.exec_module(coverage)


def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: Path) -> None:
    test_path: Final = "tests/integration/management/test_contract.py"
    test_file: Final = tmp_path / test_path
    test_file.parent.mkdir(parents=True)
    test_file.write_text("def test_contract(): pass\n")
    (tmp_path / "tests/integration/run.py").write_text(
        "from types import MappingProxyType\nfrom typing import Final\n"
        'GROUPS: Final = MappingProxyType({"management": ("management",)})\n'
    )
    paths, findings = coverage._integration_ownership(tmp_path)
    assert not paths
    assert [finding.detail for finding in findings] == ["dedicated CircleCI runner is missing"]
    circle: Final = tmp_path / ".circleci/config.yml"
    circle.parent.mkdir()
    circle.write_text(
        yaml.safe_dump(
            {
                "jobs": {
                    "integration_contracts": {
                        "steps": [{"run": {"command": "bash .circleci/scripts/run_integration.sh management"}}]
                    }
                },
                "workflows": {"integration": {"jobs": [{"integration_contracts": {"suite": "management"}}]}},
            }
        )
    )
    paths, findings = coverage._integration_ownership(tmp_path)
    assert paths == frozenset({test_path})
    assert findings == ()
    configured: Final = yaml.safe_load(circle.read_text())
    configured["workflows"]["integration"]["jobs"] = [
        {"integration_contracts": {"matrix": {"parameters": {"suite": ["providers"]}}}}
    ]
    circle.write_text(yaml.safe_dump(configured))
    _, findings = coverage._integration_ownership(tmp_path)
    assert [(finding.subject, finding.detail) for finding in findings] == [
        ("management", "canonical integration group is not scheduled by CircleCI")
    ]
    configured["workflows"]["integration"]["jobs"][0]["integration_contracts"]["matrix"]["parameters"]["suite"] = [
        "management"
    ]
    circle.write_text(yaml.safe_dump(configured))
    workflow: Final = tmp_path / ".github/workflows/test.yml"
    workflow.parent.mkdir(parents=True)
    workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": "pytest tests/integration"}]}}}))
    _, findings = coverage._integration_ownership(tmp_path)
    assert [(finding.subject, finding.detail) for finding in findings] == [
        (test_path, "integration contract is also selected by GitHub Actions")
    ]


def test_an_ancestor_directory_covers_a_file_but_does_not_name_it():
    # The whole point of the split: `tests/x` answers "does it run?" but not
    # "which shard owns it?" — accepting it for the latter is how a new child
    # silently drops out of a tree that has no catch-all bucket.
    assert coverage._token_covers("tests/test_litellm", "tests/test_litellm/caching") is True
    assert coverage._token_names("tests/test_litellm", "tests/test_litellm/caching") is False


def test_an_exact_token_both_covers_and_names():
    assert coverage._token_covers("tests/test_litellm/caching", "tests/test_litellm/caching") is True
    assert coverage._token_names("tests/test_litellm/caching", "tests/test_litellm/caching") is True


def test_a_glob_names_only_what_it_matches_not_what_sits_below_it():
    glob = "tests/test_litellm/test_*.py"
    assert coverage._token_names(glob, "tests/test_litellm/test_router.py") is True
    assert coverage._token_names(glob, "tests/test_litellm/test_router.py/nested.py") is False
    assert coverage._token_names(glob, "tests/test_litellm/proxy/test_router.py") is False


def test_a_glob_still_covers_the_subtree_for_the_census():
    assert coverage._token_covers("tests/llm_translation/**", "tests/llm_translation/a/b.py") is True


def test_a_directory_earns_a_shard_by_holding_tests_not_by_its_name(tmp_path):
    fixtures = tmp_path / "expected_payloads"
    fixtures.mkdir()
    (fixtures / "body.json").write_text("{}")
    tests = tmp_path / "some_area"
    tests.mkdir()
    (tests / "test_thing.py").write_text("def test_thing(): assert True\n")

    assert coverage._holds_tests(fixtures) is False
    assert coverage._holds_tests(tests) is True


def test_shard_children_lists_test_dirs_and_test_files_and_skips_fixture_dirs(tmp_path):
    root = tmp_path / "tests" / "tree"
    (root / "billing").mkdir(parents=True)
    (root / "billing" / "test_billing.py").write_text("def test_b(): assert True\n")
    (root / "test_configs").mkdir()
    (root / "test_configs" / "config.yaml").write_text("model_list: []\n")
    (root / "test_top_level.py").write_text("def test_t(): assert True\n")
    (root / "helpers.py").write_text("VALUE = 1\n")

    assert coverage._shard_children("tests/tree", tmp_path) == (
        "tests/tree/billing",
        "tests/tree/test_top_level.py",
    )


def test_an_unnamed_child_is_reported_and_a_named_one_is_not(tmp_path):
    root = tmp_path / "tests" / "tree"
    (root / "claimed").mkdir(parents=True)
    (root / "claimed" / "test_a.py").write_text("def test_a(): assert True\n")
    (root / "orphan").mkdir()
    (root / "orphan" / "test_b.py").write_text("def test_b(): assert True\n")

    findings = coverage._unassigned_shard_children(
        frozenset({"tests/tree/claimed"}), roots=("tests/tree",), repo_root=tmp_path
    )

    assert tuple(f.subject for f in findings) == ("tests/tree/orphan",)


def test_the_parent_token_alone_does_not_satisfy_any_child(tmp_path):
    root = tmp_path / "tests" / "tree"
    (root / "billing").mkdir(parents=True)
    (root / "billing" / "test_a.py").write_text("def test_a(): assert True\n")

    findings = coverage._unassigned_shard_children(frozenset({"tests/tree"}), roots=("tests/tree",), repo_root=tmp_path)

    assert tuple(f.subject for f in findings) == ("tests/tree/billing",)


def test_a_child_that_is_itself_a_sharded_root_is_checked_there_not_here(tmp_path):
    root = tmp_path / "tests" / "tree"
    (root / "proxy" / "endpoints").mkdir(parents=True)
    (root / "proxy" / "endpoints" / "test_a.py").write_text("def test_a(): assert True\n")

    findings = coverage._unassigned_shard_children(
        frozenset(), roots=("tests/tree", "tests/tree/proxy"), repo_root=tmp_path
    )

    assert tuple(f.subject for f in findings) == ("tests/tree/proxy/endpoints",)


def test_every_sharded_root_named_in_the_script_exists_on_disk():
    # A stale root would make the guard pass by checking nothing.
    missing = [root for root in coverage.SHARDED_ROOTS if not (_REPO_ROOT / root).is_dir()]
    assert missing == []


def test_the_repo_as_it_stands_has_every_shard_child_assigned():
    findings = coverage._unassigned_shard_children(coverage._invoked_test_tokens(coverage._all_scalars()))
    assert [f.subject for f in findings] == []


# --------------------------------------------------------------------------- #
# Slice guard: a job can glob a file and its -k can then throw the file out
# --------------------------------------------------------------------------- #


def _slice(**overrides):
    defaults = dict(
        job="a_job",
        globs=("tests/x/**/test_*.py",),
        named=frozenset(),
        required=(),
        excluded=(),
        understood=True,
    )
    return coverage.Slice(**{**defaults, **overrides})


def test_a_term_in_the_path_deselects_the_whole_file():
    # -k matches the module's path as well as the names inside it, so "not caching"
    # removes every test in test_caching.py, not merely the ones named for a cache.
    slice_ = _slice(excluded=("caching",))
    assert slice_.claims("tests/x/test_caching.py", frozenset({"test_get"})) is False
    assert slice_.claims("tests/x/test_router.py", frozenset({"test_get"})) is True


def test_matching_is_substring_not_word_so_cache_and_caching_are_different_terms():
    # The real config excludes both, because "cache" does not occur inside "caching";
    # collapsing them to one term would quietly let a whole file back in.
    assert _slice(excluded=("cache",)).claims("tests/x/test_caching.py", frozenset()) is True
    assert _slice(excluded=("cache",)).claims("tests/x/test_dual_cache.py", frozenset()) is False


def test_a_positive_term_can_be_satisfied_by_a_name_inside_the_file():
    # A job running -k "langfuse" claims test_logging.py when a test inside is named
    # for langfuse, so treating the path alone as the match would report a false gap.
    slice_ = _slice(required=("langfuse",))
    assert slice_.claims("tests/x/test_logging.py", frozenset({"test_langfuse_emits"})) is True
    assert slice_.claims("tests/x/test_logging.py", frozenset({"test_datadog_emits"})) is False


def test_a_file_the_job_never_globs_is_not_its_problem():
    assert _slice().claims("tests/other/test_a.py", frozenset()) is False


def test_an_explicitly_named_file_is_claimed_whatever_the_keywords_say():
    # redis_caching_unit_tests names test_dual_cache.py outright, which is what keeps
    # that file out of the report even though every -k in the globbing jobs drops it.
    slice_ = _slice(globs=(), named=frozenset({"tests/x/test_dual_cache.py"}), excluded=("cache",))
    assert slice_.claims("tests/x/test_dual_cache.py", frozenset()) is True


def test_an_unparsed_keyword_expression_claims_everything_it_globs():
    # Staying silent beats guessing: an expression this parser cannot model must never
    # be the reason a file is reported as unrun.
    assert _slice(understood=False, excluded=("cache",)).claims("tests/x/test_caching.py", frozenset()) is True


def test_keyword_terms_splits_an_and_chain_into_required_and_excluded():
    required, excluded, understood = coverage._keyword_terms(("langfuse and not cache and not router",))
    assert (required, excluded, understood) == (("langfuse",), ("cache", "router"), True)


def test_keyword_terms_refuses_to_model_an_or_expression():
    assert coverage._keyword_terms(("cache or router",)) == ((), (), False)


def test_keyword_terms_refuses_to_attribute_a_selector_across_several_commands():
    # A job running two pytest commands offers no way to tell which glob a -k belongs
    # to, and pairing one command's exclusion with the other's glob would invent a gap.
    assert coverage._keyword_terms(("not cache",), attributable=False) == ((), (), False)
    assert coverage._keyword_terms((), attributable=False) == ((), (), True)


def test_an_excluded_term_matching_only_an_inner_name_leaves_the_file_claimed():
    # -k "not cache" drops test_cache_key inside test_router.py and keeps the rest, so
    # the file still runs. Reporting it would be a false alarm; the guard is per-file.
    slice_ = _slice(excluded=("cache",))
    assert slice_.claims("tests/x/test_router.py", frozenset({"test_cache_key"})) is True


def test_character_class_globs_match_the_letter_shards_circleci_uses():
    # tests/local_testing is split by first letter; without character-class support every
    # file in it looks unglobbed, and the slice guard would report the whole directory.
    glob = "tests/local_testing/**/test_[a-mA-M]*.py"
    assert coverage._token_covers(glob, "tests/local_testing/test_caching.py") is True
    assert coverage._token_covers(glob, "tests/local_testing/test_router.py") is False


def test_the_repo_as_it_stands_has_no_unrecorded_slice_gap():
    findings = coverage._deselected_everywhere(coverage._load_allowlist())
    assert [f.subject for f in findings] == []


def _allowlist(*, tests: tuple[str, ...] = (), dockerfiles: tuple[str, ...] = ()):
    return coverage.Allowlist(
        test_paths=(coverage.AllowEntry(paths=tests, reason="r"),) if tests else (),
        dockerfiles=(coverage.AllowEntry(paths=dockerfiles, reason="r"),) if dockerfiles else (),
    )


def test_an_allowlist_path_whose_file_is_gone_is_reported():
    findings = coverage._stale_allowlist_paths(
        _allowlist(tests=("tests/gone/test_a.py",)),
        test_files=("tests/live/test_b.py",),
        dockerfiles=(),
    )
    assert [f.subject for f in findings] == ["tests/gone/test_a.py"]
    assert "test_paths" in findings[0].detail


def test_an_allowlist_path_that_still_matches_a_file_is_left_alone():
    findings = coverage._stale_allowlist_paths(
        _allowlist(tests=("tests/live/test_b.py",)),
        test_files=("tests/live/test_b.py",),
        dockerfiles=(),
    )
    assert findings == ()


def test_a_directory_entry_survives_while_any_file_below_it_remains():
    findings = coverage._stale_allowlist_paths(
        _allowlist(tests=("tests/live",)),
        test_files=("tests/live/nested/test_b.py",),
        dockerfiles=(),
    )
    assert findings == ()


def test_a_glob_entry_matching_nothing_is_reported_like_any_other():
    findings = coverage._stale_allowlist_paths(
        _allowlist(tests=("tests/live/test_z*.py",)),
        test_files=("tests/live/test_b.py",),
        dockerfiles=(),
    )
    assert [f.subject for f in findings] == ["tests/live/test_z*.py"]


def test_a_stale_dockerfile_entry_is_named_under_its_own_section():
    findings = coverage._stale_allowlist_paths(
        _allowlist(dockerfiles=("docker/Dockerfile.gone",)),
        test_files=(),
        dockerfiles=("docker/Dockerfile.database",),
    )
    assert [(f.subject, "dockerfiles" in f.detail) for f in findings] == [("docker/Dockerfile.gone", True)]


def test_the_repo_as_it_stands_has_no_stale_allowlist_entry():
    findings = coverage._stale_allowlist_paths(
        coverage._load_allowlist(),
        test_files=coverage._test_files(),
        dockerfiles=coverage._dockerfiles(),
    )
    assert [f.subject for f in findings] == []


def test_a_dockerfile_directory_entry_is_stale_because_only_an_exact_path_exempts_one():
    findings = coverage._stale_allowlist_paths(
        _allowlist(dockerfiles=("docker",)),
        test_files=(),
        dockerfiles=("docker/Dockerfile.database",),
    )
    assert [f.subject for f in findings] == ["docker"]


def test_a_workflow_that_names_a_file_clears_it_from_the_slice_check():
    named = coverage._workflow_named_tokens()
    assert named, "the workflows must name some test paths or the check proves nothing"
    assert any(coverage._token_covers(token, "tests/local_testing/test_caching_handler.py") for token in named)


def test_the_slice_check_credits_only_workflows_never_the_circleci_config():
    named = coverage._workflow_named_tokens()
    circleci_only = "tests/proxy_admin_ui_tests"
    assert not any(coverage._token_covers(token, f"{circleci_only}/test_key_management.py") for token in named), (
        "a tree only CircleCI globs must not be credited to a workflow"
    )


def test_a_file_no_workflow_names_is_still_reported_when_every_slice_drops_it():
    named = coverage._workflow_named_tokens()
    assert not any(coverage._token_covers(token, "tests/local_testing/test_caching.py") for token in named), (
        "test_caching.py is allowlisted, not run; crediting it would hide a real gap"
    )