mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
a43f128a74
Introduce the e2e coverage denominator: 282 behavior cells across the six tracking modules (LLMs, MCPs, Management/UI, Reliability & Performance, Logging & Guardrails, Other), one validated YAML row each, plus a collector that diffs the registry against @pytest.mark.covers markers and reports coverage per module. The registry rows validate against a pydantic discriminated union so a row cannot carry a field from another module. The collector is static: a collect-only pass reads the markers, so it runs no test and needs no live proxy. Register the covers marker suite-wide so that pass works under --strict-markers. This is a draft for review. Tiers are proposed rather than signed off, and a few cells still need a support check or a prune.
26 lines
902 B
Python
26 lines
902 B
Python
"""Load and validate the registry: the denominator, built in one shot from the YAMLs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
from .schema import CELL_ADAPTER, Cell
|
|
|
|
REGISTRY_DIR = Path(__file__).resolve().parent
|
|
|
|
|
|
def load_registry(registry_dir: Path = REGISTRY_DIR) -> tuple[Cell, ...]:
|
|
"""Every cell across every `*.yaml`, validated. Raises on a schema violation or
|
|
a duplicate id, since either would corrupt the coverage denominator."""
|
|
cells = tuple(
|
|
CELL_ADAPTER.validate_python(row)
|
|
for path in sorted(registry_dir.glob("*.yaml"))
|
|
for row in (yaml.safe_load(path.read_text()) or ())
|
|
)
|
|
duplicates = sorted(cid for cid, n in Counter(c.id for c in cells).items() if n > 1)
|
|
if duplicates:
|
|
raise ValueError(f"duplicate cell ids in registry: {duplicates}")
|
|
return cells
|