litellm/tests/e2e/coverage_registry/test_collector.py

"""Tests for the coverage-registry tooling: pure logic plus a registry canary.

No `e2e` marker, so these run without a proxy. They exercise the coverage math and
the registry loader, and guard the checked-in registry against schema drift and
duplicate ids.
"""

from __future__ import annotations

from pathlib import Path

import pytest

from coverage_registry.collector import (
    compute_coverage,
    render,
    render_json,
    render_loki,
    render_prometheus,
)
from coverage_registry.registry import load_registry
from coverage_registry.schema import (
    GuardrailCell,
    LlmCell,
    LlmEndpoint,
    LoggingCell,
    Tier,
    loki_module_label,
)


def _llm(
    cell_id: str, tier: Tier, subject_endpoint: LlmEndpoint = "chat_completions"
) -> LlmCell:
    return LlmCell(
        id=cell_id,
        module="llm",
        tier=tier,
        assertions=("works",),
        source="test",
        subject_endpoint=subject_endpoint,
        route="openai",
        capability="basic",
        streaming="nonstream",
    )


def test_compute_coverage_counts_covered_p0_and_gaps() -> None:
    cells = (_llm("llm.a", Tier.P0), _llm("llm.b", Tier.P0), _llm("llm.c", Tier.P1))
    report = compute_coverage(cells, frozenset({"llm.a"}))
    assert (report.total, report.covered) == (3, 1)
    assert (report.p0_total, report.p0_covered) == (2, 1)
    assert report.p0_gaps == ("llm.b",)
    assert report.orphan_markers == ()


def test_orphan_marker_is_reported_not_counted() -> None:
    cells = (_llm("llm.a", Tier.P0),)
    report = compute_coverage(cells, frozenset({"llm.a", "llm.ghost"}))
    assert report.covered == 1
    assert report.orphan_markers == ("llm.ghost",)


def test_logging_and_guardrail_roll_up_into_one_module() -> None:
    cells = (
        LoggingCell(
            id="logging.x",
            module="logging",
            tier=Tier.P0,
            assertions=("logs_spend",),
            source="t",
            event="success",
            exercised_on=("chat_completions",),
        ),
        GuardrailCell(
            id="guardrail.y",
            module="guardrail",
            tier=Tier.P1,
            assertions=("blocks",),
            source="t",
            hook_point="pre_call",
            exercised_on=("chat_completions",),
        ),
    )
    report = compute_coverage(cells, frozenset())
    logging_and_guardrails = next(
        m for m in report.modules if m.module == "Logging & Guardrails"
    )
    assert logging_and_guardrails.total == 2


def test_llm_cells_roll_up_by_core_endpoint() -> None:
    cells = (
        _llm("llm.chat", Tier.P0, "chat_completions"),
        _llm("llm.messages", Tier.P0, "messages"),
        _llm("llm.responses", Tier.P1, "responses"),
        _llm("llm.batches", Tier.P0, "batches"),
        _llm("llm.realtime", Tier.P1, "realtime"),
    )
    report = compute_coverage(cells, frozenset({"llm.chat", "llm.batches"}))

    core = next(m for m in report.modules if m.module == "Core LLMs")
    non_core = next(m for m in report.modules if m.module == "Non-Core LLMs")

    assert (core.total, core.covered, core.p0_total, core.p0_covered) == (3, 1, 2, 1)
    assert (
        non_core.total,
        non_core.covered,
        non_core.p0_total,
        non_core.p0_covered,
    ) == (2, 1, 1, 1)


def test_text_render_uses_plain_coverage_language() -> None:
    report = compute_coverage(
        (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")),
        frozenset({"llm.chat"}),
    )

    text = render(report)

    assert "COVERAGE" in text
    assert "Headline coverage: 1/2  (50.0%)" in text
    assert "P0 COVERED" not in text


def test_json_render_exposes_module_coverage_for_grafana_jobs() -> None:
    report = compute_coverage(
        (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")),
        frozenset({"llm.chat"}),
    )

    payload = render_json(report)

    assert '"coverage_percent": 50.0' in payload
    assert '"module": "Core LLMs"' in payload
    assert '"module": "Non-Core LLMs"' in payload


def test_prometheus_render_exposes_module_coverage_timeseries() -> None:
    report = compute_coverage(
        (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")),
        frozenset({"llm.chat"}),
    )

    metrics = render_prometheus(report)

    assert 'litellm_e2e_coverage_cells{module="Core LLMs",state="covered"} 1' in metrics
    assert 'litellm_e2e_coverage_percent{module="Core LLMs"} 100.000000' in metrics
    assert 'litellm_e2e_coverage_percent{module="Non-Core LLMs"} 0.000000' in metrics
    assert "litellm_e2e_coverage_orphan_markers 0" in metrics


def test_loki_render_exposes_exact_stdout_lines_for_loki() -> None:
    report = compute_coverage(
        (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")),
        frozenset({"llm.chat"}),
    )

    lines = render_loki(report).splitlines()

    assert len(lines) == 1 + len(report.modules)
    assert lines[0] == "COVERAGE_TOTAL percent=50.0 covered=1 total=2"
    assert (
        lines[1] == "COVERAGE_MODULE module=core_llms percent=100.0 covered=1 total=1"
    )
    assert (
        lines[2] == "COVERAGE_MODULE module=non_core_llms percent=0.0 covered=0 total=1"
    )
    assert [line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:]] == [
        loki_module_label(module.module) for module in report.modules
    ]
    assert all(
        " " not in line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:]
    )


def test_real_registry_loads_and_ids_are_unique() -> None:
    cells = load_registry()
    ids = [c.id for c in cells]
    assert len(cells) > 250
    assert len(ids) == len(set(ids))
    assert any(c.id == "logging.prometheus.success.exports_metric" for c in cells)


def test_load_registry_rejects_duplicate_ids(tmp_path: Path) -> None:
    row = (
        "- {id: llm.dup, module: llm, tier: P0, assertions: [works], source: t, "
        "subject_endpoint: chat_completions, route: openai, capability: basic, streaming: nonstream}\n"
    )
    (tmp_path / "a.yaml").write_text(row)
    (tmp_path / "b.yaml").write_text(row)
    with pytest.raises(ValueError, match="duplicate cell ids"):
        load_registry(tmp_path)