From 682d22090e4b5a976b32d07124f1d91cf2fdbb7e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 7 Jul 2026 17:37:58 -0700 Subject: [PATCH] Add e2e coverage dashboard metrics --- .../coverage_registry/GRAFANA_DASHBOARD.md | 92 ++++++++++++++ tests/e2e/coverage_registry/README.md | 12 +- tests/e2e/coverage_registry/collector.py | 116 ++++++++++++++++-- tests/e2e/coverage_registry/test_collector.py | 47 ++++++- 4 files changed, 249 insertions(+), 18 deletions(-) create mode 100644 tests/e2e/coverage_registry/GRAFANA_DASHBOARD.md diff --git a/tests/e2e/coverage_registry/GRAFANA_DASHBOARD.md b/tests/e2e/coverage_registry/GRAFANA_DASHBOARD.md new file mode 100644 index 00000000000..13e1cf2d5ee --- /dev/null +++ b/tests/e2e/coverage_registry/GRAFANA_DASHBOARD.md @@ -0,0 +1,92 @@ +# Grafana Dashboard Brief + +This dashboard should answer two questions: + +1. How much e2e coverage do we have by module right now? +2. Is e2e coverage by module improving or regressing over time? + +The dashboard should use plain **Coverage** language. Avoid exposing P0/P1/P2 in the +default view; tiers can be added later as a filter or drilldown. + +## Panels + +### Coverage by Module + +Use a bar chart or table with one row per module: + +- Core LLMs +- Non-Core LLMs +- MCPs +- Management/UI +- Reliability & Performance +- Logging & Guardrails +- Other + +Each row should show: + +- `covered` +- `total` +- `coverage_percent` + +Formula: + +```text +coverage_percent = covered / total * 100 +``` + +### Coverage Trend by Module + +Use a time-series chart with one line per module. + +- X-axis: CI run timestamp, scrape timestamp, or pushed metric timestamp +- Y-axis: `coverage_percent` +- Series label: module name + +This shows whether coverage is improving across dates. + +## Data Contract + +Generate coverage data from the registry collector: + +```bash +cd tests/e2e +PYTHONPATH=. python -m coverage_registry.collector --format prometheus --strict +``` + +For artifact-based jobs, JSON is also available: + +```bash +cd tests/e2e +PYTHONPATH=. python -m coverage_registry.collector --format json --strict +``` + +Prometheus metrics: + +```text +litellm_e2e_coverage_cells{module="",state="covered"} +litellm_e2e_coverage_cells{module="",state="total"} +litellm_e2e_coverage_percent{module=""} +litellm_e2e_coverage_orphan_markers +litellm_e2e_coverage_collection_errors +``` + +Grafana gets the trend by storing these metrics over time. No date needs to be encoded +inside the metric itself. + +## Alerts + +Start with two alerts: + +- Unknown marker count is greater than zero. +- Coverage percent for any module drops compared with the previous successful run. + +After existing collection warnings are fixed, add: + +- Collection error count is greater than zero. + +## Non-Goals + +- Do not use line coverage for this dashboard. This is behavior coverage, not source-line + coverage. +- Do not let tests invent module names. Tests only declare `@pytest.mark.covers(...)`; + the registry decides which module a cell belongs to. diff --git a/tests/e2e/coverage_registry/README.md b/tests/e2e/coverage_registry/README.md index 78b006c7208..ef94b5639c2 100644 --- a/tests/e2e/coverage_registry/README.md +++ b/tests/e2e/coverage_registry/README.md @@ -3,7 +3,8 @@ This directory is the **denominator** for e2e test coverage: the set of behaviors we want covered, one row per behavior, checked into the repo so coverage is a number we can track instead of a guess. It implements the plan in the "E2E Coverage Tracking" -note; the naming grammar lives in `tests/e2e/CLAUDE.md`. +note; the naming grammar lives in `tests/e2e/CLAUDE.md`. The Grafana dashboard contract +lives in `GRAFANA_DASHBOARD.md`. ## The model @@ -40,9 +41,12 @@ proxy. Whether a covered cell currently passes or fails is a separate, live conc cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector ``` -The headline is P0 coverage. The collector also lists markers that point at ids not in -the registry, so a typo or an unenumerated behavior surfaces instead of being silently -dropped. +Use `--format prometheus` or `--format json` for CI jobs that publish coverage to +Grafana. + +The headline is overall coverage. The collector also lists markers that point at ids +not in the registry, so a typo or an unenumerated behavior surfaces instead of being +silently dropped. Use strict mode in CI once existing draft markers are reconciled: diff --git a/tests/e2e/coverage_registry/collector.py b/tests/e2e/coverage_registry/collector.py index 6350ab6251b..3b577106605 100644 --- a/tests/e2e/coverage_registry/collector.py +++ b/tests/e2e/coverage_registry/collector.py @@ -12,6 +12,7 @@ from __future__ import annotations import contextlib import io +import json import sys from argparse import ArgumentParser from dataclasses import dataclass @@ -75,6 +76,10 @@ class ModuleCoverage: p0_total: int p0_covered: int + @property + def coverage_percent(self) -> float: + return _percent(self.covered, self.total) + @dataclass(frozen=True, slots=True) class CoverageReport: @@ -87,6 +92,14 @@ class CoverageReport: orphan_markers: tuple[str, ...] collection_errors: tuple[str, ...] + @property + def coverage_percent(self) -> float: + return _percent(self.covered, self.total) + + +def _percent(covered: int, total: int) -> float: + return (100.0 * covered / total) if total else 0.0 + def _module_coverage( module: str, cells: tuple[Cell, ...], covered: frozenset[str] @@ -121,25 +134,20 @@ def compute_coverage( ) -def _row(label: str, covered: int, total: int, p0_covered: int, p0_total: int) -> str: +def _row(label: str, covered: int, total: int) -> str: frac = f"{covered}/{total}" - p0 = f"{p0_covered}/{p0_total}" - return f"{label:30}{frac:>12}{p0:>14}" + return f"{label:30}{frac:>12}{_percent(covered, total):>11.1f}%" def render(report: CoverageReport) -> str: - rows = tuple( - _row(m.module, m.covered, m.total, m.p0_covered, m.p0_total) - for m in report.modules - ) - pct = (100.0 * report.p0_covered / report.p0_total) if report.p0_total else 0.0 + rows = tuple(_row(m.module, m.covered, m.total) for m in report.modules) lines = ( - f"{'MODULE':30}{'COVERED':>12}{'P0 COVERED':>14}", + f"{'MODULE':30}{'COVERED':>12}{'COVERAGE':>12}", *rows, - "-" * 56, - _row("ALL", report.covered, report.total, report.p0_covered, report.p0_total), + "-" * 54, + _row("ALL", report.covered, report.total), "", - f"Headline (P0 coverage): {report.p0_covered}/{report.p0_total} ({pct:.1f}%)", + f"Headline coverage: {report.covered}/{report.total} ({report.coverage_percent:.1f}%)", ) orphans = ( ( @@ -162,8 +170,83 @@ def render(report: CoverageReport) -> str: return "\n".join((*lines, *orphans, *warning)) +def _report_dict(report: CoverageReport) -> dict[str, object]: + return { + "covered": report.covered, + "total": report.total, + "coverage_percent": report.coverage_percent, + "modules": [ + { + "module": m.module, + "covered": m.covered, + "total": m.total, + "coverage_percent": m.coverage_percent, + "p0_covered": m.p0_covered, + "p0_total": m.p0_total, + } + for m in report.modules + ], + "orphan_markers": list(report.orphan_markers), + "collection_errors": list(report.collection_errors), + } + + +def render_json(report: CoverageReport) -> str: + return json.dumps(_report_dict(report), indent=2, sort_keys=True) + + +def _label_value(value: str) -> str: + return value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + + +def render_prometheus(report: CoverageReport) -> str: + lines = [ + "# HELP litellm_e2e_coverage_cells E2E coverage registry cells by module and state.", + "# TYPE litellm_e2e_coverage_cells gauge", + ] + for module in report.modules: + label = _label_value(module.module) + lines.append( + f'litellm_e2e_coverage_cells{{module="{label}",state="covered"}} {module.covered}' + ) + lines.append( + f'litellm_e2e_coverage_cells{{module="{label}",state="total"}} {module.total}' + ) + lines.extend( + [ + f'litellm_e2e_coverage_cells{{module="ALL",state="covered"}} {report.covered}', + f'litellm_e2e_coverage_cells{{module="ALL",state="total"}} {report.total}', + "# HELP litellm_e2e_coverage_percent E2E coverage percent by module.", + "# TYPE litellm_e2e_coverage_percent gauge", + ] + ) + for module in report.modules: + label = _label_value(module.module) + lines.append( + f'litellm_e2e_coverage_percent{{module="{label}"}} {module.coverage_percent:.6f}' + ) + lines.extend( + [ + f'litellm_e2e_coverage_percent{{module="ALL"}} {report.coverage_percent:.6f}', + "# HELP litellm_e2e_coverage_orphan_markers Coverage markers not found in the registry.", + "# TYPE litellm_e2e_coverage_orphan_markers gauge", + f"litellm_e2e_coverage_orphan_markers {len(report.orphan_markers)}", + "# HELP litellm_e2e_coverage_collection_errors Pytest nodes that failed during collection.", + "# TYPE litellm_e2e_coverage_collection_errors gauge", + f"litellm_e2e_coverage_collection_errors {len(report.collection_errors)}", + ] + ) + return "\n".join(lines) + + def main() -> int: parser = ArgumentParser() + parser.add_argument( + "--format", + choices=("text", "json", "prometheus"), + default="text", + help="Output format. Use prometheus or json for Grafana ingestion jobs.", + ) parser.add_argument( "--strict", action="store_true", @@ -178,7 +261,14 @@ def main() -> int: cells = load_registry() covered, errors = collect_covered_ids() report = compute_coverage(cells, covered, errors) - print(render(report)) # noqa: T201 # CLI entrypoint output + output = { + "text": render, + "json": render_json, + "prometheus": render_prometheus, + }[ + args.format + ](report) + print(output) # noqa: T201 # CLI entrypoint output if args.strict and report.orphan_markers: return 1 if args.fail_on_collection_errors and report.collection_errors: diff --git a/tests/e2e/coverage_registry/test_collector.py b/tests/e2e/coverage_registry/test_collector.py index 5178e217f31..355bc52730d 100644 --- a/tests/e2e/coverage_registry/test_collector.py +++ b/tests/e2e/coverage_registry/test_collector.py @@ -11,7 +11,12 @@ from pathlib import Path import pytest -from coverage_registry.collector import compute_coverage +from coverage_registry.collector import ( + compute_coverage, + render, + render_json, + render_prometheus, +) from coverage_registry.registry import load_registry from coverage_registry.schema import ( GuardrailCell, @@ -104,6 +109,46 @@ def test_llm_cells_roll_up_by_core_endpoint() -> None: ) == (2, 1, 1, 1) +def test_text_render_uses_plain_coverage_language() -> None: + report = compute_coverage( + (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), + frozenset({"llm.chat"}), + ) + + text = render(report) + + assert "COVERAGE" in text + assert "Headline coverage: 1/2 (50.0%)" in text + assert "P0 COVERED" not in text + + +def test_json_render_exposes_module_coverage_for_grafana_jobs() -> None: + report = compute_coverage( + (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), + frozenset({"llm.chat"}), + ) + + payload = render_json(report) + + assert '"coverage_percent": 50.0' in payload + assert '"module": "Core LLMs"' in payload + assert '"module": "Non-Core LLMs"' in payload + + +def test_prometheus_render_exposes_module_coverage_timeseries() -> None: + report = compute_coverage( + (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), + frozenset({"llm.chat"}), + ) + + metrics = render_prometheus(report) + + assert 'litellm_e2e_coverage_cells{module="Core LLMs",state="covered"} 1' in metrics + assert 'litellm_e2e_coverage_percent{module="Core LLMs"} 100.000000' in metrics + assert 'litellm_e2e_coverage_percent{module="Non-Core LLMs"} 0.000000' in metrics + assert "litellm_e2e_coverage_orphan_markers 0" in metrics + + def test_real_registry_loads_and_ids_are_unique() -> None: cells = load_registry() ids = [c.id for c in cells]