Add e2e coverage dashboard metrics

tests/e2e/coverage_registry/GRAFANA_DASHBOARD.md

@@ -0,0 +1,92 @@
+# Grafana Dashboard Brief
+
+This dashboard should answer two questions:
+
+1. How much e2e coverage do we have by module right now?
+2. Is e2e coverage by module improving or regressing over time?
+
+The dashboard should use plain **Coverage** language. Avoid exposing P0/P1/P2 in the
+default view; tiers can be added later as a filter or drilldown.
+
+## Panels
+
+### Coverage by Module
+
+Use a bar chart or table with one row per module:
+
+- Core LLMs
+- Non-Core LLMs
+- MCPs
+- Management/UI
+- Reliability & Performance
+- Logging & Guardrails
+- Other
+
+Each row should show:
+
+- `covered`
+- `total`
+- `coverage_percent`
+
+Formula:
+
+```text
+coverage_percent = covered / total * 100
+```
+
+### Coverage Trend by Module
+
+Use a time-series chart with one line per module.
+
+- X-axis: CI run timestamp, scrape timestamp, or pushed metric timestamp
+- Y-axis: `coverage_percent`
+- Series label: module name
+
+This shows whether coverage is improving across dates.
+
+## Data Contract
+
+Generate coverage data from the registry collector:
+
+```bash
+cd tests/e2e
+PYTHONPATH=. python -m coverage_registry.collector --format prometheus --strict
+```
+
+For artifact-based jobs, JSON is also available:
+
+```bash
+cd tests/e2e
+PYTHONPATH=. python -m coverage_registry.collector --format json --strict
+```
+
+Prometheus metrics:
+
+```text
+litellm_e2e_coverage_cells{module="<module>",state="covered"} <count>
+litellm_e2e_coverage_cells{module="<module>",state="total"} <count>
+litellm_e2e_coverage_percent{module="<module>"} <percent>
+litellm_e2e_coverage_orphan_markers <count>
+litellm_e2e_coverage_collection_errors <count>
+```
+
+Grafana gets the trend by storing these metrics over time. No date needs to be encoded
+inside the metric itself.
+
+## Alerts
+
+Start with two alerts:
+
+- Unknown marker count is greater than zero.
+- Coverage percent for any module drops compared with the previous successful run.
+
+After existing collection warnings are fixed, add:
+
+- Collection error count is greater than zero.
+
+## Non-Goals
+
+- Do not use line coverage for this dashboard. This is behavior coverage, not source-line
+  coverage.
+- Do not let tests invent module names. Tests only declare `@pytest.mark.covers(...)`;
+  the registry decides which module a cell belongs to.

tests/e2e/coverage_registry/README.md

@@ -3,7 +3,8 @@
 This directory is the **denominator** for e2e test coverage: the set of behaviors we
 want covered, one row per behavior, checked into the repo so coverage is a number we
 can track instead of a guess. It implements the plan in the "E2E Coverage Tracking"
-note; the naming grammar lives in `tests/e2e/CLAUDE.md`.
+note; the naming grammar lives in `tests/e2e/CLAUDE.md`. The Grafana dashboard contract
+lives in `GRAFANA_DASHBOARD.md`.
 
 ## The model
 
@@ -40,9 +41,12 @@ proxy. Whether a covered cell currently passes or fails is a separate, live conc
 cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector
 ```
 
-The headline is P0 coverage. The collector also lists markers that point at ids not in
-the registry, so a typo or an unenumerated behavior surfaces instead of being silently
-dropped.
+Use `--format prometheus` or `--format json` for CI jobs that publish coverage to
+Grafana.
+
+The headline is overall coverage. The collector also lists markers that point at ids
+not in the registry, so a typo or an unenumerated behavior surfaces instead of being
+silently dropped.
 
 Use strict mode in CI once existing draft markers are reconciled:
 

tests/e2e/coverage_registry/collector.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import contextlib
 import io
+import json
 import sys
 from argparse import ArgumentParser
 from dataclasses import dataclass
@@ -75,6 +76,10 @@ class ModuleCoverage:
     p0_total: int
     p0_covered: int
 
+    @property
+    def coverage_percent(self) -> float:
+        return _percent(self.covered, self.total)
+
 
 @dataclass(frozen=True, slots=True)
 class CoverageReport:
@@ -87,6 +92,14 @@ class CoverageReport:
     orphan_markers: tuple[str, ...]
     collection_errors: tuple[str, ...]
 
+    @property
+    def coverage_percent(self) -> float:
+        return _percent(self.covered, self.total)
+
+
+def _percent(covered: int, total: int) -> float:
+    return (100.0 * covered / total) if total else 0.0
+
 
 def _module_coverage(
     module: str, cells: tuple[Cell, ...], covered: frozenset[str]
@@ -121,25 +134,20 @@ def compute_coverage(
     )
 
 
-def _row(label: str, covered: int, total: int, p0_covered: int, p0_total: int) -> str:
+def _row(label: str, covered: int, total: int) -> str:
     frac = f"{covered}/{total}"
-    p0 = f"{p0_covered}/{p0_total}"
-    return f"{label:30}{frac:>12}{p0:>14}"
+    return f"{label:30}{frac:>12}{_percent(covered, total):>11.1f}%"
 
 
 def render(report: CoverageReport) -> str:
-    rows = tuple(
-        _row(m.module, m.covered, m.total, m.p0_covered, m.p0_total)
-        for m in report.modules
-    )
-    pct = (100.0 * report.p0_covered / report.p0_total) if report.p0_total else 0.0
+    rows = tuple(_row(m.module, m.covered, m.total) for m in report.modules)
     lines = (
-        f"{'MODULE':30}{'COVERED':>12}{'P0 COVERED':>14}",
+        f"{'MODULE':30}{'COVERED':>12}{'COVERAGE':>12}",
         *rows,
-        "-" * 56,
-        _row("ALL", report.covered, report.total, report.p0_covered, report.p0_total),
+        "-" * 54,
+        _row("ALL", report.covered, report.total),
         "",
-        f"Headline (P0 coverage): {report.p0_covered}/{report.p0_total}  ({pct:.1f}%)",
+        f"Headline coverage: {report.covered}/{report.total}  ({report.coverage_percent:.1f}%)",
     )
     orphans = (
         (
@@ -162,8 +170,83 @@ def render(report: CoverageReport) -> str:
     return "\n".join((*lines, *orphans, *warning))
 
 
+def _report_dict(report: CoverageReport) -> dict[str, object]:
+    return {
+        "covered": report.covered,
+        "total": report.total,
+        "coverage_percent": report.coverage_percent,
+        "modules": [
+            {
+                "module": m.module,
+                "covered": m.covered,
+                "total": m.total,
+                "coverage_percent": m.coverage_percent,
+                "p0_covered": m.p0_covered,
+                "p0_total": m.p0_total,
+            }
+            for m in report.modules
+        ],
+        "orphan_markers": list(report.orphan_markers),
+        "collection_errors": list(report.collection_errors),
+    }
+
+
+def render_json(report: CoverageReport) -> str:
+    return json.dumps(_report_dict(report), indent=2, sort_keys=True)
+
+
+def _label_value(value: str) -> str:
+    return value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
+
+
+def render_prometheus(report: CoverageReport) -> str:
+    lines = [
+        "# HELP litellm_e2e_coverage_cells E2E coverage registry cells by module and state.",
+        "# TYPE litellm_e2e_coverage_cells gauge",
+    ]
+    for module in report.modules:
+        label = _label_value(module.module)
+        lines.append(
+            f'litellm_e2e_coverage_cells{{module="{label}",state="covered"}} {module.covered}'
+        )
+        lines.append(
+            f'litellm_e2e_coverage_cells{{module="{label}",state="total"}} {module.total}'
+        )
+    lines.extend(
+        [
+            f'litellm_e2e_coverage_cells{{module="ALL",state="covered"}} {report.covered}',
+            f'litellm_e2e_coverage_cells{{module="ALL",state="total"}} {report.total}',
+            "# HELP litellm_e2e_coverage_percent E2E coverage percent by module.",
+            "# TYPE litellm_e2e_coverage_percent gauge",
+        ]
+    )
+    for module in report.modules:
+        label = _label_value(module.module)
+        lines.append(
+            f'litellm_e2e_coverage_percent{{module="{label}"}} {module.coverage_percent:.6f}'
+        )
+    lines.extend(
+        [
+            f'litellm_e2e_coverage_percent{{module="ALL"}} {report.coverage_percent:.6f}',
+            "# HELP litellm_e2e_coverage_orphan_markers Coverage markers not found in the registry.",
+            "# TYPE litellm_e2e_coverage_orphan_markers gauge",
+            f"litellm_e2e_coverage_orphan_markers {len(report.orphan_markers)}",
+            "# HELP litellm_e2e_coverage_collection_errors Pytest nodes that failed during collection.",
+            "# TYPE litellm_e2e_coverage_collection_errors gauge",
+            f"litellm_e2e_coverage_collection_errors {len(report.collection_errors)}",
+        ]
+    )
+    return "\n".join(lines)
+
+
 def main() -> int:
     parser = ArgumentParser()
+    parser.add_argument(
+        "--format",
+        choices=("text", "json", "prometheus"),
+        default="text",
+        help="Output format. Use prometheus or json for Grafana ingestion jobs.",
+    )
     parser.add_argument(
         "--strict",
         action="store_true",
@@ -178,7 +261,14 @@ def main() -> int:
     cells = load_registry()
     covered, errors = collect_covered_ids()
     report = compute_coverage(cells, covered, errors)
-    print(render(report))  # noqa: T201  # CLI entrypoint output
+    output = {
+        "text": render,
+        "json": render_json,
+        "prometheus": render_prometheus,
+    }[
+        args.format
+    ](report)
+    print(output)  # noqa: T201  # CLI entrypoint output
     if args.strict and report.orphan_markers:
         return 1
     if args.fail_on_collection_errors and report.collection_errors:

tests/e2e/coverage_registry/test_collector.py

@@ -11,7 +11,12 @@ from pathlib import Path
 
 import pytest
 
-from coverage_registry.collector import compute_coverage
+from coverage_registry.collector import (
+    compute_coverage,
+    render,
+    render_json,
+    render_prometheus,
+)
 from coverage_registry.registry import load_registry
 from coverage_registry.schema import (
     GuardrailCell,
@@ -104,6 +109,46 @@ def test_llm_cells_roll_up_by_core_endpoint() -> None:
     ) == (2, 1, 1, 1)
 
 
+def test_text_render_uses_plain_coverage_language() -> None:
+    report = compute_coverage(
+        (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")),
+        frozenset({"llm.chat"}),
+    )
+
+    text = render(report)
+
+    assert "COVERAGE" in text
+    assert "Headline coverage: 1/2  (50.0%)" in text
+    assert "P0 COVERED" not in text
+
+
+def test_json_render_exposes_module_coverage_for_grafana_jobs() -> None:
+    report = compute_coverage(
+        (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")),
+        frozenset({"llm.chat"}),
+    )
+
+    payload = render_json(report)
+
+    assert '"coverage_percent": 50.0' in payload
+    assert '"module": "Core LLMs"' in payload
+    assert '"module": "Non-Core LLMs"' in payload
+
+
+def test_prometheus_render_exposes_module_coverage_timeseries() -> None:
+    report = compute_coverage(
+        (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")),
+        frozenset({"llm.chat"}),
+    )
+
+    metrics = render_prometheus(report)
+
+    assert 'litellm_e2e_coverage_cells{module="Core LLMs",state="covered"} 1' in metrics
+    assert 'litellm_e2e_coverage_percent{module="Core LLMs"} 100.000000' in metrics
+    assert 'litellm_e2e_coverage_percent{module="Non-Core LLMs"} 0.000000' in metrics
+    assert "litellm_e2e_coverage_orphan_markers 0" in metrics
+
+
 def test_real_registry_loads_and_ids_are_unique() -> None:
     cells = load_registry()
     ids = [c.id for c in cells]