From c151b15ea667dabd522b8278e916c7c265f93927 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 8 Jul 2026 10:39:32 -0700 Subject: [PATCH] test: emit e2e coverage lines for loki --- tests/e2e/CLAUDE.md | 4 +- tests/e2e/coverage_registry/README.md | 24 +++++++---- tests/e2e/coverage_registry/collector.py | 27 +++++++++--- tests/e2e/coverage_registry/schema.py | 30 +++++++------- tests/e2e/coverage_registry/test_collector.py | 41 +++++++++++++++---- 5 files changed, 88 insertions(+), 38 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 89fa102eb38..e35c79ea204 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -63,7 +63,7 @@ The harness is fully typed and new code must not add `Any` or widen the basedpyr The set of tests we want is a registry checked into this repo, one row per behavior; that file is the definition of done and the denominator. Each e2e test declares what it covers with `@pytest.mark.covers("...")`, and a small collector diffs the registry against the tests and ships coverage to the existing Grafana. No Allure, no new dependencies -Coverage is organized as module > feature > test. Dashboard modules are Core LLMs, Non-Core LLMs, MCPs, Management/UI, Reliability & Performance, Logging & Guardrails, and Other. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` +Coverage is organized as module > feature > test. Dashboard module ids are `core_llms`, `non_core_llms`, `mcp`, `management_ui`, `reliability_performance`, `logging_guardrails`, and `other`. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` The metric is coverage: the share of registry rows that have a passing covering test, reported to Grafana per module so a gap surfaces as an uncovered row rather than a silent absence @@ -71,7 +71,7 @@ Tests do not declare a dashboard module directly. They only declare the registry ### Naming grammar per module -LLMs - endpoint features (subject = the route), seeded from the Claude Code compat matrix. `chat_completions`, `messages`, and `responses` are Core LLMs. Other LLM endpoints, including `batches` and `realtime`, roll up as Non-Core LLMs. +LLMs - endpoint features (subject = the route), seeded from the Claude Code compat matrix. `chat_completions`, `messages`, and `responses` roll up to `core_llms`. Other LLM endpoints, including `batches` and `realtime`, roll up to `non_core_llms`. ``` llm..... diff --git a/tests/e2e/coverage_registry/README.md b/tests/e2e/coverage_registry/README.md index 4177cba7766..62b62318304 100644 --- a/tests/e2e/coverage_registry/README.md +++ b/tests/e2e/coverage_registry/README.md @@ -9,18 +9,18 @@ note; the naming grammar lives in `tests/e2e/CLAUDE.md`. A **cell** is one customer-noticeable behavior a single e2e test can assert pass/fail on, for example `llm.chat_completions.bedrock_converse.tool_use.stream.works`. Cells are -grouped `module > feature > test`, with LLM cells split into Core LLMs and Non-Core -LLMs for dashboarding. Each cell carries a tier (P0/P1/P2), a source, and a +grouped `module > feature > test`, with LLM cells split into `core_llms` and +`non_core_llms` for dashboarding. Each cell carries a tier (P0/P1/P2), a source, and a `fail_before_fix` flag. The rows live in per-prefix YAML files (`llm_*.yaml`, `mgmt.yaml`, `mcp.yaml`, `reliability.yaml`, `logging.yaml`, `guardrail.yaml`, `other.yaml`) and validate against the discriminated union in `schema.py`, so an LLM row cannot carry a guardrail field and vice versa. `llm` rows with `subject_endpoint` of `chat_completions`, `messages`, or -`responses` roll up to "Core LLMs"; all other LLM endpoints roll up to "Non-Core -LLMs". LLM endpoint, route, and capability values are typed in `schema.py`, so new -taxonomy values require an explicit schema change. `logging` and `guardrail` are two -id-prefixes that roll up into the single "Logging & Guardrails" dashboard module. +`responses` roll up to `core_llms`; all other LLM endpoints roll up to `non_core_llms`. +LLM endpoint, route, and capability values are typed in `schema.py`, so new taxonomy +values require an explicit schema change. `logging` and `guardrail` are two id-prefixes +that roll up into the single `logging_guardrails` dashboard module. A test declares what it covers with a marker: @@ -40,8 +40,16 @@ proxy. Whether a covered cell currently passes or fails is a separate, live conc cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector ``` -Use `--format prometheus` or `--format json` for CI jobs that publish coverage to -Grafana. +Use `--format loki` after the e2e pytest run in the same Kubernetes job/pod to print +structured stdout lines for Loki: + +``` +cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector --format loki --strict +``` + +This emits exactly one `COVERAGE_TOTAL` line and one `COVERAGE_MODULE` line per module +in `MODULE_ORDER`. Use `--format prometheus` or `--format json` for jobs that publish +coverage through other ingestion paths. The headline is overall coverage. The collector also lists markers that point at ids not in the registry, so a typo or an unenumerated behavior surfaces instead of being diff --git a/tests/e2e/coverage_registry/collector.py b/tests/e2e/coverage_registry/collector.py index 3b577106605..3c03cd80f0a 100644 --- a/tests/e2e/coverage_registry/collector.py +++ b/tests/e2e/coverage_registry/collector.py @@ -239,13 +239,31 @@ def render_prometheus(report: CoverageReport) -> str: return "\n".join(lines) +def render_loki(report: CoverageReport) -> str: + lines = [ + ( + f"COVERAGE_TOTAL percent={report.coverage_percent:.1f} " + f"covered={report.covered} total={report.total}" + ) + ] + lines.extend( + ( + f"COVERAGE_MODULE module={module.module} " + f"percent={module.coverage_percent:.1f} " + f"covered={module.covered} total={module.total}" + ) + for module in report.modules + ) + return "\n".join(lines) + + def main() -> int: parser = ArgumentParser() parser.add_argument( "--format", - choices=("text", "json", "prometheus"), + choices=("text", "json", "prometheus", "loki"), default="text", - help="Output format. Use prometheus or json for Grafana ingestion jobs.", + help="Output format. Use loki for structured stdout lines in the e2e job.", ) parser.add_argument( "--strict", @@ -265,9 +283,8 @@ def main() -> int: "text": render, "json": render_json, "prometheus": render_prometheus, - }[ - args.format - ](report) + "loki": render_loki, + }[args.format](report) print(output) # noqa: T201 # CLI entrypoint output if args.strict and report.orphan_markers: return 1 diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 2e2a00e78ba..54902051ca0 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -139,22 +139,22 @@ CORE_LLM_ENDPOINTS: frozenset[str] = frozenset( ) PREFIX_ROLLUP: dict[str, str] = { - "mcp": "MCPs", - "mgmt": "Management/UI", - "reliability": "Reliability & Performance", - "logging": "Logging & Guardrails", - "guardrail": "Logging & Guardrails", - "other": "Other", + "mcp": "mcp", + "mgmt": "management_ui", + "reliability": "reliability_performance", + "logging": "logging_guardrails", + "guardrail": "logging_guardrails", + "other": "other", } MODULE_ORDER: tuple[str, ...] = ( - "Core LLMs", - "Non-Core LLMs", - "MCPs", - "Management/UI", - "Reliability & Performance", - "Logging & Guardrails", - "Other", + "core_llms", + "non_core_llms", + "mcp", + "management_ui", + "reliability_performance", + "logging_guardrails", + "other", ) @@ -162,6 +162,6 @@ def dashboard_module(cell: Cell) -> str: """Return the Grafana/reporting module for a registry cell.""" if isinstance(cell, LlmCell): if cell.subject_endpoint in CORE_LLM_ENDPOINTS: - return "Core LLMs" - return "Non-Core LLMs" + return "core_llms" + return "non_core_llms" return PREFIX_ROLLUP[cell.module] diff --git a/tests/e2e/coverage_registry/test_collector.py b/tests/e2e/coverage_registry/test_collector.py index 355bc52730d..8b1a5900982 100644 --- a/tests/e2e/coverage_registry/test_collector.py +++ b/tests/e2e/coverage_registry/test_collector.py @@ -15,6 +15,7 @@ from coverage_registry.collector import ( compute_coverage, render, render_json, + render_loki, render_prometheus, ) from coverage_registry.registry import load_registry @@ -82,7 +83,7 @@ def test_logging_and_guardrail_roll_up_into_one_module() -> None: ) report = compute_coverage(cells, frozenset()) logging_and_guardrails = next( - m for m in report.modules if m.module == "Logging & Guardrails" + m for m in report.modules if m.module == "logging_guardrails" ) assert logging_and_guardrails.total == 2 @@ -97,8 +98,8 @@ def test_llm_cells_roll_up_by_core_endpoint() -> None: ) report = compute_coverage(cells, frozenset({"llm.chat", "llm.batches"})) - core = next(m for m in report.modules if m.module == "Core LLMs") - non_core = next(m for m in report.modules if m.module == "Non-Core LLMs") + core = next(m for m in report.modules if m.module == "core_llms") + non_core = next(m for m in report.modules if m.module == "non_core_llms") assert (core.total, core.covered, core.p0_total, core.p0_covered) == (3, 1, 2, 1) assert ( @@ -131,8 +132,8 @@ def test_json_render_exposes_module_coverage_for_grafana_jobs() -> None: payload = render_json(report) assert '"coverage_percent": 50.0' in payload - assert '"module": "Core LLMs"' in payload - assert '"module": "Non-Core LLMs"' in payload + assert '"module": "core_llms"' in payload + assert '"module": "non_core_llms"' in payload def test_prometheus_render_exposes_module_coverage_timeseries() -> None: @@ -143,12 +144,36 @@ def test_prometheus_render_exposes_module_coverage_timeseries() -> None: metrics = render_prometheus(report) - assert 'litellm_e2e_coverage_cells{module="Core LLMs",state="covered"} 1' in metrics - assert 'litellm_e2e_coverage_percent{module="Core LLMs"} 100.000000' in metrics - assert 'litellm_e2e_coverage_percent{module="Non-Core LLMs"} 0.000000' in metrics + assert 'litellm_e2e_coverage_cells{module="core_llms",state="covered"} 1' in metrics + assert 'litellm_e2e_coverage_percent{module="core_llms"} 100.000000' in metrics + assert 'litellm_e2e_coverage_percent{module="non_core_llms"} 0.000000' in metrics assert "litellm_e2e_coverage_orphan_markers 0" in metrics +def test_loki_render_exposes_exact_stdout_lines_for_loki() -> None: + report = compute_coverage( + (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), + frozenset({"llm.chat"}), + ) + + lines = render_loki(report).splitlines() + + assert len(lines) == 1 + len(report.modules) + assert lines[0] == "COVERAGE_TOTAL percent=50.0 covered=1 total=2" + assert ( + lines[1] == "COVERAGE_MODULE module=core_llms percent=100.0 covered=1 total=1" + ) + assert ( + lines[2] == "COVERAGE_MODULE module=non_core_llms percent=0.0 covered=0 total=1" + ) + assert [line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:]] == [ + module.module for module in report.modules + ] + assert all( + " " not in line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:] + ) + + def test_real_registry_loads_and_ids_are_unique() -> None: cells = load_registry() ids = [c.id for c in cells]