From 9310d179c2a301ef1381163dc9e0c8e1beae9a80 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 7 Jul 2026 16:53:03 -0700 Subject: [PATCH] Split LLM e2e coverage modules --- tests/e2e/CLAUDE.md | 15 ++-- tests/e2e/coverage_registry/README.md | 22 ++++- tests/e2e/coverage_registry/collector.py | 52 +++++++++--- .../coverage_registry/llm_conversational.yaml | 1 + tests/e2e/coverage_registry/schema.py | 80 ++++++++++++++++--- tests/e2e/coverage_registry/test_collector.py | 40 +++++++++- .../test_chat_completions_regression_e2e.py | 26 ++++-- .../test_provider_features_e2e.py | 16 +++- tests/e2e/management/test_management_e2e.py | 8 +- 9 files changed, 212 insertions(+), 48 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 20ab0212854..89fa102eb38 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -63,23 +63,26 @@ The harness is fully typed and new code must not add `Any` or widen the basedpyr The set of tests we want is a registry checked into this repo, one row per behavior; that file is the definition of done and the denominator. Each e2e test declares what it covers with `@pytest.mark.covers("...")`, and a small collector diffs the registry against the tests and ships coverage to the existing Grafana. No Allure, no new dependencies -Coverage is organized as module > feature > test. There are six modules: LLMs, MCPs, Management/UI, Reliability & Performance, Logging & Guardrails, and Other. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` +Coverage is organized as module > feature > test. Dashboard modules are Core LLMs, Non-Core LLMs, MCPs, Management/UI, Reliability & Performance, Logging & Guardrails, and Other. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` The metric is coverage: the share of registry rows that have a passing covering test, reported to Grafana per module so a gap surfaces as an uncovered row rather than a silent absence +Tests do not declare a dashboard module directly. They only declare the registry cell id with `@pytest.mark.covers("...")`; the registry row decides the module, tier, endpoint, and dashboard rollup. Run `python -m coverage_registry.collector --strict` when you want CI to reject unknown marker ids. Add `--fail-on-collection-errors` when the job should also fail on pytest collection errors. + ### Naming grammar per module -LLMs - endpoint features (subject = the route), seeded from the Claude Code compat matrix +LLMs - endpoint features (subject = the route), seeded from the Claude Code compat matrix. `chat_completions`, `messages`, and `responses` are Core LLMs. Other LLM endpoints, including `batches` and `realtime`, roll up as Non-Core LLMs. ``` llm..... endpoint : chat_completions | messages | responses | embeddings | batches | files | rerank | images_generations | audio_speech | audio_transcriptions | moderations - route : openai | azure_openai | anthropic | bedrock_invoke | bedrock_converse | vertex | azure_foundry + | realtime + route : openai | azure_openai | anthropic | bedrock_converse | vertex | azure_foundry + | cohere | together_ai (vocab varies per endpoint; messages is anthropic-format only) - capability : basic | tool_use | prompt_cache_5m | prompt_cache_1h | vision | thinking - | thinking_tool_use | pdf_input | web_search | structured_output | count_tokens - | tool_search | long_context_1m + capability : basic | tool_use | prompt_cache_5m | vision | thinking | structured_output + | service_tier streaming : stream | nonstream (omit where n/a) assertion : works | cost_logged label (not in id): model = haiku-4.5 | sonnet-4.6 | opus-4.7 | gpt-* diff --git a/tests/e2e/coverage_registry/README.md b/tests/e2e/coverage_registry/README.md index a8aab3fcfbc..78b006c7208 100644 --- a/tests/e2e/coverage_registry/README.md +++ b/tests/e2e/coverage_registry/README.md @@ -9,14 +9,18 @@ note; the naming grammar lives in `tests/e2e/CLAUDE.md`. A **cell** is one customer-noticeable behavior a single e2e test can assert pass/fail on, for example `llm.chat_completions.bedrock_converse.tool_use.stream.works`. Cells are -grouped `module > feature > test`, six dashboard modules in all. Each cell carries a -tier (P0/P1/P2), a source, and a `fail_before_fix` flag. +grouped `module > feature > test`, with LLM cells split into Core LLMs and Non-Core +LLMs for dashboarding. Each cell carries a tier (P0/P1/P2), a source, and a +`fail_before_fix` flag. The rows live in per-prefix YAML files (`llm_*.yaml`, `mgmt.yaml`, `mcp.yaml`, `reliability.yaml`, `logging.yaml`, `guardrail.yaml`, `other.yaml`) and validate against the discriminated union in `schema.py`, so an LLM row cannot carry a guardrail field and -vice versa. `logging` and `guardrail` are two id-prefixes that roll up into the single -"Logging & Guardrails" dashboard module. +vice versa. `llm` rows with `subject_endpoint` of `chat_completions`, `messages`, or +`responses` roll up to "Core LLMs"; all other LLM endpoints roll up to "Non-Core +LLMs". LLM endpoint, route, and capability values are typed in `schema.py`, so new +taxonomy values require an explicit schema change. `logging` and `guardrail` are two +id-prefixes that roll up into the single "Logging & Guardrails" dashboard module. A test declares what it covers with a marker: @@ -40,6 +44,16 @@ The headline is P0 coverage. The collector also lists markers that point at ids the registry, so a typo or an unenumerated behavior surfaces instead of being silently dropped. +Use strict mode in CI once existing draft markers are reconciled: + +``` +cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector --strict +``` + +Strict mode exits non-zero on `@pytest.mark.covers(...)` ids that are not checked into +the registry. Add `--fail-on-collection-errors` when the job should also fail on pytest +collection errors. + ## Status: this is a draft for review The cells were enumerated from the codebase and the tiers are a first proposal. Known diff --git a/tests/e2e/coverage_registry/collector.py b/tests/e2e/coverage_registry/collector.py index cc2ce16e3ea..6350ab6251b 100644 --- a/tests/e2e/coverage_registry/collector.py +++ b/tests/e2e/coverage_registry/collector.py @@ -13,13 +13,14 @@ from __future__ import annotations import contextlib import io import sys +from argparse import ArgumentParser from dataclasses import dataclass from pathlib import Path import pytest from .registry import load_registry -from .schema import MODULE_ORDER, ROLLUP, Cell, Tier +from .schema import MODULE_ORDER, Cell, Tier, dashboard_module E2E_DIR = Path(__file__).resolve().parent.parent @@ -46,12 +47,21 @@ class _CoversSink: self.collection_errors = (*self.collection_errors, report.nodeid) -def collect_covered_ids(e2e_dir: Path = E2E_DIR) -> tuple[frozenset[str], tuple[str, ...]]: +def collect_covered_ids( + e2e_dir: Path = E2E_DIR, +) -> tuple[frozenset[str], tuple[str, ...]]: """Return (covered cell ids, nodeids that failed to import).""" sink = _CoversSink() with contextlib.redirect_stdout(io.StringIO()): pytest.main( - ["--collect-only", "-qq", "--continue-on-collection-errors", "-p", "no:cacheprovider", str(e2e_dir)], + [ + "--collect-only", + "-qq", + "--continue-on-collection-errors", + "-p", + "no:cacheprovider", + str(e2e_dir), + ], plugins=[sink], ) return sink.covered_ids, sink.collection_errors @@ -78,8 +88,10 @@ class CoverageReport: collection_errors: tuple[str, ...] -def _module_coverage(module: str, cells: tuple[Cell, ...], covered: frozenset[str]) -> ModuleCoverage: - in_module = tuple(c for c in cells if ROLLUP[c.module] == module) +def _module_coverage( + module: str, cells: tuple[Cell, ...], covered: frozenset[str] +) -> ModuleCoverage: + in_module = tuple(c for c in cells if dashboard_module(c) == module) p0 = tuple(c for c in in_module if c.tier is Tier.P0) return ModuleCoverage( module=module, @@ -116,7 +128,10 @@ def _row(label: str, covered: int, total: int, p0_covered: int, p0_total: int) - def render(report: CoverageReport) -> str: - rows = tuple(_row(m.module, m.covered, m.total, m.p0_covered, m.p0_total) for m in report.modules) + rows = tuple( + _row(m.module, m.covered, m.total, m.p0_covered, m.p0_total) + for m in report.modules + ) pct = (100.0 * report.p0_covered / report.p0_total) if report.p0_total else 0.0 lines = ( f"{'MODULE':30}{'COVERED':>12}{'P0 COVERED':>14}", @@ -129,7 +144,8 @@ def render(report: CoverageReport) -> str: orphans = ( ( f"\n{len(report.orphan_markers)} marker(s) point at ids not in the registry " - f"(reconcile: fix the marker or add the cell):\n " + "\n ".join(report.orphan_markers), + f"(reconcile: fix the marker or add the cell):\n " + + "\n ".join(report.orphan_markers), ) if report.orphan_markers else () @@ -137,7 +153,8 @@ def render(report: CoverageReport) -> str: warning = ( ( f"\nWARNING: {len(report.collection_errors)} node(s) failed to import during " - f"collection, so coverage may undercount:\n " + "\n ".join(report.collection_errors), + f"collection, so coverage may undercount:\n " + + "\n ".join(report.collection_errors), ) if report.collection_errors else () @@ -146,9 +163,26 @@ def render(report: CoverageReport) -> str: def main() -> int: + parser = ArgumentParser() + parser.add_argument( + "--strict", + action="store_true", + help="Exit non-zero if markers outside the registry are found.", + ) + parser.add_argument( + "--fail-on-collection-errors", + action="store_true", + help="Exit non-zero if pytest collection errors are found.", + ) + args = parser.parse_args() cells = load_registry() covered, errors = collect_covered_ids() - print(render(compute_coverage(cells, covered, errors))) # noqa: T201 # CLI entrypoint output + report = compute_coverage(cells, covered, errors) + print(render(report)) # noqa: T201 # CLI entrypoint output + if args.strict and report.orphan_markers: + return 1 + if args.fail_on_collection_errors and report.collection_errors: + return 1 return 0 diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 365776da8bf..7360aacb916 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -6,6 +6,7 @@ - {id: llm.chat_completions.openai.tool_use.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: tool_use, streaming: stream, assertions: [works], source: "model_prices json", rationale: "Tool calls over streaming"} - {id: llm.chat_completions.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "gpt-4o vision; high usage"} - {id: llm.chat_completions.openai.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching cost optimization"} +- {id: llm.chat_completions.openai.service_tier.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: nonstream, assertions: [works], source: "OpenAI service_tier param", rationale: "OpenAI scale-tier request option is forwarded and echoed"} - {id: llm.chat_completions.openai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "o-series reasoning; emerging"} - {id: llm.chat_completions.openai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "response_schema extraction"} - {id: llm.chat_completions.anthropic.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route translated to Anthropic"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 756bde33e3d..2e2a00e78ba 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -1,9 +1,9 @@ """Registry row schema: the contract every denominator cell validates against. A cell is one customer-noticeable behavior a single e2e test can assert pass/fail -on. `module` is the id's segment-1 prefix (seven of them); the six-way dashboard -rollup merges logging + guardrail via ROLLUP. The union is discriminated on -`module`, so an LLM row cannot carry a guardrail field and vice versa. +on. `module` is the id's segment-1 prefix (seven of them); dashboard rollups can +split or merge those prefixes. The union is discriminated on `module`, so an LLM +row cannot carry a guardrail field and vice versa. """ from __future__ import annotations @@ -25,6 +25,43 @@ class FailBeforeFix(str, Enum): unproven = "unproven" +LlmEndpoint = Literal[ + "chat_completions", + "messages", + "responses", + "embeddings", + "batches", + "files", + "rerank", + "images_generations", + "audio_speech", + "audio_transcriptions", + "moderations", + "realtime", +] + +LlmRoute = Literal[ + "anthropic", + "azure_foundry", + "azure_openai", + "bedrock_converse", + "cohere", + "openai", + "together_ai", + "vertex", +] + +LlmCapability = Literal[ + "basic", + "prompt_cache_5m", + "service_tier", + "structured_output", + "thinking", + "tool_use", + "vision", +] + + class _Base(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -39,9 +76,9 @@ class _Base(BaseModel): class LlmCell(_Base): module: Literal["llm"] - subject_endpoint: str - route: str - capability: str + subject_endpoint: LlmEndpoint + route: LlmRoute + capability: LlmCapability streaming: Literal["stream", "nonstream", "na"] @@ -81,14 +118,27 @@ class OtherCell(_Base): Cell = Annotated[ - LlmCell | MgmtCell | McpCell | ReliabilityCell | LoggingCell | GuardrailCell | OtherCell, + LlmCell + | MgmtCell + | McpCell + | ReliabilityCell + | LoggingCell + | GuardrailCell + | OtherCell, Field(discriminator="module"), ] CELL_ADAPTER: TypeAdapter[Cell] = TypeAdapter(Cell) -ROLLUP: dict[str, str] = { - "llm": "LLMs", +CORE_LLM_ENDPOINTS: frozenset[str] = frozenset( + { + "chat_completions", + "messages", + "responses", + } +) + +PREFIX_ROLLUP: dict[str, str] = { "mcp": "MCPs", "mgmt": "Management/UI", "reliability": "Reliability & Performance", @@ -98,10 +148,20 @@ ROLLUP: dict[str, str] = { } MODULE_ORDER: tuple[str, ...] = ( - "LLMs", + "Core LLMs", + "Non-Core LLMs", "MCPs", "Management/UI", "Reliability & Performance", "Logging & Guardrails", "Other", ) + + +def dashboard_module(cell: Cell) -> str: + """Return the Grafana/reporting module for a registry cell.""" + if isinstance(cell, LlmCell): + if cell.subject_endpoint in CORE_LLM_ENDPOINTS: + return "Core LLMs" + return "Non-Core LLMs" + return PREFIX_ROLLUP[cell.module] diff --git a/tests/e2e/coverage_registry/test_collector.py b/tests/e2e/coverage_registry/test_collector.py index 065ebafdb3b..5178e217f31 100644 --- a/tests/e2e/coverage_registry/test_collector.py +++ b/tests/e2e/coverage_registry/test_collector.py @@ -13,17 +13,25 @@ import pytest from coverage_registry.collector import compute_coverage from coverage_registry.registry import load_registry -from coverage_registry.schema import GuardrailCell, LlmCell, LoggingCell, Tier +from coverage_registry.schema import ( + GuardrailCell, + LlmCell, + LlmEndpoint, + LoggingCell, + Tier, +) -def _llm(cell_id: str, tier: Tier) -> LlmCell: +def _llm( + cell_id: str, tier: Tier, subject_endpoint: LlmEndpoint = "chat_completions" +) -> LlmCell: return LlmCell( id=cell_id, module="llm", tier=tier, assertions=("works",), source="test", - subject_endpoint="chat_completions", + subject_endpoint=subject_endpoint, route="openai", capability="basic", streaming="nonstream", @@ -68,10 +76,34 @@ def test_logging_and_guardrail_roll_up_into_one_module() -> None: ), ) report = compute_coverage(cells, frozenset()) - logging_and_guardrails = next(m for m in report.modules if m.module == "Logging & Guardrails") + logging_and_guardrails = next( + m for m in report.modules if m.module == "Logging & Guardrails" + ) assert logging_and_guardrails.total == 2 +def test_llm_cells_roll_up_by_core_endpoint() -> None: + cells = ( + _llm("llm.chat", Tier.P0, "chat_completions"), + _llm("llm.messages", Tier.P0, "messages"), + _llm("llm.responses", Tier.P1, "responses"), + _llm("llm.batches", Tier.P0, "batches"), + _llm("llm.realtime", Tier.P1, "realtime"), + ) + report = compute_coverage(cells, frozenset({"llm.chat", "llm.batches"})) + + core = next(m for m in report.modules if m.module == "Core LLMs") + non_core = next(m for m in report.modules if m.module == "Non-Core LLMs") + + assert (core.total, core.covered, core.p0_total, core.p0_covered) == (3, 1, 2, 1) + assert ( + non_core.total, + non_core.covered, + non_core.p0_total, + non_core.p0_covered, + ) == (2, 1, 1, 1) + + def test_real_registry_loads_and_ids_are_unique() -> None: cells = load_registry() ids = [c.id for c in cells] diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 269cb5d6d22..5cc4ff308fa 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -33,7 +33,12 @@ class TestChatCompletionsRegression: CHAT_MODELS, ids=[f"{model}-{route}" for model, route in CHAT_MODELS], ) - @pytest.mark.covers("llm.chat_completions.provider.basic.nonstream.works", exercised_on=[]) + @pytest.mark.covers( + "llm.chat_completions.openai.basic.nonstream.works", + "llm.chat_completions.anthropic.basic.nonstream.works", + "llm.chat_completions.vertex.basic.nonstream.works", + exercised_on=[], + ) def test_chat_returns_real_completion( self, client: PassthroughClient, scoped_key: str, model: str, route: str ) -> None: @@ -43,16 +48,23 @@ class TestChatCompletionsRegression: ChatBody( model=model, messages=[ - ChatMessage(role="user", content=f"reply with one word {unique_marker()}") + ChatMessage( + role="user", + content=f"reply with one word {unique_marker()}", + ) ], max_tokens=512, ), ) ) - assert response.model, f"{model} ({route}): response carried no model name: {response}" - assert response.choices, f"{model} ({route}): response had no choices: {response}" + assert ( + response.model + ), f"{model} ({route}): response carried no model name: {response}" + assert ( + response.choices + ), f"{model} ({route}): response had no choices: {response}" message = response.choices[0].message - assert message is not None and message.content and message.content.strip(), ( - f"{model} ({route}): 200 with an empty completion (#28991): {response}" - ) + assert ( + message is not None and message.content and message.content.strip() + ), f"{model} ({route}): 200 with an empty completion (#28991): {response}" diff --git a/tests/e2e/llm_translation/test_provider_features_e2e.py b/tests/e2e/llm_translation/test_provider_features_e2e.py index 9c99c1be161..cf05a4306b4 100644 --- a/tests/e2e/llm_translation/test_provider_features_e2e.py +++ b/tests/e2e/llm_translation/test_provider_features_e2e.py @@ -76,13 +76,18 @@ def post_chat(client: PassthroughClient, key: str, body: BaseModel) -> ChatRespo class TestServiceTier: - @pytest.mark.covers("llm.chat_completions.openai.service_tier.works", exercised_on=[]) + @pytest.mark.covers( + "llm.chat_completions.openai.service_tier.nonstream.works", exercised_on=[] + ) def test_openai_service_tier_is_echoed( self, client: PassthroughClient, resources: ResourceManager ) -> None: model = f"e2e-service-tier-{unique_marker()}" model_id = client.gateway.create_model( - model, LiteLLMParamsBody(model="openai/gpt-5.5", api_key="os.environ/OPENAI_API_KEY") + model, + LiteLLMParamsBody( + model="openai/gpt-5.5", api_key="os.environ/OPENAI_API_KEY" + ), ) resources.defer(lambda: client.gateway.delete_model(model_id)) key = resources.key() @@ -106,7 +111,8 @@ class TestServiceTier: class TestPromptCaching: @pytest.mark.covers( - "llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.cache_hit", exercised_on=[] + "llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.works", + exercised_on=[], ) def test_bedrock_cache_control_produces_cache_read( self, client: PassthroughClient, resources: ResourceManager @@ -129,7 +135,9 @@ class TestPromptCaching: RichMessage( role="user", content=[ - CacheTextBlock(text=cacheable_prefix(), cache_control=CacheControl()), + CacheTextBlock( + text=cacheable_prefix(), cache_control=CacheControl() + ), CacheTextBlock(text="Answer in one word: acknowledged?"), ], ) diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index 3beb039b8bd..cbd5db0d59f 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -167,7 +167,7 @@ class TestKeyRoutes: class TestTeamRoutes: - @pytest.mark.covers("management.team.new.persists") + @pytest.mark.covers("mgmt.team.new.persists") def test_new_persists_to_team_info_and_binds_keys( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -212,7 +212,7 @@ class TestTeamRoutes: class TestUserRoutes: - @pytest.mark.covers("mgmt.user.new.persists") + @pytest.mark.covers("mgmt.user.new.happy_path") def test_new_persists_to_user_info(self, client: ManagementClient, resources: ResourceManager) -> None: email = f"e2e-mgmt-{unique_marker()}@example.com" user_id = _create_user(client, resources, UserNewBody(user_email=email, user_role="internal_user")) @@ -225,7 +225,7 @@ class TestUserRoutes: class TestOrganizationRoutes: - @pytest.mark.covers("mgmt.organization.new.persists") + @pytest.mark.covers("mgmt.organization.new.happy_path") def test_new_persists_to_organization_info( self, client: ManagementClient, resources: ResourceManager ) -> None: @@ -252,7 +252,7 @@ def _assert_route_forbidden(route: str, outcome: StreamingResponse) -> None: class TestManagementRoutePermissions: - @pytest.mark.covers("mgmt.key.generate.member_forbidden") + @pytest.mark.covers("other.auth.virtual_key.route_permission_enforced") def test_llm_only_key_forbidden_from_management_writes( self, client: ManagementClient, resources: ResourceManager ) -> None: