From d896b59923b0dc3f08866fa2c2a5610ff650ede8 Mon Sep 17 00:00:00 2001 From: yassin Date: Fri, 17 Jul 2026 17:36:28 +0000 Subject: [PATCH] refactor(e2e): generate coverage denominator from product surface Derive the coverage denominator from the schema vocabulary crossed with model_prices_and_context_window.json instead of hand-listed YAML rows, keep the human fields (tier, source, rationale, fail_before_fix, supported) in a curated overlay keyed by cell id, and parse endpoint/route/capability/streaming from the cell id rather than storing them. A test PR now only adds a covers marker. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 6 +- tests/e2e/coverage_registry/README.md | 104 +++-- tests/e2e/coverage_registry/collector.py | 68 +-- tests/e2e/coverage_registry/guardrail.yaml | 29 -- .../llm_claude_code_compat.yaml | 110 ----- .../coverage_registry/llm_conversational.yaml | 56 --- .../llm_nonconversational.yaml | 45 -- tests/e2e/coverage_registry/logging.yaml | 25 -- tests/e2e/coverage_registry/mcp.yaml | 113 ----- tests/e2e/coverage_registry/mgmt.yaml | 68 --- tests/e2e/coverage_registry/other.yaml | 28 -- tests/e2e/coverage_registry/overlay.py | 24 ++ tests/e2e/coverage_registry/overlay.yaml | 408 ++++++++++++++++++ .../e2e/coverage_registry/product_surface.py | 228 ++++++++++ .../coverage_registry/quota_management.yaml | 35 -- tests/e2e/coverage_registry/registry.py | 44 +- tests/e2e/coverage_registry/reliability.yaml | 27 -- tests/e2e/coverage_registry/schema.py | 170 ++++---- tests/e2e/coverage_registry/test_collector.py | 200 +++++---- 19 files changed, 1002 insertions(+), 786 deletions(-) delete mode 100644 tests/e2e/coverage_registry/guardrail.yaml delete mode 100644 tests/e2e/coverage_registry/llm_claude_code_compat.yaml delete mode 100644 tests/e2e/coverage_registry/llm_conversational.yaml delete mode 100644 tests/e2e/coverage_registry/llm_nonconversational.yaml delete mode 100644 tests/e2e/coverage_registry/logging.yaml delete mode 100644 tests/e2e/coverage_registry/mcp.yaml delete mode 100644 tests/e2e/coverage_registry/mgmt.yaml delete mode 100644 tests/e2e/coverage_registry/other.yaml create mode 100644 tests/e2e/coverage_registry/overlay.py create mode 100644 tests/e2e/coverage_registry/overlay.yaml create mode 100644 tests/e2e/coverage_registry/product_surface.py delete mode 100644 tests/e2e/coverage_registry/quota_management.yaml delete mode 100644 tests/e2e/coverage_registry/reliability.yaml diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 0e1eafb5196..8b5168c3f21 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -61,13 +61,13 @@ The harness is fully typed with no error budget: `make lint-e2e-basedpyright` mu ## Coverage registry -The set of tests we want is a registry checked into this repo, one row per behavior; that file is the definition of done and the denominator. Each e2e test declares what it covers with `@pytest.mark.covers("...")`, and a small collector diffs the registry against the tests and ships coverage to the existing Grafana. No Allure, no new dependencies +The set of tests we want is a denominator generated from the product surface, not a hand-written list; that generated set is the definition of done. Each e2e test declares what it covers with `@pytest.mark.covers("...")`, and a small collector diffs the denominator against the tests and ships coverage to the existing Grafana. No Allure, no new dependencies. A hand-written list is self-fulfilling (the row and its test land in the same PR, so a wanted-but-untested behavior never surfaces as a gap), so the denominator is derived instead; see `coverage_registry/README.md` Coverage is organized as module > feature > test. Dashboard modules are `Core LLMs`, `Non-Core LLMs`, `MCPs`, `Management/UI`, `Reliability & Performance`, `Quota Management`, `Logging & Guardrails`, and `Other`. The Loki stdout formatter maps those display modules to log-safe labels (`core_llms`, `non_core_llms`, `mcp`, `management_ui`, `reliability_performance`, `quota_management`, `logging_guardrails`, and `other`) without changing JSON or Prometheus labels. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` -The metric is coverage: the share of registry rows that have a passing covering test, reported to Grafana per module so a gap surfaces as an uncovered row rather than a silent absence +The metric is coverage: the share of denominator cells that have a passing covering test, reported to Grafana per module so a gap surfaces as an uncovered cell rather than a silent absence -Tests do not declare a dashboard module directly. They only declare the registry cell id with `@pytest.mark.covers("...")`; the registry row decides the module, tier, endpoint, and dashboard rollup. Run `python -m coverage_registry.collector --strict` when you want CI to reject unknown marker ids. Add `--fail-on-collection-errors` when the job should also fail on pytest collection errors. +Tests do not declare a dashboard module directly, and they never add a denominator row. They only declare the cell id with `@pytest.mark.covers("...")`; the module, endpoint, and dashboard rollup are parsed from that id, and the conversational-core cell set is generated in `coverage_registry/product_surface.py` from the schema vocabulary crossed with `model_prices_and_context_window.json`. The only hand-curated data is `coverage_registry/overlay.yaml`, keyed by cell id, carrying tier, source, rationale, fail_before_fix, and supported; it is owner-gated and ordinary test PRs do not touch it. Run `python -m coverage_registry.collector --strict` when you want CI to reject unknown marker ids. Add `--fail-on-collection-errors` when the job should also fail on pytest collection errors. ### Naming grammar per module diff --git a/tests/e2e/coverage_registry/README.md b/tests/e2e/coverage_registry/README.md index aef4c16c89a..56206015474 100644 --- a/tests/e2e/coverage_registry/README.md +++ b/tests/e2e/coverage_registry/README.md @@ -1,29 +1,52 @@ # e2e coverage registry -This directory is the **denominator** for e2e test coverage: the set of behaviors we -want covered, one row per behavior, checked into the repo so coverage is a number we -can track instead of a guess. It implements the plan in the "E2E Coverage Tracking" -note; the naming grammar lives in `tests/e2e/CLAUDE.md`. +This directory holds the **denominator** for e2e test coverage: the set of behaviors we +want covered. The denominator is generated from the product surface rather than +hand-listed, so it grows on its own when the product gains a surface; a test PR only adds +a `@pytest.mark.covers(...)` marker. The naming grammar lives in `tests/e2e/CLAUDE.md`. + +## Why it is generated + +The denominator used to be a hand-written set of YAML rows, one per behavior. That is +self-fulfilling: the row and its covering test land in the same PR, so a wanted-but-untested +behavior never shows up as a gap. Deriving the denominator from the tests instead is just +as useless; it is 100% by construction. So the set of cells is derived from what the proxy +actually supports, and the tests are diffed against it. ## The model -A **cell** is one customer-noticeable behavior a single e2e test can assert pass/fail -on, for example `llm.chat_completions.bedrock_converse.tool_use.stream.works`. Cells are -grouped `module > feature > test`, with LLM cells split into `Core LLMs` and -`Non-Core LLMs` for dashboarding. Each cell carries a tier (P0/P1/P2), a source, and a -`fail_before_fix` flag. +A **cell** is one customer-noticeable behavior a single e2e test can assert pass/fail on, +for example `llm.chat_completions.bedrock_converse.tool_use.stream.works`. Cells are grouped +`module > feature > test`, with LLM cells split into `Core LLMs` and `Non-Core LLMs` for +dashboarding. The module, endpoint, route, capability and streaming are parsed back out of +the id (see `parse_llm_id` / `parse_module` in `schema.py`); nothing restates them, so an id +and its facets can never drift. -The rows live in per-prefix YAML files (`llm_*.yaml`, `mgmt.yaml`, `mcp.yaml`, -`reliability.yaml`, `quota_management.yaml`, `logging.yaml`, `guardrail.yaml`, -`other.yaml`) and validate against -the discriminated union in `schema.py`, so an LLM row cannot carry a guardrail field and -vice versa. `llm` rows with `subject_endpoint` of `chat_completions`, `messages`, or -`responses` roll up to `Core LLMs`; all other LLM endpoints roll up to `Non-Core LLMs`. -LLM endpoint, route, and capability values are typed in `schema.py`, so new taxonomy -values require an explicit schema change. `logging` and `guardrail` are two id-prefixes -that roll up into the single `Logging & Guardrails` dashboard module. +The denominator is the union of two sources. -A test declares what it covers with a marker: +Generated surface (`product_surface.py`). The conversational core (chat_completions, +messages, responses) is generated from the typed vocabularies in `schema.py` crossed with +the capability metadata in `model_prices_and_context_window.json`, the same file the proxy +ships. A flagged capability (tool_use, vision, thinking, structured_output, prompt caching) +is emitted for a route only when a model on that route advertises the matching `supports_*` +flag, so adding provider support in that json grows the denominator with no edit here. The +Anthropic-format `messages` surface is the Claude Code compatibility matrix, so its +capabilities are the CLI feature set and are not gated by model flags. The live route table +(`litellm.proxy._types.LiteLLMRoutes`) is read as a drift check: the collector warns when +the vocabulary enumerates an LLM endpoint the proxy no longer serves. This exact set of +product-surface sources is a design decision open to review. + +Curated overlay (`overlay.yaml`). The only per-cell data a human decides lives here, keyed +by cell id: `tier`, `source`, `rationale`, `fail_before_fix`, and `supported`. The overlay +never decides whether a behavior exists; it annotates a generated cell, and it also +enumerates the ids that generation does not yet produce (the non-core LLM operations such as +batches, files, rerank, embeddings, audio and images, and the behavior modules mgmt, mcp, +reliability, quota, logging, guardrail, other, which have no clean cartesian to generate +from). A generated cell with no overlay row defaults to P2, so a newly generated surface +shows up as an uncovered gap rather than vanishing. Ordinary test PRs never touch this file; +it should be owner-gated via CODEOWNERS (not added here). + +A test declares what it covers with a marker, and nothing else: ```python @pytest.mark.covers("llm.chat_completions.openai.tool_use.stream.works") @@ -33,7 +56,7 @@ def test_openai_streaming_tool_calls(self) -> None: ## The number -`collector.py` diffs the registry against those markers and reports coverage per module. +`collector.py` diffs the denominator against those markers and reports coverage per module. It is static: a collect-only pass reads the markers, so it runs no test and needs no live proxy. Whether a covered cell currently passes or fails is a separate, live concern. @@ -48,35 +71,30 @@ structured stdout lines for Loki: cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector --format loki --strict ``` -This emits exactly one `COVERAGE_TOTAL` line and one `COVERAGE_MODULE` line per module -in `MODULE_ORDER`, in that order. Loki uses log-safe `module=` labels from -`LOKI_MODULE_LABELS` (`core_llms`, `management_ui`, etc.) so existing JSON and -Prometheus consumers keep their human-readable module names unchanged. +This emits exactly one `COVERAGE_TOTAL` line and one `COVERAGE_MODULE` line per module in +`MODULE_ORDER`, in that order. Loki uses log-safe `module=` labels from `LOKI_MODULE_LABELS` +(`core_llms`, `management_ui`, etc.) so existing JSON and Prometheus consumers keep their +human-readable module names unchanged. -The headline is overall coverage. The collector also lists markers that point at ids -not in the registry, so a typo or an unenumerated behavior surfaces instead of being -silently dropped. +The headline is overall coverage. The collector also lists markers that point at ids not in +the denominator, so a typo or an unenumerated behavior surfaces instead of being silently +dropped, and warns on route-table drift. -Use strict mode in CI once existing draft markers are reconciled: +Use strict mode in CI once existing markers are reconciled: ``` cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector --strict ``` -Strict mode exits non-zero on `@pytest.mark.covers(...)` ids that are not checked into -the registry. Add `--fail-on-collection-errors` when the job should also fail on pytest -collection errors. +Strict mode exits non-zero on `@pytest.mark.covers(...)` ids that are not in the denominator. +Add `--fail-on-collection-errors` when the job should also fail on pytest collection errors. -## Status: this is a draft for review +## Follow-up -The cells were enumerated from the codebase and the tiers are a first proposal. Known -things to settle before treating the set as final: - -- tiers are proposed, not signed off; 125 P0 is a lot to prove fail-before-fix, so P0 may - want tightening -- a few cells need a support check or a prune (for example `llm.embeddings.anthropic.*` - and `reliability.perf.throughput.under_slo`) -- auth is covered in two places (`other.auth.*` and the mgmt authz assertions); the - boundary needs a decision, and the auth cluster may deserve promotion to its own module -- the P2 "niche" cells each stand in for a large tail of integrations/providers by design, - so the denominator is deliberately P0-weighted rather than a full inventory +Generation currently covers the conversational core only. The non-core LLM operations and +the behavior modules still live in the overlay because they have no clean product-surface +enumeration yet; wiring their own sources into generation (the route table for endpoint +operations, `guardrail_hooks/` for guardrail providers, `integrations/` for logging targets, +`router_strategy/` for reliability behaviors) is the remaining work, tracked rather than +faked. Tiers in the overlay were migrated from the previous hand-written rows and are a +first proposal, not a sign-off. diff --git a/tests/e2e/coverage_registry/collector.py b/tests/e2e/coverage_registry/collector.py index 50ef23bcb1a..f6c1ba19428 100644 --- a/tests/e2e/coverage_registry/collector.py +++ b/tests/e2e/coverage_registry/collector.py @@ -22,8 +22,9 @@ from typing import Literal import pytest from pydantic import BaseModel +from .product_surface import ROUTE_CHECKABLE_ENDPOINTS, route_table_endpoints from .registry import load_registry -from .schema import MODULE_ORDER, Cell, Tier, dashboard_module, loki_module_label +from .schema import MODULE_ORDER, Cell, Tier, dashboard_module, loki_module_label, parse_llm_id E2E_DIR = Path(__file__).resolve().parent.parent @@ -38,13 +39,9 @@ class _CoversSink: def pytest_collection_finish(self, session: pytest.Session) -> None: marker_args: tuple[tuple[object, ...], ...] = tuple( - marker.args - for item in session.items - for marker in item.iter_markers(name="covers") - ) - self.covered_ids = frozenset( - arg for args in marker_args for arg in args if isinstance(arg, str) + marker.args for item in session.items for marker in item.iter_markers(name="covers") ) + self.covered_ids = frozenset(arg for args in marker_args for arg in args if isinstance(arg, str)) def pytest_collectreport(self, report: pytest.CollectReport) -> None: if report.failed: @@ -94,6 +91,7 @@ class CoverageReport: p0_gaps: tuple[str, ...] orphan_markers: tuple[str, ...] collection_errors: tuple[str, ...] + route_table_drift: tuple[str, ...] = () @property def coverage_percent(self) -> float: @@ -104,9 +102,7 @@ def _percent(covered: int, total: int) -> float: return (100.0 * covered / total) if total else 0.0 -def _module_coverage( - module: str, cells: tuple[Cell, ...], covered: frozenset[str] -) -> ModuleCoverage: +def _module_coverage(module: str, cells: tuple[Cell, ...], covered: frozenset[str]) -> ModuleCoverage: in_module = tuple(c for c in cells if dashboard_module(c) == module) p0 = tuple(c for c in in_module if c.tier is Tier.P0) return ModuleCoverage( @@ -122,6 +118,7 @@ def compute_coverage( cells: tuple[Cell, ...], covered: frozenset[str], collection_errors: tuple[str, ...] = (), + route_table_drift: tuple[str, ...] = (), ) -> CoverageReport: p0_cells = tuple(c for c in cells if c.tier is Tier.P0) registry_ids = frozenset(c.id for c in cells) @@ -134,9 +131,22 @@ def compute_coverage( p0_gaps=tuple(sorted(c.id for c in p0_cells if c.id not in covered)), orphan_markers=tuple(sorted(covered - registry_ids)), collection_errors=collection_errors, + route_table_drift=route_table_drift, ) +def compute_route_table_drift(cells: tuple[Cell, ...]) -> tuple[str, ...]: + """LLM endpoints the denominator enumerates that the live proxy route table does + not serve. Empty when the route table cannot be read (litellm not importable) or + when everything the denominator enumerates is served.""" + served = route_table_endpoints() + if served is None: + return () + enumerated = frozenset(parsed.endpoint for c in cells if (parsed := parse_llm_id(c.id)) is not None) + checkable = enumerated & ROUTE_CHECKABLE_ENDPOINTS + return tuple(sorted(endpoint for endpoint in checkable if endpoint not in served)) + + def _row(label: str, covered: int, total: int) -> str: frac = f"{covered}/{total}" return f"{label:30}{frac:>12}{_percent(covered, total):>11.1f}%" @@ -155,8 +165,7 @@ def render(report: CoverageReport) -> str: orphans = ( ( f"\n{len(report.orphan_markers)} marker(s) point at ids not in the registry " - f"(reconcile: fix the marker or add the cell):\n " - + "\n ".join(report.orphan_markers), + f"(reconcile: fix the marker or add the cell):\n " + "\n ".join(report.orphan_markers), ) if report.orphan_markers else () @@ -164,13 +173,21 @@ def render(report: CoverageReport) -> str: warning = ( ( f"\nWARNING: {len(report.collection_errors)} node(s) failed to import during " - f"collection, so coverage may undercount:\n " - + "\n ".join(report.collection_errors), + f"collection, so coverage may undercount:\n " + "\n ".join(report.collection_errors), ) if report.collection_errors else () ) - return "\n".join((*lines, *orphans, *warning)) + drift = ( + ( + f"\nWARNING: {len(report.route_table_drift)} enumerated LLM endpoint(s) are not in " + f"the live proxy route table (denominator drift, reconcile the schema vocab):\n " + + "\n ".join(report.route_table_drift), + ) + if report.route_table_drift + else () + ) + return "\n".join((*lines, *orphans, *warning, *drift)) def _report_dict(report: CoverageReport) -> dict[str, object]: @@ -209,12 +226,8 @@ def render_prometheus(report: CoverageReport) -> str: ] for module in report.modules: label = _label_value(module.module) - lines.append( - f'litellm_e2e_coverage_cells{{module="{label}",state="covered"}} {module.covered}' - ) - lines.append( - f'litellm_e2e_coverage_cells{{module="{label}",state="total"}} {module.total}' - ) + lines.append(f'litellm_e2e_coverage_cells{{module="{label}",state="covered"}} {module.covered}') + lines.append(f'litellm_e2e_coverage_cells{{module="{label}",state="total"}} {module.total}') lines.extend( [ f'litellm_e2e_coverage_cells{{module="ALL",state="covered"}} {report.covered}', @@ -225,9 +238,7 @@ def render_prometheus(report: CoverageReport) -> str: ) for module in report.modules: label = _label_value(module.module) - lines.append( - f'litellm_e2e_coverage_percent{{module="{label}"}} {module.coverage_percent:.6f}' - ) + lines.append(f'litellm_e2e_coverage_percent{{module="{label}"}} {module.coverage_percent:.6f}') lines.extend( [ f'litellm_e2e_coverage_percent{{module="ALL"}} {report.coverage_percent:.6f}', @@ -243,12 +254,7 @@ def render_prometheus(report: CoverageReport) -> str: def render_loki(report: CoverageReport) -> str: - lines = [ - ( - f"COVERAGE_TOTAL percent={report.coverage_percent:.1f} " - f"covered={report.covered} total={report.total}" - ) - ] + lines = [(f"COVERAGE_TOTAL percent={report.coverage_percent:.1f} covered={report.covered} total={report.total}")] lines.extend( ( f"COVERAGE_MODULE module={loki_module_label(module.module)} " @@ -287,7 +293,7 @@ def main() -> int: args = _CliArgs.model_validate(vars(parser.parse_args())) cells = load_registry() covered, errors = collect_covered_ids() - report = compute_coverage(cells, covered, errors) + report = compute_coverage(cells, covered, errors, compute_route_table_drift(cells)) output = { "text": render, "json": render_json, diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml deleted file mode 100644 index 792cbaaff7c..00000000000 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Guardrail enforcement (behavior features). Grounded in litellm/proxy/guardrails/guardrail_hooks/. -# Rolls up into the "Logging & Guardrails" dashboard module together with logging.* -- {id: guardrail.presidio.pre_call.masks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "PII masking pre-call; data-leak blast radius"} -- {id: guardrail.presidio.post_call.masks, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Mask PII in model output"} -- {id: guardrail.presidio.logging_only.masks, module: guardrail, tier: P0, hook_point: logging_only, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Redact in logs without blocking"} -- {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"} -- {id: guardrail.bedrock.during.blocks, module: guardrail, tier: P0, hook_point: during, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "During-call moderation for streaming"} -- {id: guardrail.bedrock.post_call.blocks, module: guardrail, tier: P0, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "Block harmful output"} -- {id: guardrail.lakera.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/lakera_ai_v2.py", rationale: "Prompt-injection block pre-execution"} -- {id: guardrail.lakera.post_call.blocks, module: guardrail, tier: P0, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/lakera_ai_v2.py", rationale: "Post-call injection on multi-turn chains"} -- {id: guardrail.openai_moderations.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/openai/moderations.py", rationale: "Content policy for regulated industries"} -- {id: guardrail.aim.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/aim/aim.py", rationale: "Security guardrail malicious-input"} -- {id: guardrail.aim.post_call.blocks, module: guardrail, tier: P1, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/aim/aim.py", rationale: "Output security check"} -- {id: guardrail.ibm_guardrails.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/ibm_guardrails/ibm_detector.py", rationale: "Enterprise multi-policy"} -- {id: guardrail.ibm_guardrails.post_call.blocks, module: guardrail, tier: P1, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/ibm_guardrails/ibm_detector.py", rationale: "Output policy validation"} -- {id: guardrail.semantic_guard.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/semantic_guard", rationale: "Semantic policy compliance"} -- {id: guardrail.block_code_execution.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/block_code_execution", rationale: "Code-injection prevention"} -- {id: guardrail.tool_permission.pre_call.allows, module: guardrail, tier: P1, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "guardrail_hooks/tool_permission.py", rationale: "Grant allowed tools"} -- {id: guardrail.tool_permission.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/tool_permission.py", rationale: "Block unauthorized tools"} -- {id: guardrail.microsoft_purview.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/microsoft_purview/purview_dlp.py", rationale: "DLP sensitive-data disclosure"} -- {id: guardrail.headroom.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/headroom/headroom.py", rationale: "Anomaly detection threshold"} -- {id: guardrail.generic_guardrail_api.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py", rationale: "Vendor-agnostic custom API"} -- {id: guardrail.pangea.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/pangea/pangea.py", rationale: "API security + DLP"} -- {id: guardrail.niche_providers.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE cohort: lasso/hiddenlayer/model_armor/qualifire/guardrails_ai/cato/cisco/akto/prompt_security/promptguard/zscaler/vigil/etc"} -- {id: guardrail.niche_providers.post_call.blocks, module: guardrail, tier: P2, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE niche output filtering"} -- {id: guardrail.niche_providers.pre_call.allows, module: guardrail, tier: P2, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE niche allow-path passthrough"} -- {id: guardrail.tool_policy.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/tool_policy/tool_policy_guardrail.py", rationale: "Tool-use policy enforcement"} -- {id: guardrail.mcp_security.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/mcp_security", rationale: "MCP protocol security"} -- {id: guardrail.llm_as_a_judge.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/llm_as_a_judge", rationale: "LLM-based judgment guardrail"} diff --git a/tests/e2e/coverage_registry/llm_claude_code_compat.yaml b/tests/e2e/coverage_registry/llm_claude_code_compat.yaml deleted file mode 100644 index 6edf890f7ec..00000000000 --- a/tests/e2e/coverage_registry/llm_claude_code_compat.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Claude Code compatibility matrix: /v1/messages coverage across the five provider surfaces -# claude-code drives (anthropic direct, azure ai foundry, bedrock invoke, bedrock converse, -# vertex ai). Each row is one (feature x provider) cell in the matrix. The seven anthropic-direct -# rows already declared in llm_conversational.yaml are NOT duplicated here; the four other -# provider surfaces plus every feature not already listed for anthropic direct are declared below. -# -# Grammar: llm.messages....works -# route : anthropic | azure_foundry | bedrock_converse | bedrock_invoke | vertex -# capability : basic | tool_use | vision | thinking | prompt_cache_5m | prompt_cache_1h -# | structured_output | pdf_input | long_context_1m -# | thinking_with_tool_use | tool_search | count_tokens | web_search -# streaming : stream | nonstream - -# ---- basic / non-streaming ---- -- {id: llm.messages.azure_foundry.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic messaging over Azure AI Foundry Anthropic deployments"} -- {id: llm.messages.bedrock_converse.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic messaging over Bedrock Converse Anthropic"} -- {id: llm.messages.bedrock_invoke.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic messaging over Bedrock Invoke Anthropic"} -- {id: llm.messages.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic messaging over Vertex AI Anthropic"} - -# ---- basic / streaming ---- -- {id: llm.messages.azure_foundry.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: basic, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic streaming over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic streaming over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic streaming over Bedrock Invoke"} -- {id: llm.messages.vertex.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: basic, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Basic streaming over Vertex AI"} - -# ---- tool_use / non-streaming ---- -- {id: llm.messages.azure_foundry.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Tool use over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Tool use over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Tool use over Bedrock Invoke"} -- {id: llm.messages.vertex.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Tool use over Vertex AI"} - -# ---- tool_use / streaming ---- -- {id: llm.messages.azure_foundry.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Streaming tool use over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Streaming tool use over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Streaming tool use over Bedrock Invoke"} -- {id: llm.messages.vertex.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: tool_use, streaming: stream, assertions: [works], source: "claude_code compat matrix", rationale: "Streaming tool use over Vertex AI"} - -# ---- vision ---- -- {id: llm.messages.azure_foundry.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Vision over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Vision over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Vision over Bedrock Invoke"} -- {id: llm.messages.vertex.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: vision, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Vision over Vertex AI"} - -# ---- thinking ---- -- {id: llm.messages.azure_foundry.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Extended thinking over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Extended thinking over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Extended thinking over Bedrock Invoke"} -- {id: llm.messages.vertex.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: thinking, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Extended thinking over Vertex AI"} - -# ---- prompt_cache_5m ---- -- {id: llm.messages.azure_foundry.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "5m prompt cache over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "5m prompt cache over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "5m prompt cache over Bedrock Invoke"} -- {id: llm.messages.vertex.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "5m prompt cache over Vertex AI"} - -# ---- prompt_cache_1h ---- -- {id: llm.messages.anthropic.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1h prompt cache over Anthropic direct"} -- {id: llm.messages.azure_foundry.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1h prompt cache over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1h prompt cache over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1h prompt cache over Bedrock Invoke"} -- {id: llm.messages.vertex.prompt_cache_1h.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: prompt_cache_1h, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1h prompt cache over Vertex AI"} - -# ---- structured_output ---- -- {id: llm.messages.anthropic.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Structured outputs (--json-schema) over Anthropic direct"} -- {id: llm.messages.azure_foundry.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Structured outputs over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Structured outputs over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Structured outputs over Bedrock Invoke"} -- {id: llm.messages.vertex.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: structured_output, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Structured outputs over Vertex AI"} - -# ---- pdf_input ---- -- {id: llm.messages.anthropic.pdf_input.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: pdf_input, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "PDF document input over Anthropic direct"} -- {id: llm.messages.azure_foundry.pdf_input.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: pdf_input, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "PDF document input over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.pdf_input.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: pdf_input, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "PDF document input over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.pdf_input.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: pdf_input, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "PDF document input over Bedrock Invoke"} -- {id: llm.messages.vertex.pdf_input.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: pdf_input, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "PDF document input over Vertex AI"} - -# ---- long_context_1m ---- -- {id: llm.messages.anthropic.long_context_1m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: long_context_1m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1M context beta over Anthropic direct"} -- {id: llm.messages.azure_foundry.long_context_1m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: long_context_1m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1M context beta over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.long_context_1m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: long_context_1m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1M context beta over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.long_context_1m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: long_context_1m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1M context beta over Bedrock Invoke"} -- {id: llm.messages.vertex.long_context_1m.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: long_context_1m, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "1M context beta over Vertex AI"} - -# ---- thinking_with_tool_use ---- -- {id: llm.messages.anthropic.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Thinking + tool_use interleaved over Anthropic direct"} -- {id: llm.messages.azure_foundry.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Thinking + tool_use interleaved over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Thinking + tool_use interleaved over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Thinking + tool_use interleaved over Bedrock Invoke"} -- {id: llm.messages.vertex.thinking_with_tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: thinking_with_tool_use, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Thinking + tool_use interleaved over Vertex AI"} - -# ---- tool_search ---- -- {id: llm.messages.anthropic.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search_tool_regex_20251119 discovery tool over Anthropic direct"} -- {id: llm.messages.azure_foundry.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Bedrock Invoke"} -- {id: llm.messages.vertex.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: tool_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "tool_search discovery tool over Vertex AI"} - -# ---- count_tokens ---- -- {id: llm.messages.anthropic.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Anthropic direct"} -- {id: llm.messages.azure_foundry.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Bedrock Invoke"} -- {id: llm.messages.vertex.count_tokens.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: count_tokens, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "/v1/messages/count_tokens over Vertex AI"} - -# ---- web_search ---- -- {id: llm.messages.anthropic.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Web search server tool over Anthropic direct"} -- {id: llm.messages.azure_foundry.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: azure_foundry, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Web search server tool over Azure AI Foundry"} -- {id: llm.messages.bedrock_converse.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_converse, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Web search server tool over Bedrock Converse"} -- {id: llm.messages.bedrock_invoke.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: bedrock_invoke, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Web search server tool over Bedrock Invoke"} -- {id: llm.messages.vertex.web_search.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: vertex, capability: web_search, streaming: nonstream, assertions: [works], source: "claude_code compat matrix", rationale: "Web search server tool over Vertex AI"} diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml deleted file mode 100644 index be8a291c6fc..00000000000 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# LLM conversational endpoints (chat_completions, messages, responses). Grounded in proxy handlers + model_prices json. -- {id: llm.chat_completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Core endpoint/route/capability"} -- {id: llm.chat_completions.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Core streaming"} -- {id: llm.chat_completions.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "proxy_server.py:8455", rationale: "Cost logging regression catch"} -- {id: llm.chat_completions.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "OpenAI function_calling; high usage"} -- {id: llm.chat_completions.openai.tool_use.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: tool_use, streaming: stream, assertions: [works], source: "model_prices json", rationale: "Tool calls over streaming"} -- {id: llm.chat_completions.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "gpt-4o vision; high usage"} -- {id: llm.chat_completions.openai.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching cost optimization"} -- {id: llm.chat_completions.openai.service_tier.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: nonstream, assertions: [works], source: "OpenAI service_tier param", rationale: "OpenAI scale-tier request option is forwarded and echoed"} -- {id: llm.chat_completions.openai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "o-series reasoning; emerging"} -- {id: llm.chat_completions.openai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "response_schema extraction"} -- {id: llm.chat_completions.anthropic.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route translated to Anthropic"} -- {id: llm.chat_completions.anthropic.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming translation"} -- {id: llm.chat_completions.anthropic.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Claude tool_use; high usage"} -- {id: llm.chat_completions.anthropic.tool_use.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: tool_use, streaming: stream, assertions: [works], source: "model_prices json", rationale: "Streaming tool calls"} -- {id: llm.chat_completions.anthropic.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Claude vision; high usage"} -- {id: llm.chat_completions.anthropic.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Claude prompt caching"} -- {id: llm.chat_completions.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Claude extended thinking"} -- {id: llm.chat_completions.anthropic.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: anthropic, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Claude response_schema"} -- {id: llm.chat_completions.bedrock_converse.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Bedrock Converse unified"} -- {id: llm.chat_completions.bedrock_converse.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming over Converse"} -- {id: llm.chat_completions.bedrock_converse.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Converse function_calling; AWS adoption"} -- {id: llm.chat_completions.bedrock_converse.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Bedrock vision (Anthropic/Nova)"} -- {id: llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic-on-Bedrock caching"} -- {id: llm.chat_completions.bedrock_converse.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic thinking on Bedrock"} -- {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"} -- {id: llm.chat_completions.vertex.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming over Vertex"} -- {id: llm.chat_completions.vertex.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vertex Gemini function_calling"} -- {id: llm.chat_completions.vertex.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Gemini vision"} -- {id: llm.chat_completions.vertex.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: vertex, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vertex Gemini prompt caching"} -- {id: llm.chat_completions.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Azure OpenAI deployments"} -- {id: llm.chat_completions.azure_openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Azure OpenAI function_calling"} -- {id: llm.chat_completions.azure_foundry.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: azure_foundry, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Azure Foundry (azure_ai); newer, smoke"} -- {id: llm.messages.anthropic.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "anthropic_endpoints/endpoints.py:64", rationale: "Core endpoint; Anthropic Messages native"} -- {id: llm.messages.anthropic.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "anthropic_endpoints/endpoints.py:64", rationale: "Streaming Messages API"} -- {id: llm.messages.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "anthropic_endpoints/endpoints.py:64", rationale: "Cost logged on passthrough"} -- {id: llm.messages.anthropic.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Tool calls via Messages API"} -- {id: llm.messages.anthropic.tool_use.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: tool_use, streaming: stream, assertions: [works], source: "model_prices json", rationale: "Streaming tool calls"} -- {id: llm.messages.anthropic.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Messages API"} -- {id: llm.messages.anthropic.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching via Messages API"} -- {id: llm.messages.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Extended thinking via Messages API"} -- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven} -- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (#32831)", fail_before_fix: proven} -- {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"} -- {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"} -- {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"} -- {id: llm.responses.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Tool calls via Responses API"} -- {id: llm.responses.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Responses API"} -- {id: llm.responses.anthropic.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Anthropic translation (smoke)"} -- {id: llm.responses.anthropic.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Responses tool calls w/ Anthropic"} -- {id: llm.responses.bedrock_converse.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Bedrock Converse (smoke)"} -- {id: llm.responses.bedrock_converse.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: bedrock_converse, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Responses tool calls w/ Converse"} -- {id: llm.responses.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Vertex (smoke)"} -- {id: llm.responses.vertex.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Responses tool calls w/ Vertex"} -- {id: llm.responses.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Azure OpenAI (smoke)"} -- {id: llm.responses.azure_openai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Responses tool calls w/ Azure OpenAI"} diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml deleted file mode 100644 index b01b219476d..00000000000 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# LLM non-conversational endpoints. Grounded in litellm/proxy endpoints + llms/ handlers. -- {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"} -- {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"} -- {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"} -- {id: llm.embeddings.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/embed/embedding.py", rationale: "Bedrock Titan embeddings"} -- {id: llm.embeddings.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_embeddings/embedding_handler.py", rationale: "Vertex embeddings"} -- {id: llm.embeddings.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "llms/cohere/embed/handler.py", rationale: "Cohere embeddings"} -- {id: llm.embeddings.anthropic.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "llms/anthropic/chat/handler.py", rationale: "Anthropic vector API (verify support)"} -- {id: llm.batches.openai.create.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Core batch create"} -- {id: llm.batches.openai.retrieve.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Batch retrieve, id round-trip + status"} -- {id: llm.batches.openai.cancel.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Batch cancel"} -- {id: llm.batches.openai.list.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Batch list envelope"} -- {id: llm.batches.openai.file_lifecycle.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "File upload/retrieve/delete for batch flow"} -- {id: llm.batches.openai_encoded.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py", rationale: "Encoded scenario lifecycle"} -- {id: llm.batches.openai_unified.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py", rationale: "Unified/managed-id scenario"} -- {id: llm.batches.openai_model_param.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py", rationale: "Model-param scenario"} -- {id: llm.batches.openai_provider_fallback.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py", rationale: "Provider-fallback raw-id scenario"} -- {id: llm.batches.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Azure batches all scenarios"} -- {id: llm.batches.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Vertex batches"} -- {id: llm.batches.bedrock.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Bedrock batches (encoded/unified only)"} -- {id: llm.batches.openai.key_model_access_denied.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Key model restriction 403 on upload/create"} -- {id: llm.files.openai.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai_files_endpoints/files_endpoints.py:46", rationale: "File upload returns OpenAIFileObject"} -- {id: llm.files.openai.retrieve.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "files_endpoints.py", rationale: "File retrieve by id"} -- {id: llm.files.openai.delete.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "files_endpoints.py", rationale: "File delete returns deleted=true"} -- {id: llm.files.openai.list.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "files_endpoints.py", rationale: "File list paginated"} -- {id: llm.files.azure_openai.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:45", rationale: "Azure file upload managed backend"} -- {id: llm.files.vertex.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:52", rationale: "Vertex file upload to GCS"} -- {id: llm.files.bedrock.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:59", rationale: "Bedrock file upload to S3"} -- {id: llm.rerank.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_rerank_e2e.py:29", rationale: "Cohere rerank, top_n + relevance_score"} -- {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"} -- {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"} -- {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"} -- {id: llm.images_generations.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure DALL-E"} -- {id: llm.images_generations.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_ai/image_generation/image_generation_handler.py", rationale: "Vertex Imagen"} -- {id: llm.images_generations.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "bedrock/image_generation/image_handler.py", rationale: "Bedrock Titan Image"} -- {id: llm.images_generations.black_forest_labs.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "black_forest_labs/image_generation/handler.py", rationale: "BFL Flux via OpenAI-compat"} -- {id: llm.audio_speech.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_audio_speech_e2e.py:22", rationale: "OpenAI TTS binary audio"} -- {id: llm.audio_speech.openai.basic.stream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: openai, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:9043", rationale: "TTS streaming chunk generator"} -- {id: llm.audio_speech.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure TTS"} -- {id: llm.audio_speech.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_ai/text_to_speech/text_to_speech_handler.py", rationale: "Vertex TTS"} -- {id: llm.audio_transcriptions.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "OpenAI Whisper"} -- {id: llm.audio_transcriptions.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure STT"} -- {id: llm.audio_transcriptions.soniox.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "soniox/audio_transcription/handler.py", rationale: "Soniox via OpenAI-compat (smoke)"} -- {id: llm.audio_transcriptions.nvidia_riva.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "nvidia_riva/audio_transcription/handler.py", rationale: "NVIDIA Riva (smoke)"} -- {id: llm.moderations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: moderations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py", rationale: "OpenAI moderations (only provider)"} diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml deleted file mode 100644 index 0f703632805..00000000000 --- a/tests/e2e/coverage_registry/logging.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Logging integration delivery (behavior features). Grounded in litellm/integrations/. -- {id: logging.s3.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/s3_v2.py", rationale: "Primary audit trail; batch flush no-drop"} -- {id: logging.s3.failure.writes_object, module: logging, tier: P0, event: failure, assertions: [writes_object], exercised_on: [chat_completions, messages], source: "integrations/s3_v2.py", rationale: "Failed calls persisted for compliance"} -- {id: logging.gcs_bucket.success.writes_object, module: logging, tier: P0, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages, embeddings], source: "integrations/gcs_bucket/gcs_bucket.py", rationale: "GCS parallel to S3"} -- {id: logging.datadog.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/datadog/datadog.py", rationale: "Powers dashboards/alerts; cardinality regressions common"} -- {id: logging.datadog.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/datadog/datadog.py", rationale: "Streaming aggregates usage after the last chunk; delivery and cost must survive that path"} -- {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"} -- {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"} -- {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"} -- {id: logging.otel.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/logger.py", rationale: "Streaming closes the LLM span from the stream path; historically prone to duplicate/orphaned spans"} -- {id: logging.otel.stream.records_ttft, module: logging, tier: P1, event: stream, assertions: [records_ttft], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/mappers/genai.py", rationale: "TTFT is the streaming latency SLI; a zero or span-length value silently corrupts dashboards"} -- {id: logging.otel.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions, messages], source: "integrations/otel/logger.py", rationale: "Error spans for observability continuity"} -- {id: logging.braintrust.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/braintrust_logging.py", rationale: "Evals platform spend"} -- {id: logging.langsmith.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/langsmith.py", rationale: "LangChain ecosystem"} -- {id: logging.arize.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, embeddings], source: "integrations/arize/arize.py", rationale: "ML-ops observability"} -- {id: logging.mlflow.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: "integrations/mlflow.py", rationale: "Experiment tracking cost/run"} -- {id: logging.opik.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: "integrations/opik/opik.py", rationale: "Eval platform spend/case"} -- {id: logging.openmeter.success.exports_metric, module: logging, tier: P1, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/openmeter.py", rationale: "Usage metering for billing"} -- {id: logging.literal_ai.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/literal_ai.py", rationale: "Tracing platform spend"} -- {id: logging.posthog.success.exports_metric, module: logging, tier: P1, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages], source: "integrations/posthog.py", rationale: "Product analytics batching"} -- {id: logging.azure_storage.success.writes_object, module: logging, tier: P1, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages], source: "integrations/azure_storage/azure_storage.py", rationale: "Azure blob for enterprise"} -- {id: logging.cloudzero.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: "integrations/cloudzero/cloudzero.py", rationale: "Cost ops correlation"} -- {id: logging.focus.success.writes_object, module: logging, tier: P1, event: success, assertions: [writes_object], exercised_on: [chat_completions, messages], source: "integrations/focus/focus_logger.py", rationale: "Cost mgmt multi-destination export"} -- {id: logging.niche_integrations.success.logs_spend, module: logging, tier: P2, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE cohort: athina/galileo/deepeval/langtrace/weave/lunary/humanloop/traceloop/helicone/argilla/newrelic/sqs/supabase/dynamodb/agentops/lago/etc"} -- {id: logging.niche_integrations.failure.logs_spend, module: logging, tier: P2, event: failure, assertions: [logs_spend], exercised_on: [chat_completions], source: grammar, rationale: "SMOKE niche failure path"} diff --git a/tests/e2e/coverage_registry/mcp.yaml b/tests/e2e/coverage_registry/mcp.yaml deleted file mode 100644 index d477b257cb0..00000000000 --- a/tests/e2e/coverage_registry/mcp.yaml +++ /dev/null @@ -1,113 +0,0 @@ -# MCP module. Grounded in litellm/proxy/_experimental/mcp_server/. See tests/e2e/CLAUDE.md for the grammar. -- id: mcp.list_tools.api_key.succeeds - module: mcp - tier: P0 - operation: list_tools - auth_family: api_key - assertions: [succeeds] - source: "server.py:637" - rationale: Core operation; most common auth path; high usage -- id: mcp.list_tools.api_key.denied_without_permission - module: mcp - tier: P0 - operation: list_tools - auth_family: api_key - assertions: [denied_without_permission] - source: "mcp_server_manager.py:1409" - rationale: Permission guard is high blast-radius; multi-tenant safety -- id: mcp.call_tool.api_key.succeeds - module: mcp - tier: P0 - operation: call_tool - auth_family: api_key - assertions: [succeeds] - source: "server.py:849" - rationale: Primary operation; customer-critical; high usage -- id: mcp.call_tool.api_key.denied_without_permission - module: mcp - tier: P0 - operation: call_tool - auth_family: api_key - assertions: [denied_without_permission] - source: "rest_endpoints.py:305-386" - rationale: Tool-level permission guard; multi-tenant safety -- id: mcp.list_tools.bearer.succeeds - module: mcp - tier: P1 - operation: list_tools - auth_family: bearer - assertions: [succeeds] - source: "server.py:662" - rationale: OAuth/bearer token flow; upstream delegation -- id: mcp.call_tool.bearer.succeeds - module: mcp - tier: P1 - operation: call_tool - auth_family: bearer - assertions: [succeeds] - source: "server.py:886" - rationale: Bearer token forwarding for tool invocation -- id: mcp.list_tools.oauth.succeeds - module: mcp - tier: P1 - operation: list_tools - auth_family: oauth - assertions: [succeeds] - source: "rest_endpoints.py:138-188" - rationale: Interactive OAuth2 flow; live token management -- id: mcp.call_tool.oauth.succeeds - module: mcp - tier: P1 - operation: call_tool - auth_family: oauth - assertions: [succeeds] - source: "db.py user_oauth_credential lookup" - rationale: OAuth2 token passthrough; per-user credential storage -- id: mcp.list_tools.none.succeeds - module: mcp - tier: P1 - operation: list_tools - auth_family: none - assertions: [succeeds] - source: "mcp_server_manager.py:1485-1492" - rationale: Public/anonymous servers; delegate_auth_to_upstream -- id: mcp.call_tool.none.succeeds - module: mcp - tier: P1 - operation: call_tool - auth_family: none - assertions: [succeeds] - source: "rest_endpoints.py:305-334" - rationale: No upstream auth required; demo servers -- id: mcp.get_prompt.api_key.succeeds - module: mcp - tier: P1 - operation: get_prompt - auth_family: api_key - assertions: [succeeds] - source: "server.py:1042" - rationale: Prompt op; same auth stack as tools -- id: mcp.read_resource.api_key.succeeds - module: mcp - tier: P1 - operation: read_resource - auth_family: api_key - assertions: [succeeds] - source: "server.py:1177" - rationale: Resource op; same permission model as tools -- id: mcp.list_prompts.api_key.succeeds - module: mcp - tier: P2 - operation: list_prompts - auth_family: api_key - assertions: [succeeds] - source: "server.py:993" - rationale: Smoke-level; same auth stack as list_tools -- id: mcp.list_resources.api_key.succeeds - module: mcp - tier: P2 - operation: list_resources - auth_family: api_key - assertions: [succeeds] - source: "server.py:1089" - rationale: Smoke; rarely used; same auth model as tools diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml deleted file mode 100644 index 4fbaac0a205..00000000000 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# Management/UI endpoint features. Grounded in litellm/proxy/management_endpoints/. -- {id: mgmt.key.generate.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "key_management_endpoints.py:1444", rationale: "API key survives DB roundtrip"} -- {id: mgmt.key.generate.admin_only, module: mgmt, tier: P0, surface: api, assertions: [admin_only], source: "key_management_endpoints.py:1444", rationale: "Only master/team-admin creates keys"} -- {id: mgmt.key.generate.happy_path, module: mgmt, tier: P0, surface: ui, assertions: [happy_path], source: "ui_sso.py:420", rationale: "SSO-driven key gen (UI path)"} -- {id: mgmt.key.update.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "key_management_endpoints.py:2462", rationale: "Budget/model changes persist"} -- {id: mgmt.key.update.admin_only, module: mgmt, tier: P0, surface: api, assertions: [admin_only], source: "key_management_endpoints.py:2462", rationale: "Non-admin cannot escalate perms"} -- {id: mgmt.key.update.happy_path, module: mgmt, tier: P1, surface: ui, assertions: [happy_path], source: "key_management_endpoints.py:2462", rationale: "Key edit through the dashboard"} -- {id: mgmt.key.delete.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "key_management_endpoints.py:3122", rationale: "Deletion revokes future calls"} -- {id: mgmt.key.delete.admin_only, module: mgmt, tier: P0, surface: api, assertions: [admin_only], source: "key_management_endpoints.py:3122", rationale: "Non-owner cannot delete"} -- {id: mgmt.key.info.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "key_management_endpoints.py:3380", rationale: "Info reflects all writes"} -- {id: mgmt.team.new.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "team_endpoints.py:897", rationale: "team_id/alias/budgets stored"} -- {id: mgmt.team.new.admin_only, module: mgmt, tier: P0, surface: api, assertions: [admin_only], source: "team_endpoints.py:897", rationale: "Only org-admin/master creates teams"} -- {id: mgmt.team.member_add.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "team_endpoints.py:2424", rationale: "Membership + per-member budget persist"} -- {id: mgmt.team.member_add.member_forbidden, module: mgmt, tier: P0, surface: api, assertions: [member_forbidden], source: "team_endpoints.py:2424", rationale: "Non-admin forbidden to add"} -- {id: mgmt.team.member_delete.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "team_endpoints.py:2800", rationale: "Removal revokes team key access"} -- {id: mgmt.team.member_delete.member_forbidden, module: mgmt, tier: P0, surface: api, assertions: [member_forbidden], source: "team_endpoints.py:2800", rationale: "Non-admin forbidden to remove"} -- {id: mgmt.budget.new.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "budget_management_endpoints.py:40", rationale: "max/soft/reset windows persist"} -- {id: mgmt.budget.new.admin_only, module: mgmt, tier: P0, surface: api, assertions: [admin_only], source: "budget_management_endpoints.py:40", rationale: "Requires master/admin"} -- {id: mgmt.model.add.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "model_management_endpoints.py:1201", rationale: "Registration persists for routing"} -- {id: mgmt.model.add.admin_only, module: mgmt, tier: P0, surface: api, assertions: [admin_only], source: "model_management_endpoints.py:1201", rationale: "Non-admin cannot inject model config"} -- {id: mgmt.user.new.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "internal_user_endpoints.py:360", rationale: "User creation full cycle"} -- {id: mgmt.key.list.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:5119", rationale: "Key inventory pagination"} -- {id: mgmt.key.block.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "key_management_endpoints.py:5849", rationale: "Blocked stays blocked on restart"} -- {id: mgmt.key.unblock.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "key_management_endpoints.py:5960", rationale: "Unblock restores access"} -- {id: mgmt.key.regenerate.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:6071", rationale: "Rotation: new works, old invalid"} -- {id: mgmt.key.health.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4292", rationale: "Key health endpoint"} -- {id: mgmt.key.bulk_update.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:2677", rationale: "Batch key updates"} -- {id: mgmt.team.update.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "team_endpoints.py:1582", rationale: "Metadata/budget updates persist"} -- {id: mgmt.team.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "team_endpoints.py:1750", rationale: "Deletion prevents key access"} -- {id: mgmt.team.block.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "team_endpoints.py", rationale: "Block suspends all members"} -- {id: mgmt.team.info.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "team_endpoints.py:2244", rationale: "Metadata+members+budgets"} -- {id: mgmt.team.list.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "team_endpoints.py:3645", rationale: "Pagination/filtering"} -- {id: mgmt.team.member_update.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "team_endpoints.py:2768", rationale: "Member budget/role updates persist"} -- {id: mgmt.user.update.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "internal_user_endpoints.py:555", rationale: "Metadata/perm updates persist"} -- {id: mgmt.user.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "internal_user_endpoints.py:640", rationale: "Deletion revokes keys+teams"} -- {id: mgmt.user.list.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "internal_user_endpoints.py:475", rationale: "Admin view all users"} -- {id: mgmt.user.info.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "internal_user_endpoints.py:440", rationale: "Roles/perms/team membership"} -- {id: mgmt.organization.new.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "organization_endpoints.py:403", rationale: "Org for multi-tenant isolation"} -- {id: mgmt.organization.update.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "organization_endpoints.py:545", rationale: "Org metadata updates persist"} -- {id: mgmt.organization.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "organization_endpoints.py:710", rationale: "Cascades to teams/keys"} -- {id: mgmt.organization.member_add.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "organization_endpoints.py:835", rationale: "Org member onboarding"} -- {id: mgmt.customer.new.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "customer_endpoints.py:372", rationale: "End-user for spend tracking"} -- {id: mgmt.customer.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "customer_endpoints.py:480", rationale: "Removes from spend tracking"} -- {id: mgmt.end_user.new.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "customer_endpoints.py:730", rationale: "End-user create (synonym)"} -- {id: mgmt.tag.new.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "tag_management_endpoints.py:160", rationale: "Tag for spend categorization"} -- {id: mgmt.tag.list.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "tag_management_endpoints.py:315", rationale: "Tag enumeration"} -- {id: mgmt.tag.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "tag_management_endpoints.py:390", rationale: "Stops future tagging"} -- {id: mgmt.model.update.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "model_management_endpoints.py:1358", rationale: "Pricing/concurrency persist"} -- {id: mgmt.model.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "model_management_endpoints.py:1045", rationale: "Removes from registry"} -- {id: mgmt.model.block.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "model_management_endpoints.py", rationale: "Blocked model stays blocked"} -- {id: mgmt.access_group.new.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "model_access_group_management_endpoints.py:450", rationale: "Model permissioning group"} -- {id: mgmt.access_group.info.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "model_access_group_management_endpoints.py:600", rationale: "Access group membership query"} -- {id: mgmt.mcp_server.register.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "mcp_management_endpoints.py:880", rationale: "MCP server registration"} -- {id: mgmt.mcp_server.approve.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1200", rationale: "Admin approval persists"} -- {id: mgmt.budget.update.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "budget_management_endpoints.py:155", rationale: "Limit changes apply"} -- {id: mgmt.budget.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "budget_management_endpoints.py:280", rationale: "Clears limits"} -- {id: mgmt.budget.list.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "budget_management_endpoints.py:215", rationale: "Budget enumeration"} -- {id: mgmt.callback.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "callback_management_endpoints.py", rationale: "Callback config (smoke)"} -- {id: mgmt.cache_settings.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "cache_settings_endpoints.py", rationale: "Cache config (smoke)"} -- {id: mgmt.cost_tracking.estimate.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "cost_tracking_settings.py", rationale: "Cost estimate (smoke)"} -- {id: mgmt.router_settings.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "router_settings_endpoints.py", rationale: "Router config (smoke)"} -- {id: mgmt.jwt_key_mapping.new.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "jwt_key_mapping_endpoints.py", rationale: "JWT->key mapping (smoke)"} -- {id: mgmt.compliance.gdpr.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "compliance_endpoints.py", rationale: "GDPR ops (smoke)"} -- {id: mgmt.tool_management.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "tool_management_endpoints.py", rationale: "Tool inventory (smoke)"} -- {id: mgmt.fallback_management.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "fallback_management_endpoints.py", rationale: "Fallback config (smoke)"} -- {id: mgmt.config_override.hashicorp_vault.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "config_override_endpoints.py", rationale: "Vault integration (smoke)"} -- {id: mgmt.workflow.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "workflow_management_endpoints.py", rationale: "Workflow tracking (smoke)"} -- {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml deleted file mode 100644 index c2efecec677..00000000000 --- a/tests/e2e/coverage_registry/other.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Other (holding pen). Grounded in litellm/proxy/auth/ + health_endpoints/ + proxy_server.py. -# PROMOTION NOTE: the auth cluster (~14 cells) is a candidate to promote to its own module once stable. -- {id: other.auth.master_key.valid_allows, module: other, tier: P0, area: auth, assertions: [valid_allows], source: "user_api_key_auth.py:1569-1588", rationale: "Master key authenticates; timing-safe compare"} -- {id: other.auth.master_key.invalid_denied, module: other, tier: P0, area: auth, assertions: [invalid_denied], source: "user_api_key_auth.py:1580", rationale: "Invalid master key rejected"} -- {id: other.auth.jwt.valid_token_allows, module: other, tier: P0, area: auth, assertions: [valid_token_allows], source: "handle_jwt.py:77-150", rationale: "Valid JWT with correct issuer + claims grants access"} -- {id: other.auth.jwt.expired_denied, module: other, tier: P0, area: auth, assertions: [expired_denied], source: "handle_jwt.py:125-135", rationale: "Expired JWT rejected even with valid signature"} -- {id: other.auth.jwt.invalid_signature_denied, module: other, tier: P0, area: auth, assertions: [invalid_signature_denied], source: "handle_jwt.py:145-150", rationale: "Bad/missing signature fails verification"} -- {id: other.auth.virtual_key.route_permission_enforced, module: other, tier: P0, area: auth, assertions: [route_permission_enforced], source: "route_checks.py:89-151", rationale: "allowed_routes whitelist denies disallowed routes"} -- {id: other.auth.virtual_key.route_group_allowed, module: other, tier: P1, area: auth, assertions: [route_group_allowed], source: "route_checks.py:106-128", rationale: "allowed_routes=[llm_api_routes] grants all LLM endpoints"} -- {id: other.auth.passthrough.model_allowlist_enforced, module: other, tier: P1, area: auth, assertions: [model_allowlist_enforced], source: "route_checks.py:135-151", rationale: "Passthrough enforces per-key model allow-lists"} -- {id: other.auth.oauth2.token_valid_allows, module: other, tier: P1, area: auth, assertions: [token_valid_allows], source: "oauth2_check.py:15-73", rationale: "OAuth2 introspection grants active token"} -- {id: other.auth.oauth2.token_invalid_denied, module: other, tier: P1, area: auth, assertions: [token_invalid_denied], source: "oauth2_check.py:37-73", rationale: "Expired/inactive OAuth2 token denied"} -- {id: other.auth.ip_allowlist.internal_ip_allows, module: other, tier: P1, area: auth, assertions: [internal_ip_allows], source: "ip_address_utils.py:54-76", rationale: "Internal CIDR bypasses public-API restriction"} -- {id: other.auth.ip_allowlist.external_ip_denied_to_private, module: other, tier: P1, area: auth, assertions: [external_ip_denied_to_private], source: "ip_address_utils.py:54-76", rationale: "External IP cannot reach internal-only resources"} -- {id: other.lifecycle.readiness.public_probe, module: other, tier: P0, area: lifecycle, assertions: [public_probe], source: "_health_endpoints.py:1551-1570", rationale: "Unauthenticated /health/readiness safe for LBs"} -- {id: other.lifecycle.readiness.reports_db_status, module: other, tier: P0, area: lifecycle, assertions: [reports_db_status], source: "_health_endpoints.py:1551-1570", rationale: "readiness distinguishes healthy vs DB-unreachable"} -- {id: other.lifecycle.readiness.shutting_down_returns_503, module: other, tier: P0, area: lifecycle, assertions: [shutting_down_returns_503], source: "_health_endpoints.py:1554-1556", rationale: "Graceful shutdown drains LB via 503"} -- {id: other.lifecycle.readiness_details.authenticated_diagnostics, module: other, tier: P1, area: lifecycle, assertions: [authenticated_diagnostics], source: "_health_endpoints.py:1574-1584", rationale: "Auth'd details expose cache/callback status"} -- {id: other.lifecycle.liveness.ping, module: other, tier: P1, area: lifecycle, assertions: [ping], source: "_health_endpoints.py:134-155", rationale: "Liveness confirms server responding"} -- {id: other.lifecycle.startup.config_loads, module: other, tier: P0, area: lifecycle, assertions: [config_loads], source: "proxy_server.py:4020-4100", rationale: "Startup loads YAML, resolves env, persists to DB"} -- {id: other.lifecycle.startup.env_vars_resolved, module: other, tier: P1, area: lifecycle, assertions: [env_vars_resolved], source: "proxy_server.py:3984-4010", rationale: "os.environ/ refs resolved at startup"} -- {id: other.lifecycle.background_health_check.interval_configurable, module: other, tier: P1, area: lifecycle, assertions: [interval_configurable], source: "proxy_server.py:3245-3310", rationale: "Background checks run at configurable interval"} -- {id: other.config.runtime_update.applies_at_runtime, module: other, tier: P0, area: config, assertions: [applies_at_runtime], source: "proxy_server.py:14014-14060", rationale: "/config/update persists to DB + invalidates cache"} -- {id: other.config.general_settings.alert_webhook_side_effect, module: other, tier: P1, area: config, assertions: [alert_webhook_side_effect], source: "proxy_server.py:14215", rationale: "alert_to_webhook_url auto-enables slack alerting"} -- {id: other.config.secret_resolution.kms_integration, module: other, tier: P1, area: config, assertions: [kms_integration], source: "proxy_server.py:3984-4010", rationale: "Resolves secrets from Vault/KMS at startup"} -- {id: other.config.overrides.audit_logged, module: other, tier: P1, area: config, assertions: [audit_logged], source: "config_override_endpoints.py:67-100", rationale: "Config override mutations audit-logged, values redacted"} -- {id: other.key_mgmt.regenerate.grace_period_honored, module: other, tier: P1, area: auth, assertions: [grace_period_honored], source: "key_management_endpoints.py:4503-4560", rationale: "Old key valid during grace_period then revoked"} -- {id: other.key_mgmt.spend_reset.resets_to_value, module: other, tier: P1, area: auth, assertions: [resets_to_value], source: "key_management_endpoints.py:4841", rationale: "reset_spend resets accumulated spend"} diff --git a/tests/e2e/coverage_registry/overlay.py b/tests/e2e/coverage_registry/overlay.py new file mode 100644 index 00000000000..2edb1f5218b --- /dev/null +++ b/tests/e2e/coverage_registry/overlay.py @@ -0,0 +1,24 @@ +"""Load the curated human overlay: the per-cell fields a person decides, keyed by id. + +The overlay never defines whether a behavior exists; the generated product surface does +that. It only annotates a cell with judgement (tier, source, rationale, fail-before-fix, +support) and enumerates the ids generation does not yet produce. Ordinary test PRs do not +touch it. +""" + +from __future__ import annotations + +from pathlib import Path + +import yaml +from pydantic import TypeAdapter + +from .schema import OverlayRow + +OVERLAY_PATH = Path(__file__).resolve().parent / "overlay.yaml" + +_OVERLAY_ADAPTER: TypeAdapter[dict[str, OverlayRow]] = TypeAdapter(dict[str, OverlayRow]) + + +def load_overlay(path: Path = OVERLAY_PATH) -> dict[str, OverlayRow]: + return _OVERLAY_ADAPTER.validate_python(yaml.safe_load(path.read_text()) or {}) diff --git a/tests/e2e/coverage_registry/overlay.yaml b/tests/e2e/coverage_registry/overlay.yaml new file mode 100644 index 00000000000..af619620d7a --- /dev/null +++ b/tests/e2e/coverage_registry/overlay.yaml @@ -0,0 +1,408 @@ +# Curated human overlay for the coverage denominator, keyed by cell id. +# The denominator (the set of cells) is generated in product_surface.py; the ids that +# generation does not yet produce are also enumerated here. This file only carries the +# human judgement for a cell: its tier, where it came from (source), why it matters +# (rationale), whether a fail-before-fix was proven, and whether the product supports it. +# Ordinary test PRs never edit this file; it is owner-gated. Adding coverage means adding +# a @pytest.mark.covers marker to a test, not a row here. + + +# ---- guardrail ---- +guardrail.presidio.pre_call.masks: {tier: P0, source: guardrail_hooks/presidio.py, rationale: PII masking pre-call; data-leak blast radius} +guardrail.presidio.post_call.masks: {tier: P0, source: guardrail_hooks/presidio.py, rationale: Mask PII in model output} +guardrail.presidio.logging_only.masks: {tier: P0, source: guardrail_hooks/presidio.py, rationale: Redact in logs without blocking} +guardrail.bedrock.pre_call.blocks: {tier: P0, source: guardrail_hooks/bedrock_guardrails.py, rationale: AWS content guardrail blocks harmful input} +guardrail.bedrock.during.blocks: {tier: P0, source: guardrail_hooks/bedrock_guardrails.py, rationale: During-call moderation for streaming} +guardrail.bedrock.post_call.blocks: {tier: P0, source: guardrail_hooks/bedrock_guardrails.py, rationale: Block harmful output} +guardrail.lakera.pre_call.blocks: {tier: P0, source: guardrail_hooks/lakera_ai_v2.py, rationale: Prompt-injection block pre-execution} +guardrail.lakera.post_call.blocks: {tier: P0, source: guardrail_hooks/lakera_ai_v2.py, rationale: Post-call injection on multi-turn chains} +guardrail.openai_moderations.pre_call.blocks: {tier: P0, source: guardrail_hooks/openai/moderations.py, rationale: Content policy for regulated industries} +guardrail.aim.pre_call.blocks: {tier: P1, source: guardrail_hooks/aim/aim.py, rationale: Security guardrail malicious-input} +guardrail.aim.post_call.blocks: {tier: P1, source: guardrail_hooks/aim/aim.py, rationale: Output security check} +guardrail.ibm_guardrails.pre_call.blocks: {tier: P1, source: guardrail_hooks/ibm_guardrails/ibm_detector.py, rationale: Enterprise multi-policy} +guardrail.ibm_guardrails.post_call.blocks: {tier: P1, source: guardrail_hooks/ibm_guardrails/ibm_detector.py, rationale: Output policy validation} +guardrail.semantic_guard.pre_call.blocks: {tier: P1, source: guardrail_hooks/semantic_guard, rationale: Semantic policy compliance} +guardrail.block_code_execution.pre_call.blocks: {tier: P1, source: guardrail_hooks/block_code_execution, rationale: Code-injection prevention} +guardrail.tool_permission.pre_call.allows: {tier: P1, source: guardrail_hooks/tool_permission.py, rationale: Grant allowed tools} +guardrail.tool_permission.pre_call.blocks: {tier: P1, source: guardrail_hooks/tool_permission.py, rationale: Block unauthorized tools} +guardrail.microsoft_purview.pre_call.blocks: {tier: P1, source: guardrail_hooks/microsoft_purview/purview_dlp.py, rationale: DLP sensitive-data disclosure} +guardrail.headroom.pre_call.blocks: {tier: P1, source: guardrail_hooks/headroom/headroom.py, rationale: Anomaly detection threshold} +guardrail.generic_guardrail_api.pre_call.blocks: {tier: P1, source: guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py, rationale: Vendor-agnostic custom API} +guardrail.pangea.pre_call.blocks: {tier: P1, source: guardrail_hooks/pangea/pangea.py, rationale: API security + DLP} +guardrail.niche_providers.pre_call.blocks: {tier: P2, source: grammar, rationale: 'SMOKE cohort: lasso/hiddenlayer/model_armor/qualifire/guardrails_ai/cato/cisco/akto/prompt_security/promptguard/zscaler/vigil/etc'} +guardrail.niche_providers.post_call.blocks: {tier: P2, source: grammar, rationale: SMOKE niche output filtering} +guardrail.niche_providers.pre_call.allows: {tier: P2, source: grammar, rationale: SMOKE niche allow-path passthrough} +guardrail.tool_policy.pre_call.blocks: {tier: P2, source: guardrail_hooks/tool_policy/tool_policy_guardrail.py, rationale: Tool-use policy enforcement} +guardrail.mcp_security.pre_call.blocks: {tier: P2, source: guardrail_hooks/mcp_security, rationale: MCP protocol security} +guardrail.llm_as_a_judge.pre_call.blocks: {tier: P2, source: guardrail_hooks/llm_as_a_judge, rationale: LLM-based judgment guardrail} + +# ---- llm ---- +llm.messages.azure_foundry.basic.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Basic messaging over Azure AI Foundry Anthropic deployments} +llm.messages.bedrock_converse.basic.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Basic messaging over Bedrock Converse Anthropic} +llm.messages.bedrock_invoke.basic.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Basic messaging over Bedrock Invoke Anthropic} +llm.messages.vertex.basic.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Basic messaging over Vertex AI Anthropic} +llm.messages.azure_foundry.basic.stream.works: {tier: P1, source: claude_code compat matrix, rationale: Basic streaming over Azure AI Foundry} +llm.messages.bedrock_converse.basic.stream.works: {tier: P1, source: claude_code compat matrix, rationale: Basic streaming over Bedrock Converse} +llm.messages.bedrock_invoke.basic.stream.works: {tier: P1, source: claude_code compat matrix, rationale: Basic streaming over Bedrock Invoke} +llm.messages.vertex.basic.stream.works: {tier: P1, source: claude_code compat matrix, rationale: Basic streaming over Vertex AI} +llm.messages.azure_foundry.tool_use.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Tool use over Azure AI Foundry} +llm.messages.bedrock_converse.tool_use.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Tool use over Bedrock Converse} +llm.messages.bedrock_invoke.tool_use.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Tool use over Bedrock Invoke} +llm.messages.vertex.tool_use.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Tool use over Vertex AI} +llm.messages.azure_foundry.tool_use.stream.works: {tier: P1, source: claude_code compat matrix, rationale: Streaming tool use over Azure AI Foundry} +llm.messages.bedrock_converse.tool_use.stream.works: {tier: P1, source: claude_code compat matrix, rationale: Streaming tool use over Bedrock Converse} +llm.messages.bedrock_invoke.tool_use.stream.works: {tier: P1, source: claude_code compat matrix, rationale: Streaming tool use over Bedrock Invoke} +llm.messages.vertex.tool_use.stream.works: {tier: P1, source: claude_code compat matrix, rationale: Streaming tool use over Vertex AI} +llm.messages.azure_foundry.vision.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Vision over Azure AI Foundry} +llm.messages.bedrock_converse.vision.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Vision over Bedrock Converse} +llm.messages.bedrock_invoke.vision.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Vision over Bedrock Invoke} +llm.messages.vertex.vision.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Vision over Vertex AI} +llm.messages.azure_foundry.thinking.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Extended thinking over Azure AI Foundry} +llm.messages.bedrock_converse.thinking.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Extended thinking over Bedrock Converse} +llm.messages.bedrock_invoke.thinking.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Extended thinking over Bedrock Invoke} +llm.messages.vertex.thinking.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Extended thinking over Vertex AI} +llm.messages.azure_foundry.prompt_cache_5m.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 5m prompt cache over Azure AI Foundry} +llm.messages.bedrock_converse.prompt_cache_5m.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 5m prompt cache over Bedrock Converse} +llm.messages.bedrock_invoke.prompt_cache_5m.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 5m prompt cache over Bedrock Invoke} +llm.messages.vertex.prompt_cache_5m.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 5m prompt cache over Vertex AI} +llm.messages.anthropic.prompt_cache_1h.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 1h prompt cache over Anthropic direct} +llm.messages.azure_foundry.prompt_cache_1h.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 1h prompt cache over Azure AI Foundry} +llm.messages.bedrock_converse.prompt_cache_1h.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 1h prompt cache over Bedrock Converse} +llm.messages.bedrock_invoke.prompt_cache_1h.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 1h prompt cache over Bedrock Invoke} +llm.messages.vertex.prompt_cache_1h.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 1h prompt cache over Vertex AI} +llm.messages.anthropic.structured_output.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Structured outputs (--json-schema) over Anthropic direct} +llm.messages.azure_foundry.structured_output.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Structured outputs over Azure AI Foundry} +llm.messages.bedrock_converse.structured_output.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Structured outputs over Bedrock Converse} +llm.messages.bedrock_invoke.structured_output.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Structured outputs over Bedrock Invoke} +llm.messages.vertex.structured_output.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Structured outputs over Vertex AI} +llm.messages.anthropic.pdf_input.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: PDF document input over Anthropic direct} +llm.messages.azure_foundry.pdf_input.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: PDF document input over Azure AI Foundry} +llm.messages.bedrock_converse.pdf_input.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: PDF document input over Bedrock Converse} +llm.messages.bedrock_invoke.pdf_input.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: PDF document input over Bedrock Invoke} +llm.messages.vertex.pdf_input.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: PDF document input over Vertex AI} +llm.messages.anthropic.long_context_1m.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 1M context beta over Anthropic direct} +llm.messages.azure_foundry.long_context_1m.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 1M context beta over Azure AI Foundry} +llm.messages.bedrock_converse.long_context_1m.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 1M context beta over Bedrock Converse} +llm.messages.bedrock_invoke.long_context_1m.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 1M context beta over Bedrock Invoke} +llm.messages.vertex.long_context_1m.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: 1M context beta over Vertex AI} +llm.messages.anthropic.thinking_with_tool_use.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Thinking + tool_use interleaved over Anthropic direct} +llm.messages.azure_foundry.thinking_with_tool_use.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Thinking + tool_use interleaved over Azure AI Foundry} +llm.messages.bedrock_converse.thinking_with_tool_use.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Thinking + tool_use interleaved over Bedrock Converse} +llm.messages.bedrock_invoke.thinking_with_tool_use.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Thinking + tool_use interleaved over Bedrock Invoke} +llm.messages.vertex.thinking_with_tool_use.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Thinking + tool_use interleaved over Vertex AI} +llm.messages.anthropic.tool_search.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: tool_search_tool_regex_20251119 discovery tool over Anthropic direct} +llm.messages.azure_foundry.tool_search.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: tool_search discovery tool over Azure AI Foundry} +llm.messages.bedrock_converse.tool_search.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: tool_search discovery tool over Bedrock Converse} +llm.messages.bedrock_invoke.tool_search.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: tool_search discovery tool over Bedrock Invoke} +llm.messages.vertex.tool_search.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: tool_search discovery tool over Vertex AI} +llm.messages.anthropic.count_tokens.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: /v1/messages/count_tokens over Anthropic direct} +llm.messages.azure_foundry.count_tokens.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: /v1/messages/count_tokens over Azure AI Foundry} +llm.messages.bedrock_converse.count_tokens.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: /v1/messages/count_tokens over Bedrock Converse} +llm.messages.bedrock_invoke.count_tokens.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: /v1/messages/count_tokens over Bedrock Invoke} +llm.messages.vertex.count_tokens.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: /v1/messages/count_tokens over Vertex AI} +llm.messages.anthropic.web_search.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Web search server tool over Anthropic direct} +llm.messages.azure_foundry.web_search.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Web search server tool over Azure AI Foundry} +llm.messages.bedrock_converse.web_search.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Web search server tool over Bedrock Converse} +llm.messages.bedrock_invoke.web_search.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Web search server tool over Bedrock Invoke} +llm.messages.vertex.web_search.nonstream.works: {tier: P1, source: claude_code compat matrix, rationale: Web search server tool over Vertex AI} +llm.chat_completions.openai.basic.nonstream.works: {tier: P0, source: 'proxy_server.py:8455', rationale: Core endpoint/route/capability} +llm.chat_completions.openai.basic.stream.works: {tier: P0, source: 'proxy_server.py:8455', rationale: Core streaming} +llm.chat_completions.openai.basic.nonstream.cost_logged: {tier: P0, source: 'proxy_server.py:8455', rationale: Cost logging regression catch} +llm.chat_completions.openai.tool_use.nonstream.works: {tier: P0, source: model_prices json, rationale: OpenAI function_calling; high usage} +llm.chat_completions.openai.tool_use.stream.works: {tier: P0, source: model_prices json, rationale: Tool calls over streaming} +llm.chat_completions.openai.vision.nonstream.works: {tier: P0, source: model_prices json, rationale: gpt-4o vision; high usage} +llm.chat_completions.openai.prompt_cache_5m.nonstream.works: {tier: P0, source: model_prices json, rationale: Prompt caching cost optimization} +llm.chat_completions.openai.service_tier.nonstream.works: {tier: P1, source: OpenAI service_tier param, rationale: OpenAI scale-tier request option is forwarded and echoed} +llm.chat_completions.openai.thinking.nonstream.works: {tier: P1, source: model_prices json, rationale: o-series reasoning; emerging} +llm.chat_completions.openai.structured_output.nonstream.works: {tier: P1, source: model_prices json, rationale: response_schema extraction} +llm.chat_completions.anthropic.basic.nonstream.works: {tier: P0, source: 'proxy_server.py:8455', rationale: P0 route translated to Anthropic} +llm.chat_completions.anthropic.basic.stream.works: {tier: P0, source: 'proxy_server.py:8455', rationale: Streaming translation} +llm.chat_completions.anthropic.tool_use.nonstream.works: {tier: P0, source: model_prices json, rationale: Claude tool_use; high usage} +llm.chat_completions.anthropic.tool_use.stream.works: {tier: P0, source: model_prices json, rationale: Streaming tool calls} +llm.chat_completions.anthropic.vision.nonstream.works: {tier: P0, source: model_prices json, rationale: Claude vision; high usage} +llm.chat_completions.anthropic.prompt_cache_5m.nonstream.works: {tier: P0, source: model_prices json, rationale: Claude prompt caching} +llm.chat_completions.anthropic.thinking.nonstream.works: {tier: P1, source: model_prices json, rationale: Claude extended thinking} +llm.chat_completions.anthropic.structured_output.nonstream.works: {tier: P1, source: model_prices json, rationale: Claude response_schema} +llm.chat_completions.bedrock_converse.basic.nonstream.works: {tier: P0, source: 'proxy_server.py:8455', rationale: P0 route; Bedrock Converse unified} +llm.chat_completions.bedrock_converse.basic.stream.works: {tier: P0, source: 'proxy_server.py:8455', rationale: Streaming over Converse} +llm.chat_completions.bedrock_converse.tool_use.nonstream.works: {tier: P0, source: model_prices json, rationale: Converse function_calling; AWS adoption} +llm.chat_completions.bedrock_converse.vision.nonstream.works: {tier: P0, source: model_prices json, rationale: Bedrock vision (Anthropic/Nova)} +llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.works: {tier: P1, source: model_prices json, rationale: Anthropic-on-Bedrock caching} +llm.chat_completions.bedrock_converse.thinking.nonstream.works: {tier: P1, source: model_prices json, rationale: Anthropic thinking on Bedrock} +llm.chat_completions.vertex.basic.nonstream.works: {tier: P0, source: 'proxy_server.py:8455', rationale: P0 route; Vertex AI} +llm.chat_completions.vertex.basic.stream.works: {tier: P0, source: 'proxy_server.py:8455', rationale: Streaming over Vertex} +llm.chat_completions.vertex.tool_use.nonstream.works: {tier: P0, source: model_prices json, rationale: Vertex Gemini function_calling} +llm.chat_completions.vertex.vision.nonstream.works: {tier: P0, source: model_prices json, rationale: Gemini vision} +llm.chat_completions.vertex.prompt_cache_5m.nonstream.works: {tier: P1, source: model_prices json, rationale: Vertex Gemini prompt caching} +llm.chat_completions.azure_openai.basic.nonstream.works: {tier: P0, source: 'proxy_server.py:8455', rationale: P0 route; Azure OpenAI deployments} +llm.chat_completions.azure_openai.tool_use.nonstream.works: {tier: P0, source: model_prices json, rationale: Azure OpenAI function_calling} +llm.chat_completions.azure_foundry.basic.nonstream.works: {tier: P1, source: 'proxy_server.py:8455', rationale: 'Azure Foundry (azure_ai); newer, smoke'} +llm.messages.anthropic.basic.nonstream.works: {tier: P0, source: 'anthropic_endpoints/endpoints.py:64', rationale: Core endpoint; Anthropic Messages native} +llm.messages.anthropic.basic.stream.works: {tier: P0, source: 'anthropic_endpoints/endpoints.py:64', rationale: Streaming Messages API} +llm.messages.anthropic.basic.nonstream.cost_logged: {tier: P0, source: 'anthropic_endpoints/endpoints.py:64', rationale: Cost logged on passthrough} +llm.messages.anthropic.tool_use.nonstream.works: {tier: P0, source: model_prices json, rationale: Tool calls via Messages API} +llm.messages.anthropic.tool_use.stream.works: {tier: P0, source: model_prices json, rationale: Streaming tool calls} +llm.messages.anthropic.vision.nonstream.works: {tier: P0, source: model_prices json, rationale: Vision via Messages API} +llm.messages.anthropic.prompt_cache_5m.nonstream.works: {tier: P0, source: model_prices json, rationale: Prompt caching via Messages API} +llm.messages.anthropic.thinking.nonstream.works: {tier: P1, source: model_prices json, rationale: Extended thinking via Messages API} +llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit: {tier: P0, source: llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py, rationale: Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882), fail_before_fix: proven} +llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works: {tier: P0, source: llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py, rationale: Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (#32831), fail_before_fix: proven} +llm.responses.openai.basic.nonstream.works: {tier: P0, source: 'response_api_endpoints/endpoints.py:26', rationale: Core endpoint; OpenAI Responses native} +llm.responses.openai.basic.stream.works: {tier: P0, source: 'response_api_endpoints/endpoints.py:26', rationale: Streaming via /v1/responses} +llm.responses.openai.basic.nonstream.cost_logged: {tier: P0, source: 'response_api_endpoints/endpoints.py:26', rationale: Cost logged on responses} +llm.responses.openai.tool_use.nonstream.works: {tier: P0, source: model_prices json, rationale: Tool calls via Responses API} +llm.responses.openai.vision.nonstream.works: {tier: P0, source: model_prices json, rationale: Vision via Responses API} +llm.responses.anthropic.basic.nonstream.works: {tier: P1, source: 'response_api_endpoints/endpoints.py:26', rationale: Responses w/ Anthropic translation (smoke)} +llm.responses.anthropic.tool_use.nonstream.works: {tier: P1, source: model_prices json, rationale: Responses tool calls w/ Anthropic} +llm.responses.bedrock_converse.basic.nonstream.works: {tier: P1, source: 'response_api_endpoints/endpoints.py:26', rationale: Responses w/ Bedrock Converse (smoke)} +llm.responses.bedrock_converse.tool_use.nonstream.works: {tier: P1, source: model_prices json, rationale: Responses tool calls w/ Converse} +llm.responses.vertex.basic.nonstream.works: {tier: P1, source: 'response_api_endpoints/endpoints.py:26', rationale: Responses w/ Vertex (smoke)} +llm.responses.vertex.tool_use.nonstream.works: {tier: P1, source: model_prices json, rationale: Responses tool calls w/ Vertex} +llm.responses.azure_openai.basic.nonstream.works: {tier: P1, source: 'response_api_endpoints/endpoints.py:26', rationale: Responses w/ Azure OpenAI (smoke)} +llm.responses.azure_openai.tool_use.nonstream.works: {tier: P1, source: model_prices json, rationale: Responses tool calls w/ Azure OpenAI} +llm.embeddings.openai.basic.nonstream.works: {tier: P0, source: 'test_embeddings_endpoint_e2e.py:23', rationale: 'Core endpoint, live vector response'} +llm.embeddings.openai.basic.nonstream.cost_logged: {tier: P0, source: 'SPEND_TRACKING_COVERAGE_MATRIX.md:34', rationale: Cost tracking on embeddings} +llm.embeddings.azure_openai.basic.nonstream.works: {tier: P0, source: llms/azure/azure.py, rationale: Azure embeddings via translation} +llm.embeddings.bedrock.basic.nonstream.works: {tier: P1, source: llms/bedrock/embed/embedding.py, rationale: Bedrock Titan embeddings} +llm.embeddings.vertex.basic.nonstream.works: {tier: P1, source: vertex_embeddings/embedding_handler.py, rationale: Vertex embeddings} +llm.embeddings.cohere.basic.nonstream.works: {tier: P1, source: llms/cohere/embed/handler.py, rationale: Cohere embeddings} +llm.embeddings.anthropic.basic.nonstream.works: {tier: P1, source: llms/anthropic/chat/handler.py, rationale: Anthropic vector API (verify support)} +llm.batches.openai.create.nonstream.works: {tier: P0, source: test_batches_e2e.py, rationale: Core batch create} +llm.batches.openai.retrieve.nonstream.works: {tier: P0, source: test_batches_e2e.py, rationale: 'Batch retrieve, id round-trip + status'} +llm.batches.openai.cancel.nonstream.works: {tier: P0, source: test_batches_e2e.py, rationale: Batch cancel} +llm.batches.openai.list.nonstream.works: {tier: P0, source: test_batches_e2e.py, rationale: Batch list envelope} +llm.batches.openai.file_lifecycle.nonstream.works: {tier: P0, source: test_batches_e2e.py, rationale: File upload/retrieve/delete for batch flow} +llm.batches.openai_encoded.basic.nonstream.works: {tier: P0, source: batches/capabilities.py, rationale: Encoded scenario lifecycle} +llm.batches.openai_unified.basic.nonstream.works: {tier: P0, source: batches/capabilities.py, rationale: Unified/managed-id scenario} +llm.batches.openai_model_param.basic.nonstream.works: {tier: P0, source: batches/capabilities.py, rationale: Model-param scenario} +llm.batches.openai_provider_fallback.basic.nonstream.works: {tier: P0, source: batches/capabilities.py, rationale: Provider-fallback raw-id scenario} +llm.batches.azure_openai.basic.nonstream.works: {tier: P0, source: 'batches/capabilities.py:98', rationale: Azure batches all scenarios} +llm.batches.vertex.basic.nonstream.works: {tier: P0, source: 'batches/capabilities.py:98', rationale: Vertex batches} +llm.batches.bedrock.basic.nonstream.works: {tier: P0, source: 'batches/capabilities.py:98', rationale: Bedrock batches (encoded/unified only)} +llm.batches.openai.key_model_access_denied.nonstream.works: {tier: P0, source: test_batches_e2e.py, rationale: Key model restriction 403 on upload/create} +llm.files.openai.upload.nonstream.works: {tier: P0, source: 'openai_files_endpoints/files_endpoints.py:46', rationale: File upload returns OpenAIFileObject} +llm.files.openai.retrieve.nonstream.works: {tier: P0, source: files_endpoints.py, rationale: File retrieve by id} +llm.files.openai.delete.nonstream.works: {tier: P0, source: files_endpoints.py, rationale: File delete returns deleted=true} +llm.files.openai.list.nonstream.works: {tier: P0, source: files_endpoints.py, rationale: File list paginated} +llm.files.azure_openai.upload.nonstream.works: {tier: P0, source: 'batches/capabilities.py:45', rationale: Azure file upload managed backend} +llm.files.vertex.upload.nonstream.works: {tier: P0, source: 'batches/capabilities.py:52', rationale: Vertex file upload to GCS} +llm.files.bedrock.upload.nonstream.works: {tier: P0, source: 'batches/capabilities.py:59', rationale: Bedrock file upload to S3} +llm.rerank.cohere.basic.nonstream.works: {tier: P1, source: 'test_rerank_e2e.py:29', rationale: 'Cohere rerank, top_n + relevance_score'} +llm.rerank.bedrock.basic.nonstream.works: {tier: P1, source: llms/bedrock/rerank/handler.py, rationale: Bedrock rerank} +llm.rerank.together_ai.basic.nonstream.works: {tier: P1, source: llms/together_ai/rerank/handler.py, rationale: Together rerank} +llm.images_generations.openai.basic.nonstream.works: {tier: P1, source: 'test_image_generation_e2e.py:22', rationale: 'OpenAI image gen, b64/url'} +llm.images_generations.azure_openai.basic.nonstream.works: {tier: P1, source: llms/azure/azure.py, rationale: Azure DALL-E} +llm.images_generations.vertex.basic.nonstream.works: {tier: P1, source: vertex_ai/image_generation/image_generation_handler.py, rationale: Vertex Imagen} +llm.images_generations.bedrock.basic.nonstream.works: {tier: P1, source: bedrock/image_generation/image_handler.py, rationale: Bedrock Titan Image} +llm.images_generations.black_forest_labs.basic.nonstream.works: {tier: P1, source: black_forest_labs/image_generation/handler.py, rationale: BFL Flux via OpenAI-compat} +llm.audio_speech.openai.basic.nonstream.works: {tier: P1, source: 'test_audio_speech_e2e.py:22', rationale: OpenAI TTS binary audio} +llm.audio_speech.openai.basic.stream.works: {tier: P1, source: 'proxy_server.py:9043', rationale: TTS streaming chunk generator} +llm.audio_speech.azure_openai.basic.nonstream.works: {tier: P1, source: llms/azure/azure.py, rationale: Azure TTS} +llm.audio_speech.vertex.basic.nonstream.works: {tier: P1, source: vertex_ai/text_to_speech/text_to_speech_handler.py, rationale: Vertex TTS} +llm.audio_transcriptions.openai.basic.nonstream.works: {tier: P1, source: openai/transcriptions/handler.py, rationale: OpenAI Whisper} +llm.audio_transcriptions.azure_openai.basic.nonstream.works: {tier: P1, source: azure/audio_transcriptions.py, rationale: Azure STT} +llm.audio_transcriptions.soniox.basic.nonstream.works: {tier: P2, source: soniox/audio_transcription/handler.py, rationale: Soniox via OpenAI-compat (smoke)} +llm.audio_transcriptions.nvidia_riva.basic.nonstream.works: {tier: P2, source: nvidia_riva/audio_transcription/handler.py, rationale: NVIDIA Riva (smoke)} +llm.moderations.openai.basic.nonstream.works: {tier: P1, source: proxy_server.py, rationale: OpenAI moderations (only provider)} + +# ---- logging ---- +logging.s3.success.writes_object: {tier: P0, source: integrations/s3_v2.py, rationale: Primary audit trail; batch flush no-drop} +logging.s3.failure.writes_object: {tier: P0, source: integrations/s3_v2.py, rationale: Failed calls persisted for compliance} +logging.gcs_bucket.success.writes_object: {tier: P0, source: integrations/gcs_bucket/gcs_bucket.py, rationale: GCS parallel to S3} +logging.datadog.success.exports_metric: {tier: P0, source: integrations/datadog/datadog.py, rationale: Powers dashboards/alerts; cardinality regressions common} +logging.datadog.stream.exports_metric: {tier: P0, source: integrations/datadog/datadog.py, rationale: Streaming aggregates usage after the last chunk; delivery and cost must survive that path} +logging.datadog.failure.exports_metric: {tier: P0, source: integrations/datadog/datadog.py, rationale: Failure metrics for alerting/SLO} +logging.prometheus.success.exports_metric: {tier: P0, source: integrations/prometheus.py, rationale: Standard OSS metrics; per-key cardinality (existing e2e)} +logging.otel.success.exports_metric: {tier: P0, source: integrations/otel/logger.py, rationale: OTEL spans on every call path} +logging.otel.stream.exports_metric: {tier: P0, source: integrations/otel/logger.py, rationale: Streaming closes the LLM span from the stream path; historically prone to duplicate/orphaned spans} +logging.otel.stream.records_ttft: {tier: P1, source: integrations/otel/mappers/genai.py, rationale: TTFT is the streaming latency SLI; a zero or span-length value silently corrupts dashboards} +logging.otel.failure.exports_metric: {tier: P0, source: integrations/otel/logger.py, rationale: Error spans for observability continuity} +logging.braintrust.success.logs_spend: {tier: P1, source: integrations/braintrust_logging.py, rationale: Evals platform spend} +logging.langsmith.success.logs_spend: {tier: P1, source: integrations/langsmith.py, rationale: LangChain ecosystem} +logging.arize.success.logs_spend: {tier: P1, source: integrations/arize/arize.py, rationale: ML-ops observability} +logging.mlflow.success.logs_spend: {tier: P1, source: integrations/mlflow.py, rationale: Experiment tracking cost/run} +logging.opik.success.logs_spend: {tier: P1, source: integrations/opik/opik.py, rationale: Eval platform spend/case} +logging.openmeter.success.exports_metric: {tier: P1, source: integrations/openmeter.py, rationale: Usage metering for billing} +logging.literal_ai.success.logs_spend: {tier: P1, source: integrations/literal_ai.py, rationale: Tracing platform spend} +logging.posthog.success.exports_metric: {tier: P1, source: integrations/posthog.py, rationale: Product analytics batching} +logging.azure_storage.success.writes_object: {tier: P1, source: integrations/azure_storage/azure_storage.py, rationale: Azure blob for enterprise} +logging.cloudzero.success.logs_spend: {tier: P1, source: integrations/cloudzero/cloudzero.py, rationale: Cost ops correlation} +logging.focus.success.writes_object: {tier: P1, source: integrations/focus/focus_logger.py, rationale: Cost mgmt multi-destination export} +logging.niche_integrations.success.logs_spend: {tier: P2, source: grammar, rationale: 'SMOKE cohort: athina/galileo/deepeval/langtrace/weave/lunary/humanloop/traceloop/helicone/argilla/newrelic/sqs/supabase/dynamodb/agentops/lago/etc'} +logging.niche_integrations.failure.logs_spend: {tier: P2, source: grammar, rationale: SMOKE niche failure path} + +# ---- mcp ---- +mcp.list_tools.api_key.succeeds: {tier: P0, source: 'server.py:637', rationale: Core operation; most common auth path; high usage} +mcp.list_tools.api_key.denied_without_permission: {tier: P0, source: 'mcp_server_manager.py:1409', rationale: Permission guard is high blast-radius; multi-tenant safety} +mcp.call_tool.api_key.succeeds: {tier: P0, source: 'server.py:849', rationale: Primary operation; customer-critical; high usage} +mcp.call_tool.api_key.denied_without_permission: {tier: P0, source: 'rest_endpoints.py:305-386', rationale: Tool-level permission guard; multi-tenant safety} +mcp.list_tools.bearer.succeeds: {tier: P1, source: 'server.py:662', rationale: OAuth/bearer token flow; upstream delegation} +mcp.call_tool.bearer.succeeds: {tier: P1, source: 'server.py:886', rationale: Bearer token forwarding for tool invocation} +mcp.list_tools.oauth.succeeds: {tier: P1, source: 'rest_endpoints.py:138-188', rationale: Interactive OAuth2 flow; live token management} +mcp.call_tool.oauth.succeeds: {tier: P1, source: db.py user_oauth_credential lookup, rationale: OAuth2 token passthrough; per-user credential storage} +mcp.list_tools.none.succeeds: {tier: P1, source: 'mcp_server_manager.py:1485-1492', rationale: Public/anonymous servers; delegate_auth_to_upstream} +mcp.call_tool.none.succeeds: {tier: P1, source: 'rest_endpoints.py:305-334', rationale: No upstream auth required; demo servers} +mcp.get_prompt.api_key.succeeds: {tier: P1, source: 'server.py:1042', rationale: Prompt op; same auth stack as tools} +mcp.read_resource.api_key.succeeds: {tier: P1, source: 'server.py:1177', rationale: Resource op; same permission model as tools} +mcp.list_prompts.api_key.succeeds: {tier: P2, source: 'server.py:993', rationale: Smoke-level; same auth stack as list_tools} +mcp.list_resources.api_key.succeeds: {tier: P2, source: 'server.py:1089', rationale: Smoke; rarely used; same auth model as tools} + +# ---- mgmt ---- +mgmt.key.generate.persists: {tier: P0, source: 'key_management_endpoints.py:1444', rationale: API key survives DB roundtrip} +mgmt.key.generate.admin_only: {tier: P0, source: 'key_management_endpoints.py:1444', rationale: Only master/team-admin creates keys} +mgmt.key.generate.happy_path: {tier: P0, source: 'ui_sso.py:420', rationale: SSO-driven key gen (UI path)} +mgmt.key.update.persists: {tier: P0, source: 'key_management_endpoints.py:2462', rationale: Budget/model changes persist} +mgmt.key.update.admin_only: {tier: P0, source: 'key_management_endpoints.py:2462', rationale: Non-admin cannot escalate perms} +mgmt.key.update.happy_path: {tier: P1, source: 'key_management_endpoints.py:2462', rationale: Key edit through the dashboard} +mgmt.key.delete.persists: {tier: P0, source: 'key_management_endpoints.py:3122', rationale: Deletion revokes future calls} +mgmt.key.delete.admin_only: {tier: P0, source: 'key_management_endpoints.py:3122', rationale: Non-owner cannot delete} +mgmt.key.info.persists: {tier: P0, source: 'key_management_endpoints.py:3380', rationale: Info reflects all writes} +mgmt.team.new.persists: {tier: P0, source: 'team_endpoints.py:897', rationale: team_id/alias/budgets stored} +mgmt.team.new.admin_only: {tier: P0, source: 'team_endpoints.py:897', rationale: Only org-admin/master creates teams} +mgmt.team.member_add.persists: {tier: P0, source: 'team_endpoints.py:2424', rationale: Membership + per-member budget persist} +mgmt.team.member_add.member_forbidden: {tier: P0, source: 'team_endpoints.py:2424', rationale: Non-admin forbidden to add} +mgmt.team.member_delete.persists: {tier: P0, source: 'team_endpoints.py:2800', rationale: Removal revokes team key access} +mgmt.team.member_delete.member_forbidden: {tier: P0, source: 'team_endpoints.py:2800', rationale: Non-admin forbidden to remove} +mgmt.budget.new.persists: {tier: P0, source: 'budget_management_endpoints.py:40', rationale: max/soft/reset windows persist} +mgmt.budget.new.admin_only: {tier: P0, source: 'budget_management_endpoints.py:40', rationale: Requires master/admin} +mgmt.model.add.persists: {tier: P0, source: 'model_management_endpoints.py:1201', rationale: Registration persists for routing} +mgmt.model.add.admin_only: {tier: P0, source: 'model_management_endpoints.py:1201', rationale: Non-admin cannot inject model config} +mgmt.user.new.happy_path: {tier: P0, source: 'internal_user_endpoints.py:360', rationale: User creation full cycle} +mgmt.key.list.happy_path: {tier: P1, source: 'key_management_endpoints.py:5119', rationale: Key inventory pagination} +mgmt.key.block.persists: {tier: P1, source: 'key_management_endpoints.py:5849', rationale: Blocked stays blocked on restart} +mgmt.key.unblock.persists: {tier: P1, source: 'key_management_endpoints.py:5960', rationale: Unblock restores access} +mgmt.key.regenerate.happy_path: {tier: P1, source: 'key_management_endpoints.py:6071', rationale: 'Rotation: new works, old invalid'} +mgmt.key.health.happy_path: {tier: P1, source: 'key_management_endpoints.py:4292', rationale: Key health endpoint} +mgmt.key.bulk_update.happy_path: {tier: P1, source: 'key_management_endpoints.py:2677', rationale: Batch key updates} +mgmt.team.update.persists: {tier: P1, source: 'team_endpoints.py:1582', rationale: Metadata/budget updates persist} +mgmt.team.delete.persists: {tier: P1, source: 'team_endpoints.py:1750', rationale: Deletion prevents key access} +mgmt.team.block.persists: {tier: P1, source: team_endpoints.py, rationale: Block suspends all members} +mgmt.team.info.happy_path: {tier: P1, source: 'team_endpoints.py:2244', rationale: Metadata+members+budgets} +mgmt.team.list.happy_path: {tier: P1, source: 'team_endpoints.py:3645', rationale: Pagination/filtering} +mgmt.team.member_update.persists: {tier: P1, source: 'team_endpoints.py:2768', rationale: Member budget/role updates persist} +mgmt.user.update.persists: {tier: P1, source: 'internal_user_endpoints.py:555', rationale: Metadata/perm updates persist} +mgmt.user.delete.persists: {tier: P1, source: 'internal_user_endpoints.py:640', rationale: Deletion revokes keys+teams} +mgmt.user.list.happy_path: {tier: P1, source: 'internal_user_endpoints.py:475', rationale: Admin view all users} +mgmt.user.info.happy_path: {tier: P1, source: 'internal_user_endpoints.py:440', rationale: Roles/perms/team membership} +mgmt.organization.new.happy_path: {tier: P1, source: 'organization_endpoints.py:403', rationale: Org for multi-tenant isolation} +mgmt.organization.update.persists: {tier: P1, source: 'organization_endpoints.py:545', rationale: Org metadata updates persist} +mgmt.organization.delete.persists: {tier: P1, source: 'organization_endpoints.py:710', rationale: Cascades to teams/keys} +mgmt.organization.member_add.happy_path: {tier: P1, source: 'organization_endpoints.py:835', rationale: Org member onboarding} +mgmt.customer.new.happy_path: {tier: P1, source: 'customer_endpoints.py:372', rationale: End-user for spend tracking} +mgmt.customer.delete.persists: {tier: P1, source: 'customer_endpoints.py:480', rationale: Removes from spend tracking} +mgmt.end_user.new.happy_path: {tier: P1, source: 'customer_endpoints.py:730', rationale: End-user create (synonym)} +mgmt.tag.new.happy_path: {tier: P1, source: 'tag_management_endpoints.py:160', rationale: Tag for spend categorization} +mgmt.tag.list.happy_path: {tier: P1, source: 'tag_management_endpoints.py:315', rationale: Tag enumeration} +mgmt.tag.delete.persists: {tier: P1, source: 'tag_management_endpoints.py:390', rationale: Stops future tagging} +mgmt.model.update.persists: {tier: P1, source: 'model_management_endpoints.py:1358', rationale: Pricing/concurrency persist} +mgmt.model.delete.persists: {tier: P1, source: 'model_management_endpoints.py:1045', rationale: Removes from registry} +mgmt.model.block.persists: {tier: P1, source: model_management_endpoints.py, rationale: Blocked model stays blocked} +mgmt.access_group.new.happy_path: {tier: P1, source: 'model_access_group_management_endpoints.py:450', rationale: Model permissioning group} +mgmt.access_group.info.happy_path: {tier: P1, source: 'model_access_group_management_endpoints.py:600', rationale: Access group membership query} +mgmt.mcp_server.register.happy_path: {tier: P1, source: 'mcp_management_endpoints.py:880', rationale: MCP server registration} +mgmt.mcp_server.approve.persists: {tier: P1, source: 'mcp_management_endpoints.py:1200', rationale: Admin approval persists} +mgmt.budget.update.persists: {tier: P1, source: 'budget_management_endpoints.py:155', rationale: Limit changes apply} +mgmt.budget.delete.persists: {tier: P1, source: 'budget_management_endpoints.py:280', rationale: Clears limits} +mgmt.budget.list.happy_path: {tier: P1, source: 'budget_management_endpoints.py:215', rationale: Budget enumeration} +mgmt.callback.list.happy_path: {tier: P2, source: callback_management_endpoints.py, rationale: Callback config (smoke)} +mgmt.cache_settings.update.happy_path: {tier: P2, source: cache_settings_endpoints.py, rationale: Cache config (smoke)} +mgmt.cost_tracking.estimate.happy_path: {tier: P2, source: cost_tracking_settings.py, rationale: Cost estimate (smoke)} +mgmt.router_settings.update.happy_path: {tier: P2, source: router_settings_endpoints.py, rationale: Router config (smoke)} +mgmt.jwt_key_mapping.new.happy_path: {tier: P2, source: jwt_key_mapping_endpoints.py, rationale: JWT->key mapping (smoke)} +mgmt.compliance.gdpr.happy_path: {tier: P2, source: compliance_endpoints.py, rationale: GDPR ops (smoke)} +mgmt.tool_management.list.happy_path: {tier: P2, source: tool_management_endpoints.py, rationale: Tool inventory (smoke)} +mgmt.fallback_management.update.happy_path: {tier: P2, source: fallback_management_endpoints.py, rationale: Fallback config (smoke)} +mgmt.config_override.hashicorp_vault.happy_path: {tier: P2, source: config_override_endpoints.py, rationale: Vault integration (smoke)} +mgmt.workflow.list.happy_path: {tier: P2, source: workflow_management_endpoints.py, rationale: Workflow tracking (smoke)} +mgmt.credential_migration.check.happy_path: {tier: P2, source: 'key_management_endpoints.py:4252', rationale: Encryption migration (smoke)} + +# ---- other ---- +other.auth.master_key.valid_allows: {tier: P0, source: 'user_api_key_auth.py:1569-1588', rationale: Master key authenticates; timing-safe compare} +other.auth.master_key.invalid_denied: {tier: P0, source: 'user_api_key_auth.py:1580', rationale: Invalid master key rejected} +other.auth.jwt.valid_token_allows: {tier: P0, source: 'handle_jwt.py:77-150', rationale: Valid JWT with correct issuer + claims grants access} +other.auth.jwt.expired_denied: {tier: P0, source: 'handle_jwt.py:125-135', rationale: Expired JWT rejected even with valid signature} +other.auth.jwt.invalid_signature_denied: {tier: P0, source: 'handle_jwt.py:145-150', rationale: Bad/missing signature fails verification} +other.auth.virtual_key.route_permission_enforced: {tier: P0, source: 'route_checks.py:89-151', rationale: allowed_routes whitelist denies disallowed routes} +other.auth.virtual_key.route_group_allowed: {tier: P1, source: 'route_checks.py:106-128', rationale: 'allowed_routes=[llm_api_routes] grants all LLM endpoints'} +other.auth.passthrough.model_allowlist_enforced: {tier: P1, source: 'route_checks.py:135-151', rationale: Passthrough enforces per-key model allow-lists} +other.auth.oauth2.token_valid_allows: {tier: P1, source: 'oauth2_check.py:15-73', rationale: OAuth2 introspection grants active token} +other.auth.oauth2.token_invalid_denied: {tier: P1, source: 'oauth2_check.py:37-73', rationale: Expired/inactive OAuth2 token denied} +other.auth.ip_allowlist.internal_ip_allows: {tier: P1, source: 'ip_address_utils.py:54-76', rationale: Internal CIDR bypasses public-API restriction} +other.auth.ip_allowlist.external_ip_denied_to_private: {tier: P1, source: 'ip_address_utils.py:54-76', rationale: External IP cannot reach internal-only resources} +other.lifecycle.readiness.public_probe: {tier: P0, source: '_health_endpoints.py:1551-1570', rationale: Unauthenticated /health/readiness safe for LBs} +other.lifecycle.readiness.reports_db_status: {tier: P0, source: '_health_endpoints.py:1551-1570', rationale: readiness distinguishes healthy vs DB-unreachable} +other.lifecycle.readiness.shutting_down_returns_503: {tier: P0, source: '_health_endpoints.py:1554-1556', rationale: Graceful shutdown drains LB via 503} +other.lifecycle.readiness_details.authenticated_diagnostics: {tier: P1, source: '_health_endpoints.py:1574-1584', rationale: Auth'd details expose cache/callback status} +other.lifecycle.liveness.ping: {tier: P1, source: '_health_endpoints.py:134-155', rationale: Liveness confirms server responding} +other.lifecycle.startup.config_loads: {tier: P0, source: 'proxy_server.py:4020-4100', rationale: 'Startup loads YAML, resolves env, persists to DB'} +other.lifecycle.startup.env_vars_resolved: {tier: P1, source: 'proxy_server.py:3984-4010', rationale: os.environ/ refs resolved at startup} +other.lifecycle.background_health_check.interval_configurable: {tier: P1, source: 'proxy_server.py:3245-3310', rationale: Background checks run at configurable interval} +other.config.runtime_update.applies_at_runtime: {tier: P0, source: 'proxy_server.py:14014-14060', rationale: /config/update persists to DB + invalidates cache} +other.config.general_settings.alert_webhook_side_effect: {tier: P1, source: 'proxy_server.py:14215', rationale: alert_to_webhook_url auto-enables slack alerting} +other.config.secret_resolution.kms_integration: {tier: P1, source: 'proxy_server.py:3984-4010', rationale: Resolves secrets from Vault/KMS at startup} +other.config.overrides.audit_logged: {tier: P1, source: 'config_override_endpoints.py:67-100', rationale: 'Config override mutations audit-logged, values redacted'} +other.key_mgmt.regenerate.grace_period_honored: {tier: P1, source: 'key_management_endpoints.py:4503-4560', rationale: Old key valid during grace_period then revoked} +other.key_mgmt.spend_reset.resets_to_value: {tier: P1, source: 'key_management_endpoints.py:4841', rationale: reset_spend resets accumulated spend} + +# ---- quota_management ---- +quota_management.ratelimit.rpm.blocks_over_limit: {tier: P0, source: parallel_request_limiter_v3.py, rationale: v3 limiter enforces RPM per key/team/model; 429 on breach} +quota_management.ratelimit.tpm.blocks_over_limit: {tier: P0, source: parallel_request_limiter_v3.py, rationale: v3 limiter enforces TPM per key/team/model; 429 on breach} +quota_management.ratelimit.rpm.resets_after_window: {tier: P1, source: parallel_request_limiter_v3.py, rationale: 'Rate-limit window (LITELLM_RATE_LIMIT_WINDOW_SIZE, 60s default) expires; a blocked key serves again in the next window'} +quota_management.ratelimit.rpm.headers_report_remaining: {tier: P1, source: parallel_request_limiter_v3.py async_post_call_success_hook, rationale: 'Successful responses carry x-ratelimit-api_key-{limit,remaining}-{requests,tokens} so clients can pace'} +quota_management.ratelimit.priority_generous.picks_under_tpm: {tier: P1, source: 'dynamic_rate_limiter_v3.py:36-52', rationale: Generous mode (<80% sat) allows priority borrowing} +quota_management.ratelimit.priority_strict.picks_under_tpm: {tier: P1, source: 'dynamic_rate_limiter_v3.py:53-71', rationale: Strict mode (>=80% sat) enforces priority fairness} +quota_management.budget.key.blocks_over_limit: {tier: P0, source: proxy/auth/auth_checks.py, rationale: A key's max_budget blocks further paid calls once spend crosses it} +quota_management.budget.internal_user.blocks_over_limit: {tier: P1, source: proxy/auth/auth_checks.py, rationale: An internal user's max_budget governs personal keys} +quota_management.budget.end_user.blocks_over_limit: {tier: P1, source: proxy/auth/auth_checks.py, rationale: A customer (end-user) max_budget blocks calls attributed via user=} +quota_management.budget.organization.blocks_over_limit: {tier: P1, source: proxy/auth/auth_checks.py, rationale: An organization's max_budget blocks keys under its teams} +quota_management.budget.team_member.blocks_over_limit: {tier: P1, source: proxy/auth/auth_checks.py, rationale: A member's per-team budget blocks independently of the team budget} +quota_management.budget.tag.blocks_over_limit: {tier: P1, source: router_strategy/budget_limiter.py, rationale: Proxy-level tag budgets block tagged requests at the cap} +quota_management.budget.model_max.isolates_per_model: {tier: P1, source: proxy/hooks/model_max_budget_limiter.py, rationale: model_max_budget caps one model without touching a sibling's budget} +quota_management.budget.soft.alerts_without_blocking: {tier: P1, source: proxy/auth/auth_checks.py, rationale: soft_budget alerts but never blocks traffic} +quota_management.budget.key.resets_after_window: {tier: P1, source: proxy/common_utils/reset_budget_job.py, rationale: budget_duration zeroes key spend after the window; a blocked key serves again} +quota_management.budget.team_member.resets_after_window: {tier: P1, source: proxy/common_utils/reset_budget_job.py, rationale: Member per-team budget reset keeps advancing window after window} +quota_management.budget.key_multi_window.blocks_then_resets: {tier: P1, source: proxy/common_utils/reset_budget_job.py, rationale: budget_limits enforce within a short window and serve again in the next} +quota_management.budget.key_multi_window.resets_windows_independently: {tier: P2, source: proxy/common_utils/reset_budget_job.py, rationale: Each window of a multi-window budget resets on its own schedule} +quota_management.budget.team_multi_window.blocks_then_resets: {tier: P1, source: proxy/common_utils/reset_budget_job.py, rationale: Team budget_limits enforce and reset per window} +quota_management.budget.fallback.routes_to_fallback: {tier: P1, source: proxy/hooks/model_max_budget_limiter.py, rationale: budget_fallbacks reroute to the fallback model once the primary's budget is exhausted} +quota_management.budget.spend_counter.reseed_matches_db: {tier: P2, source: proxy/spend_tracking/budget_reservation.py, rationale: Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)} +quota_management.spend_tracking.chat_completions.logs_cost: {tier: P0, source: proxy/spend_tracking/spend_tracking_utils.py, rationale: A paid chat call writes a nonzero spend row} +quota_management.spend_tracking.stream.logs_cost: {tier: P1, source: proxy/spend_tracking/spend_tracking_utils.py, rationale: Streaming responses aggregate token counts into a spend row} +quota_management.spend_tracking.embeddings.logs_cost: {tier: P1, source: proxy/spend_tracking/spend_tracking_utils.py, rationale: Embedding calls write nonzero spend rows} +quota_management.spend_tracking.cache_hit.zero_cost: {tier: P1, source: proxy/spend_tracking/spend_tracking_utils.py, rationale: A response-cache hit logs at zero cost with the cache-hit marker} +quota_management.spend_tracking.key_rollup.matches_sum_of_logs: {tier: P1, source: proxy/db/db_spend_update_writer.py, rationale: A key's rolled-up spend equals the sum of its log rows} +quota_management.spend_tracking.concurrent_burst.loses_no_spend: {tier: P1, source: proxy/db/db_spend_update_writer.py, rationale: Concurrent calls all land as spend; no row lost to write contention} +quota_management.spend_tracking.tags.attributes_spend: {tier: P1, source: proxy/spend_tracking/spend_tracking_utils.py, rationale: Request tags round-trip to spend rows and tag rollups match tagged logs} +quota_management.spend_tracking.end_user.attributes_spend: {tier: P1, source: proxy/spend_tracking/spend_tracking_utils.py, rationale: user= attribution lands the end-user id on the spend row} +quota_management.spend_tracking.per_model.writes_own_rows: {tier: P2, source: proxy/spend_tracking/spend_tracking_utils.py, rationale: Each model on a shared key gets its own spend row} +quota_management.spend_tracking.failure.writes_failure_row: {tier: P1, source: proxy/spend_tracking/spend_log_error_logger.py, rationale: A failed call writes a failure-status spend row} +quota_management.spend_tracking.spend_calculate.returns_cost: {tier: P2, source: proxy/spend_tracking/spend_management_endpoints.py, rationale: /spend/calculate prices a hypothetical request at nonzero cost} +quota_management.spend_tracking.pagination.keeps_total: {tier: P2, source: proxy/spend_tracking/spend_management_endpoints.py, rationale: Spend-logs v2 pagination caps page size without losing the total} + +# ---- reliability ---- +reliability.fallback.5xx.routes_to_fallback: {tier: P0, source: 'litellm/router.py:2024', rationale: Reroute on provider 5xx to alternate deployment} +reliability.fallback.context_window.routes_to_fallback: {tier: P0, source: 'litellm/router.py:6108', rationale: Fallback when model exceeds context limit} +reliability.fallback.content_policy.routes_to_fallback: {tier: P0, source: 'litellm/router.py:6023', rationale: Reroute on content-policy violation} +reliability.fallback.timeout.routes_to_fallback: {tier: P0, source: 'litellm/router.py:2766', rationale: Fallback on request timeout} +reliability.retry.5xx.succeeds_within_retries: {tier: P0, source: 'litellm/router.py:6414', rationale: Transient 5xx often succeeds on retry} +reliability.retry.timeout.succeeds_within_retries: {tier: P0, source: 'get_retry_from_policy.py:44', rationale: Timeout retried per policy} +reliability.retry.429.succeeds_within_retries: {tier: P0, source: 'get_retry_from_policy.py:46', rationale: 429 retried per RateLimitErrorRetries policy} +reliability.retry.auth.succeeds_within_retries: {tier: P1, source: 'get_retry_from_policy.py:42', rationale: Transient auth glitch retry} +reliability.retry.context_window.succeeds_within_retries: {tier: P1, source: 'get_retry_from_policy.py:51', rationale: Multi-attempt on context error} +reliability.cooldown.5xx.trips_then_recovers: {tier: P0, source: 'cooldown_handlers.py:40', rationale: 'Deployment cools after repeated 5xx, recovers after cooldown_time'} +reliability.cooldown.429.trips_then_recovers: {tier: P0, source: 'cooldown_handlers.py:69', rationale: 'Cools on 429, avoids hammering exhausted provider'} +reliability.cooldown.auth.trips_then_recovers: {tier: P1, source: 'cooldown_handlers.py:74', rationale: Cools on 401 auth error} +reliability.cooldown.timeout.trips_then_recovers: {tier: P1, source: 'cooldown_handlers.py:77', rationale: Cools on 408 timeout} +reliability.routing.simple_shuffle.picks_healthy_deployment: {tier: P1, source: router_strategy/simple_shuffle.py, rationale: Baseline weighted/uniform pick} +reliability.routing.latency_based.picks_lowest_latency: {tier: P1, source: router_strategy/lowest_latency.py, rationale: Routes to lowest-latency deployment} +reliability.routing.cost_based.picks_lowest_cost: {tier: P1, source: router_strategy/lowest_cost.py, rationale: Spend-aware routing} +reliability.routing.usage_based.picks_under_tpm: {tier: P0, source: router_strategy/lowest_tpm_rpm_v2.py, rationale: Routes to lowest-TPM deployment; prevents over-allocation} +reliability.routing.least_busy.picks_lowest_traffic: {tier: P1, source: router_strategy/least_busy.py, rationale: Fewest in-flight requests} +reliability.routing.complexity_llm_classifier.routes_by_llm_tier: {tier: P1, source: router_strategy/complexity_router/complexity_router.py, rationale: v2 auto-router LLM complexity classifier runs over the proxy and routes by semantic tier instead of silently crashing on absent litellm_metadata and falling back to heuristic scoring, fail_before_fix: proven} +reliability.cache.exact.returns_cached: {tier: P1, source: litellm/caching/caching.py, rationale: Response cache returns cached on exact match} +reliability.cache.prompt_caching_model_select.returns_cached: {tier: P1, source: router_utils/prompt_caching_cache.py, rationale: Selects model supporting prompt caching for cacheable prefix} +reliability.circuit_breaker.redis.trips_then_recovers: {tier: P0, source: 'litellm/caching/redis_cache.py:99', rationale: Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops} +reliability.timeout.request_timeout.exceeds_deadline: {tier: P1, source: 'litellm/router.py:545-551', rationale: Per-request timeout raises Timeout} +reliability.timeout.stream_timeout.exceeds_deadline: {tier: P1, source: 'litellm/router.py:551', rationale: Streaming chunk-delivery timeout} +reliability.perf.latency.under_slo: {tier: P1, source: router_strategy/lowest_latency.py, rationale: Latency SLO (p50/p99) compliance} +reliability.perf.throughput.under_slo: {tier: P1, source: grammar, rationale: Throughput SLO under load} diff --git a/tests/e2e/coverage_registry/product_surface.py b/tests/e2e/coverage_registry/product_surface.py new file mode 100644 index 00000000000..faa84c4eb46 --- /dev/null +++ b/tests/e2e/coverage_registry/product_surface.py @@ -0,0 +1,228 @@ +"""Generate the LLM denominator from the product surface instead of hand-listing it. + +The set of LLM cells we want covered is derived, not curated: the endpoint, route, +capability and streaming vocabularies live in `schema.py`, and which capabilities are +real for a given route is read from `model_prices_and_context_window.json` (the same +metadata the proxy ships). Adding a provider capability there, or a value to the +schema vocabulary, grows the denominator on its own; a test PR never edits it. + +Scope: this generates the conversational core (chat_completions, messages, responses), +whose ids follow the `llm.....works` grammar. +The Anthropic-format `messages` surface is the Claude Code compatibility matrix, so its +capabilities are the CLI feature set rather than model flags and are not gated by the +json. Non-core LLM endpoints (batches, files, rerank, embeddings, audio, images) use an +operation grammar the vocabulary does not enumerate, and the behavior modules (mgmt, +mcp, reliability, quota, logging, guardrail, other) have no clean cartesian in the +schema; both are carried by the curated overlay for now, and wiring their own +product-surface sources (route table, guardrail_hooks/, integrations/, router_strategy/) +into generation is the documented follow-up. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from .schema import ( + LlmCapability, + LlmEndpoint, + LlmRoute, + LlmStreaming, + format_llm_id, +) + +REPO_ROOT = Path(__file__).resolve().parents[3] +MODEL_PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" + +CLAUDE_CODE_ROUTES: tuple[LlmRoute, ...] = ( + "anthropic", + "azure_foundry", + "bedrock_converse", + "bedrock_invoke", + "vertex", +) + +ROUTE_PROVIDERS: dict[LlmRoute, tuple[str, ...]] = { + "anthropic": ("anthropic",), + "azure_foundry": ("azure_ai",), + "azure_openai": ("azure",), + "bedrock_converse": ("bedrock_converse", "bedrock"), + "bedrock_invoke": ("bedrock",), + "cohere": ("cohere", "cohere_chat"), + "openai": ("openai",), + "together_ai": ("together_ai",), +} +VERTEX_PROVIDER_PREFIXES: tuple[str, ...] = ("vertex_ai", "gemini") + + +@dataclass(frozen=True, slots=True) +class CapabilitySpec: + capability: LlmCapability + flag: str | None + endpoints: frozenset[LlmEndpoint] + streamable: bool + + +_CONVERSATIONAL: frozenset[LlmEndpoint] = frozenset({"chat_completions", "messages", "responses"}) +_MESSAGES_ONLY: frozenset[LlmEndpoint] = frozenset({"messages"}) +_CHAT_ONLY: frozenset[LlmEndpoint] = frozenset({"chat_completions"}) + +CAPABILITY_SPECS: tuple[CapabilitySpec, ...] = ( + CapabilitySpec("basic", None, _CONVERSATIONAL, streamable=True), + CapabilitySpec("tool_use", "supports_function_calling", _CONVERSATIONAL, streamable=True), + CapabilitySpec("vision", "supports_vision", _CONVERSATIONAL, streamable=False), + CapabilitySpec("structured_output", "supports_response_schema", _CONVERSATIONAL, streamable=False), + CapabilitySpec("thinking", "supports_reasoning", _CONVERSATIONAL, streamable=False), + CapabilitySpec("prompt_cache_5m", "supports_prompt_caching", _CONVERSATIONAL, streamable=False), + CapabilitySpec("service_tier", None, _CHAT_ONLY, streamable=False), + CapabilitySpec("prompt_cache_1h", "supports_prompt_caching", _MESSAGES_ONLY, streamable=False), + CapabilitySpec("mid_conversation_system", "supports_mid_conversation_system", _MESSAGES_ONLY, streamable=False), + CapabilitySpec("pdf_input", "supports_pdf_input", _MESSAGES_ONLY, streamable=False), + CapabilitySpec("web_search", "supports_web_search", _MESSAGES_ONLY, streamable=False), + CapabilitySpec("thinking_with_tool_use", "supports_reasoning", _MESSAGES_ONLY, streamable=False), + CapabilitySpec("count_tokens", None, _MESSAGES_ONLY, streamable=False), + CapabilitySpec("long_context_1m", None, _MESSAGES_ONLY, streamable=False), + CapabilitySpec("tool_search", None, _MESSAGES_ONLY, streamable=False), +) + + +class ModelEntry(BaseModel): + """Only the model_prices fields the denominator reads. Each capability flag is + modelled explicitly so no supports_* value is threaded as an untyped dict value.""" + + model_config = ConfigDict(extra="ignore") + + litellm_provider: str | None = None + mode: str | None = None + supports_function_calling: bool = False + supports_vision: bool = False + supports_response_schema: bool = False + supports_reasoning: bool = False + supports_prompt_caching: bool = False + supports_pdf_input: bool = False + supports_web_search: bool = False + supports_mid_conversation_system: bool = False + + def enabled_flags(self) -> frozenset[str]: + values: dict[str, bool] = { + "supports_function_calling": self.supports_function_calling, + "supports_vision": self.supports_vision, + "supports_response_schema": self.supports_response_schema, + "supports_reasoning": self.supports_reasoning, + "supports_prompt_caching": self.supports_prompt_caching, + "supports_pdf_input": self.supports_pdf_input, + "supports_web_search": self.supports_web_search, + "supports_mid_conversation_system": self.supports_mid_conversation_system, + } + return frozenset(flag for flag, enabled in values.items() if enabled) + + +_ENTRIES_ADAPTER: TypeAdapter[dict[str, ModelEntry]] = TypeAdapter(dict[str, ModelEntry]) + + +def load_model_entries(path: Path = MODEL_PRICES_PATH) -> tuple[ModelEntry, ...]: + entries = _ENTRIES_ADAPTER.validate_json(path.read_bytes()) + return tuple(entries.values()) + + +def _route_of(provider: str) -> LlmRoute | None: + if provider.startswith(VERTEX_PROVIDER_PREFIXES) or provider == "gemini": + return "vertex" + return next( + (route for route, providers in ROUTE_PROVIDERS.items() if provider in providers), + None, + ) + + +def _flags_by_route(entries: tuple[ModelEntry, ...]) -> dict[LlmRoute, frozenset[str]]: + pairs: tuple[tuple[LlmRoute, frozenset[str]], ...] = tuple( + (route, entry.enabled_flags()) + for entry in entries + if entry.litellm_provider is not None + for route in (_route_of(entry.litellm_provider),) + if route is not None + ) + routes: frozenset[LlmRoute] = frozenset(route for route, _ in pairs) + return {route: frozenset[str]().union(*(flags for r, flags in pairs if r == route)) for route in routes} + + +def _routes_with_mode(entries: tuple[ModelEntry, ...], mode: str) -> frozenset[LlmRoute]: + return frozenset( + route + for entry in entries + if entry.mode == mode and entry.litellm_provider is not None + for route in (_route_of(entry.litellm_provider),) + if route is not None + ) + + +def _routes_for_endpoint(endpoint: LlmEndpoint, entries: tuple[ModelEntry, ...]) -> tuple[LlmRoute, ...]: + if endpoint == "messages": + return CLAUDE_CODE_ROUTES + if endpoint == "responses": + return tuple(sorted(_routes_with_mode(entries, "responses"))) + return tuple(sorted(_routes_with_mode(entries, "chat"))) + + +def _streamings(spec: CapabilitySpec) -> tuple[LlmStreaming, ...]: + return ("nonstream", "stream") if spec.streamable else ("nonstream",) + + +def _available( + endpoint: LlmEndpoint, + spec: CapabilitySpec, + route: LlmRoute, + flags_by_route: dict[LlmRoute, frozenset[str]], +) -> bool: + if endpoint == "messages": + return True + if spec.flag is None: + return True + return spec.flag in flags_by_route.get(route, frozenset()) + + +def generate_llm_cell_ids(entries: tuple[ModelEntry, ...] | None = None) -> frozenset[str]: + """The generated core-LLM denominator: one `...works` cell per real + (endpoint, route, capability, streaming) combination.""" + model_entries = load_model_entries() if entries is None else entries + flags_by_route = _flags_by_route(model_entries) + return frozenset( + format_llm_id(endpoint, route, spec.capability, streaming, "works") + for spec in CAPABILITY_SPECS + for endpoint in spec.endpoints + for route in _routes_for_endpoint(endpoint, model_entries) + if _available(endpoint, spec, route, flags_by_route) + for streaming in _streamings(spec) + ) + + +_ENDPOINT_ROUTE_FRAGMENTS: dict[LlmEndpoint, str] = { + "chat_completions": "/chat/completions", + "messages": "/v1/messages", + "responses": "/responses", + "embeddings": "/embeddings", + "batches": "/batches", + "files": "/files", + "rerank": "/rerank", + "images_generations": "/images/generations", + "audio_speech": "/audio/speech", + "audio_transcriptions": "/audio/transcriptions", + "moderations": "/moderations", + "realtime": "/realtime", +} + +ROUTE_CHECKABLE_ENDPOINTS: frozenset[LlmEndpoint] = frozenset(_ENDPOINT_ROUTE_FRAGMENTS) + + +def route_table_endpoints() -> frozenset[LlmEndpoint] | None: + """The LLM endpoints the proxy actually serves, read from the live route table, + or None when litellm cannot be imported (kept off the hot path so the generator + and its unit tests stay hermetic). Used by the collector as a drift check.""" + try: + from litellm.proxy._types import LiteLLMRoutes + except ImportError: + return None + served = "\n".join(str(path) for path in LiteLLMRoutes.llm_api_routes.value) + return frozenset(endpoint for endpoint, fragment in _ENDPOINT_ROUTE_FRAGMENTS.items() if fragment in served) diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml deleted file mode 100644 index fac266149f4..00000000000 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Quota Management (behavior features): rate limits, budgets, spend tracking. Grounded in -# litellm/proxy/hooks/ + litellm/proxy/auth/auth_checks.py + litellm/proxy/spend_tracking/. -- {id: quota_management.ratelimit.rpm.blocks_over_limit, module: quota_management, tier: P0, behavior: ratelimit, variant: rpm, assertions: [blocks_over_limit], exercised_on: [chat_completions, messages], source: "parallel_request_limiter_v3.py", rationale: "v3 limiter enforces RPM per key/team/model; 429 on breach"} -- {id: quota_management.ratelimit.tpm.blocks_over_limit, module: quota_management, tier: P0, behavior: ratelimit, variant: tpm, assertions: [blocks_over_limit], exercised_on: [chat_completions, messages], source: "parallel_request_limiter_v3.py", rationale: "v3 limiter enforces TPM per key/team/model; 429 on breach"} -- {id: quota_management.ratelimit.rpm.resets_after_window, module: quota_management, tier: P1, behavior: ratelimit, variant: rpm, assertions: [resets_after_window], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py", rationale: "Rate-limit window (LITELLM_RATE_LIMIT_WINDOW_SIZE, 60s default) expires; a blocked key serves again in the next window"} -- {id: quota_management.ratelimit.rpm.headers_report_remaining, module: quota_management, tier: P1, behavior: ratelimit, variant: rpm, assertions: [headers_report_remaining], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py async_post_call_success_hook", rationale: "Successful responses carry x-ratelimit-api_key-{limit,remaining}-{requests,tokens} so clients can pace"} -- {id: quota_management.ratelimit.priority_generous.picks_under_tpm, module: quota_management, tier: P1, behavior: ratelimit, variant: priority_generous, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "dynamic_rate_limiter_v3.py:36-52", rationale: "Generous mode (<80% sat) allows priority borrowing"} -- {id: quota_management.ratelimit.priority_strict.picks_under_tpm, module: quota_management, tier: P1, behavior: ratelimit, variant: priority_strict, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "dynamic_rate_limiter_v3.py:53-71", rationale: "Strict mode (>=80% sat) enforces priority fairness"} -- {id: quota_management.budget.key.blocks_over_limit, module: quota_management, tier: P0, behavior: budget, variant: key, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "A key's max_budget blocks further paid calls once spend crosses it"} -- {id: quota_management.budget.internal_user.blocks_over_limit, module: quota_management, tier: P1, behavior: budget, variant: internal_user, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "An internal user's max_budget governs personal keys"} -- {id: quota_management.budget.end_user.blocks_over_limit, module: quota_management, tier: P1, behavior: budget, variant: end_user, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "A customer (end-user) max_budget blocks calls attributed via user="} -- {id: quota_management.budget.organization.blocks_over_limit, module: quota_management, tier: P1, behavior: budget, variant: organization, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "An organization's max_budget blocks keys under its teams"} -- {id: quota_management.budget.team_member.blocks_over_limit, module: quota_management, tier: P1, behavior: budget, variant: team_member, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "A member's per-team budget blocks independently of the team budget"} -- {id: quota_management.budget.tag.blocks_over_limit, module: quota_management, tier: P1, behavior: budget, variant: tag, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "router_strategy/budget_limiter.py", rationale: "Proxy-level tag budgets block tagged requests at the cap"} -- {id: quota_management.budget.model_max.isolates_per_model, module: quota_management, tier: P1, behavior: budget, variant: model_max, assertions: [isolates_per_model], exercised_on: [chat_completions], source: "proxy/hooks/model_max_budget_limiter.py", rationale: "model_max_budget caps one model without touching a sibling's budget"} -- {id: quota_management.budget.soft.alerts_without_blocking, module: quota_management, tier: P1, behavior: budget, variant: soft, assertions: [alerts_without_blocking], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "soft_budget alerts but never blocks traffic"} -- {id: quota_management.budget.key.resets_after_window, module: quota_management, tier: P1, behavior: budget, variant: key, assertions: [resets_after_window], exercised_on: [chat_completions], source: "proxy/common_utils/reset_budget_job.py", rationale: "budget_duration zeroes key spend after the window; a blocked key serves again"} -- {id: quota_management.budget.team_member.resets_after_window, module: quota_management, tier: P1, behavior: budget, variant: team_member, assertions: [resets_after_window], exercised_on: [chat_completions], source: "proxy/common_utils/reset_budget_job.py", rationale: "Member per-team budget reset keeps advancing window after window"} -- {id: quota_management.budget.key_multi_window.blocks_then_resets, module: quota_management, tier: P1, behavior: budget, variant: key_multi_window, assertions: [blocks_then_resets], exercised_on: [chat_completions], source: "proxy/common_utils/reset_budget_job.py", rationale: "budget_limits enforce within a short window and serve again in the next"} -- {id: quota_management.budget.key_multi_window.resets_windows_independently, module: quota_management, tier: P2, behavior: budget, variant: key_multi_window, assertions: [resets_windows_independently], exercised_on: [chat_completions], source: "proxy/common_utils/reset_budget_job.py", rationale: "Each window of a multi-window budget resets on its own schedule"} -- {id: quota_management.budget.team_multi_window.blocks_then_resets, module: quota_management, tier: P1, behavior: budget, variant: team_multi_window, assertions: [blocks_then_resets], exercised_on: [chat_completions], source: "proxy/common_utils/reset_budget_job.py", rationale: "Team budget_limits enforce and reset per window"} -- {id: quota_management.budget.fallback.routes_to_fallback, module: quota_management, tier: P1, behavior: budget, variant: fallback, assertions: [routes_to_fallback], exercised_on: [messages], source: "proxy/hooks/model_max_budget_limiter.py", rationale: "budget_fallbacks reroute to the fallback model once the primary's budget is exhausted"} -- {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"} -- {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"} -- {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"} -- {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"} -- {id: quota_management.spend_tracking.cache_hit.zero_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_hit, assertions: [zero_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A response-cache hit logs at zero cost with the cache-hit marker"} -- {id: quota_management.spend_tracking.key_rollup.matches_sum_of_logs, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_rollup, assertions: [matches_sum_of_logs], exercised_on: [chat_completions], source: "proxy/db/db_spend_update_writer.py", rationale: "A key's rolled-up spend equals the sum of its log rows"} -- {id: quota_management.spend_tracking.concurrent_burst.loses_no_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: concurrent_burst, assertions: [loses_no_spend], exercised_on: [chat_completions], source: "proxy/db/db_spend_update_writer.py", rationale: "Concurrent calls all land as spend; no row lost to write contention"} -- {id: quota_management.spend_tracking.tags.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: tags, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Request tags round-trip to spend rows and tag rollups match tagged logs"} -- {id: quota_management.spend_tracking.end_user.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: end_user, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "user= attribution lands the end-user id on the spend row"} -- {id: quota_management.spend_tracking.per_model.writes_own_rows, module: quota_management, tier: P2, behavior: spend_tracking, variant: per_model, assertions: [writes_own_rows], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Each model on a shared key gets its own spend row"} -- {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"} -- {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"} -- {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"} diff --git a/tests/e2e/coverage_registry/registry.py b/tests/e2e/coverage_registry/registry.py index baeea080a9d..ba7a99d0627 100644 --- a/tests/e2e/coverage_registry/registry.py +++ b/tests/e2e/coverage_registry/registry.py @@ -1,29 +1,35 @@ -"""Load and validate the registry: the denominator, built in one shot from the YAMLs.""" +"""Build the coverage denominator: the generated product surface unioned with the ids +the overlay still enumerates, each annotated with its curated human fields.""" from __future__ import annotations -from collections import Counter from pathlib import Path -import yaml -from pydantic import TypeAdapter +from .overlay import OVERLAY_PATH, load_overlay +from .product_surface import generate_llm_cell_ids +from .schema import Cell, OverlayRow, Tier, parse_module -from .schema import Cell - -REGISTRY_DIR = Path(__file__).resolve().parent - -_CELLS_ADAPTER: TypeAdapter[tuple[Cell, ...]] = TypeAdapter(tuple[Cell, ...]) +_DEFAULT_ROW = OverlayRow(tier=Tier.P2) -def _load_cells(path: Path) -> tuple[Cell, ...]: - return _CELLS_ADAPTER.validate_python(yaml.safe_load(path.read_text()) or ()) +def _cell(cell_id: str, row: OverlayRow) -> Cell: + return Cell( + id=cell_id, + module=parse_module(cell_id), + tier=row.tier, + source=row.source, + rationale=row.rationale, + fail_before_fix=row.fail_before_fix, + supported=row.supported, + ) -def load_registry(registry_dir: Path = REGISTRY_DIR) -> tuple[Cell, ...]: - """Every cell across every `*.yaml`, validated. Raises on a schema violation or - a duplicate id, since either would corrupt the coverage denominator.""" - cells = tuple(cell for path in sorted(registry_dir.glob("*.yaml")) for cell in _load_cells(path)) - duplicates = sorted(cid for cid, n in Counter(c.id for c in cells).items() if n > 1) - if duplicates: - raise ValueError(f"duplicate cell ids in registry: {duplicates}") - return cells +def load_registry(overlay_path: Path = OVERLAY_PATH) -> tuple[Cell, ...]: + """Every denominator cell, validated. The id set is the generated LLM surface + unioned with the overlay's ids; each cell carries its overlay row when one exists + and otherwise a default P2 row, so a newly generated surface shows up as an + uncovered gap rather than silently vanishing. Raises on a bad module prefix or a + malformed overlay row, since either would corrupt the denominator.""" + overlay = load_overlay(overlay_path) + cell_ids = generate_llm_cell_ids() | frozenset(overlay) + return tuple(_cell(cid, overlay.get(cid, _DEFAULT_ROW)) for cid in sorted(cell_ids)) diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml deleted file mode 100644 index ba192c2912e..00000000000 --- a/tests/e2e/coverage_registry/reliability.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Reliability & Performance (behavior features). Grounded in litellm/router.py + router_strategy/ + router_utils/. -- {id: reliability.fallback.5xx.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: "5xx", assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:2024", rationale: "Reroute on provider 5xx to alternate deployment"} -- {id: reliability.fallback.context_window.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: context_window, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6108", rationale: "Fallback when model exceeds context limit"} -- {id: reliability.fallback.content_policy.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: content_policy, assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:6023", rationale: "Reroute on content-policy violation"} -- {id: reliability.fallback.timeout.routes_to_fallback, module: reliability, tier: P0, behavior: fallback, variant: "timeout", assertions: [routes_to_fallback], exercised_on: [chat_completions, messages], source: "litellm/router.py:2766", rationale: "Fallback on request timeout"} -- {id: reliability.retry.5xx.succeeds_within_retries, module: reliability, tier: P0, behavior: retry, variant: "5xx", assertions: [succeeds_within_retries], exercised_on: [chat_completions, messages], source: "litellm/router.py:6414", rationale: "Transient 5xx often succeeds on retry"} -- {id: reliability.retry.timeout.succeeds_within_retries, module: reliability, tier: P0, behavior: retry, variant: timeout, assertions: [succeeds_within_retries], exercised_on: [chat_completions, messages], source: "get_retry_from_policy.py:44", rationale: "Timeout retried per policy"} -- {id: reliability.retry.429.succeeds_within_retries, module: reliability, tier: P0, behavior: retry, variant: "429", assertions: [succeeds_within_retries], exercised_on: [chat_completions, messages], source: "get_retry_from_policy.py:46", rationale: "429 retried per RateLimitErrorRetries policy"} -- {id: reliability.retry.auth.succeeds_within_retries, module: reliability, tier: P1, behavior: retry, variant: auth, assertions: [succeeds_within_retries], exercised_on: [chat_completions, messages], source: "get_retry_from_policy.py:42", rationale: "Transient auth glitch retry"} -- {id: reliability.retry.context_window.succeeds_within_retries, module: reliability, tier: P1, behavior: retry, variant: context_window, assertions: [succeeds_within_retries], exercised_on: [chat_completions, messages], source: "get_retry_from_policy.py:51", rationale: "Multi-attempt on context error"} -- {id: reliability.cooldown.5xx.trips_then_recovers, module: reliability, tier: P0, behavior: cooldown, variant: "5xx", assertions: [trips_then_recovers], exercised_on: [chat_completions, messages], source: "cooldown_handlers.py:40", rationale: "Deployment cools after repeated 5xx, recovers after cooldown_time"} -- {id: reliability.cooldown.429.trips_then_recovers, module: reliability, tier: P0, behavior: cooldown, variant: "429", assertions: [trips_then_recovers], exercised_on: [chat_completions, messages], source: "cooldown_handlers.py:69", rationale: "Cools on 429, avoids hammering exhausted provider"} -- {id: reliability.cooldown.auth.trips_then_recovers, module: reliability, tier: P1, behavior: cooldown, variant: auth, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages], source: "cooldown_handlers.py:74", rationale: "Cools on 401 auth error"} -- {id: reliability.cooldown.timeout.trips_then_recovers, module: reliability, tier: P1, behavior: cooldown, variant: timeout, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages], source: "cooldown_handlers.py:77", rationale: "Cools on 408 timeout"} -- {id: reliability.routing.simple_shuffle.picks_healthy_deployment, module: reliability, tier: P1, behavior: routing, variant: simple_shuffle, assertions: [picks_healthy_deployment], exercised_on: [chat_completions, messages], source: "router_strategy/simple_shuffle.py", rationale: "Baseline weighted/uniform pick"} -- {id: reliability.routing.latency_based.picks_lowest_latency, module: reliability, tier: P1, behavior: routing, variant: latency_based, assertions: [picks_lowest_latency], exercised_on: [chat_completions, messages], source: "router_strategy/lowest_latency.py", rationale: "Routes to lowest-latency deployment"} -- {id: reliability.routing.cost_based.picks_lowest_cost, module: reliability, tier: P1, behavior: routing, variant: cost_based, assertions: [picks_lowest_cost], exercised_on: [chat_completions, messages], source: "router_strategy/lowest_cost.py", rationale: "Spend-aware routing"} -- {id: reliability.routing.usage_based.picks_under_tpm, module: reliability, tier: P0, behavior: routing, variant: usage_based, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "router_strategy/lowest_tpm_rpm_v2.py", rationale: "Routes to lowest-TPM deployment; prevents over-allocation"} -- {id: reliability.routing.least_busy.picks_lowest_traffic, module: reliability, tier: P1, behavior: routing, variant: least_busy, assertions: [picks_lowest_traffic], exercised_on: [chat_completions, messages], source: "router_strategy/least_busy.py", rationale: "Fewest in-flight requests"} -- {id: reliability.routing.complexity_llm_classifier.routes_by_llm_tier, module: reliability, tier: P1, behavior: routing, variant: complexity_llm_classifier, assertions: [routes_by_llm_tier], exercised_on: [chat_completions], source: "router_strategy/complexity_router/complexity_router.py", fail_before_fix: proven, rationale: "v2 auto-router LLM complexity classifier runs over the proxy and routes by semantic tier instead of silently crashing on absent litellm_metadata and falling back to heuristic scoring"} -- {id: reliability.cache.exact.returns_cached, module: reliability, tier: P1, behavior: cache, variant: exact, assertions: [returns_cached], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/caching.py", rationale: "Response cache returns cached on exact match"} -- {id: reliability.cache.prompt_caching_model_select.returns_cached, module: reliability, tier: P1, behavior: cache, variant: prompt_caching_model_select, assertions: [returns_cached], exercised_on: [chat_completions], source: "router_utils/prompt_caching_cache.py", rationale: "Selects model supporting prompt caching for cacheable prefix"} -- {id: reliability.circuit_breaker.redis.trips_then_recovers, module: reliability, tier: P0, behavior: circuit_breaker, variant: redis, assertions: [trips_then_recovers], exercised_on: [chat_completions, messages, embeddings], source: "litellm/caching/redis_cache.py:99", rationale: "Redis breaker CLOSED->OPEN->HALF_OPEN; guards all cache/rate-limit ops"} -- {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"} -- {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} -- {id: reliability.perf.latency.under_slo, module: reliability, tier: P1, behavior: perf, variant: latency, assertions: [under_slo], exercised_on: [chat_completions, messages], source: "router_strategy/lowest_latency.py", rationale: "Latency SLO (p50/p99) compliance"} -- {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index a76774c3bde..0ab8955953f 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -1,17 +1,31 @@ -"""Registry row schema: the contract every denominator cell validates against. +"""Coverage cell vocabulary, id grammar, and the human-overlay row shape. A cell is one customer-noticeable behavior a single e2e test can assert pass/fail -on. `module` is the id's segment-1 prefix (eight of them); dashboard rollups can -split or merge those prefixes. The union is discriminated on `module`, so an LLM -row cannot carry a guardrail field and vice versa. +on, identified by a dotted id whose first segment is the module. The structural +facets an LLM id encodes (endpoint, route, capability, streaming) are parsed back +out of the id rather than stored a second time, so an id and its fields can never +drift. The only per-cell data a human curates lives in `OverlayRow`; the set of +cells itself (the denominator) is generated in `product_surface.py`. """ from __future__ import annotations +from dataclasses import dataclass from enum import Enum -from typing import Annotated, Literal +from typing import Literal, get_args -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +Module = Literal[ + "llm", + "mgmt", + "mcp", + "reliability", + "quota_management", + "logging", + "guardrail", + "other", +] class Tier(str, Enum): @@ -70,83 +84,93 @@ LlmCapability = Literal[ "web_search", ] +LlmStreaming = Literal["stream", "nonstream", "na"] + +LLM_ENDPOINTS: frozenset[str] = frozenset(get_args(LlmEndpoint)) +LLM_ROUTES: frozenset[str] = frozenset(get_args(LlmRoute)) +LLM_CAPABILITIES: frozenset[str] = frozenset(get_args(LlmCapability)) +LLM_STREAMINGS: frozenset[str] = frozenset(get_args(LlmStreaming)) + + +@dataclass(frozen=True, slots=True) +class LlmCellId: + endpoint: LlmEndpoint + route: LlmRoute + capability: LlmCapability + streaming: LlmStreaming + assertion: str + + +def format_llm_id( + endpoint: LlmEndpoint, + route: LlmRoute, + capability: LlmCapability, + streaming: LlmStreaming, + assertion: str, +) -> str: + return f"llm.{endpoint}.{route}.{capability}.{streaming}.{assertion}" + + +_LLM_CELL_ID_ADAPTER: TypeAdapter[LlmCellId] = TypeAdapter(LlmCellId) + + +def parse_llm_id(cell_id: str) -> LlmCellId | None: + """The structural facets of an LLM id, or None when the id is not an LLM cell + whose segments all match the typed vocabulary. Non-core LLM endpoints (batches, + files) use an operation grammar that is not part of this vocabulary and return + None here by design; they are carried by the overlay, not generated.""" + parts = tuple(cell_id.split(".")) + if len(parts) != 6 or parts[0] != "llm": + return None + _, endpoint, route, capability, streaming, assertion = parts + try: + return _LLM_CELL_ID_ADAPTER.validate_python( + { + "endpoint": endpoint, + "route": route, + "capability": capability, + "streaming": streaming, + "assertion": assertion, + } + ) + except ValidationError: + return None + + +class OverlayRow(BaseModel): + """The human-curated fields for one cell, keyed by id in overlay.yaml. Holds no + structural facet; those are parsed from the id or generated.""" -class _Base(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - id: str tier: Tier - assertions: tuple[str, ...] - source: str + source: str = "" rationale: str = "" fail_before_fix: FailBeforeFix = FailBeforeFix.unproven supported: bool = True -class LlmCell(_Base): - module: Literal["llm"] - subject_endpoint: LlmEndpoint - route: LlmRoute - capability: LlmCapability - streaming: Literal["stream", "nonstream", "na"] +@dataclass(frozen=True, slots=True) +class Cell: + """A denominator cell: its id, the module parsed from that id, and the curated + overlay fields (defaulted when the id has no overlay row).""" + + id: str + module: Module + tier: Tier + source: str = "" + rationale: str = "" + fail_before_fix: FailBeforeFix = FailBeforeFix.unproven + supported: bool = True -class MgmtCell(_Base): - module: Literal["mgmt"] - surface: Literal["api", "ui"] +_MODULE_ADAPTER: TypeAdapter[Module] = TypeAdapter(Module) -class McpCell(_Base): - module: Literal["mcp"] - operation: str - auth_family: Literal["none", "api_key", "bearer", "oauth"] +def parse_module(cell_id: str) -> Module: + return _MODULE_ADAPTER.validate_python(cell_id.split(".", 1)[0]) -class ReliabilityCell(_Base): - module: Literal["reliability"] - behavior: str - variant: str - exercised_on: tuple[str, ...] - - -class QuotaCell(_Base): - module: Literal["quota_management"] - behavior: Literal["ratelimit", "budget", "spend_tracking"] - variant: str - exercised_on: tuple[str, ...] - - -class LoggingCell(_Base): - module: Literal["logging"] - event: str - exercised_on: tuple[str, ...] - - -class GuardrailCell(_Base): - module: Literal["guardrail"] - hook_point: str - exercised_on: tuple[str, ...] - - -class OtherCell(_Base): - module: Literal["other"] - area: str - - -Cell = Annotated[ - LlmCell - | MgmtCell - | McpCell - | ReliabilityCell - | QuotaCell - | LoggingCell - | GuardrailCell - | OtherCell, - Field(discriminator="module"), -] - -CELL_ADAPTER: TypeAdapter[Cell] = TypeAdapter(Cell) - CORE_LLM_ENDPOINTS: frozenset[str] = frozenset( { "chat_completions", @@ -189,11 +213,11 @@ LOKI_MODULE_LABELS: dict[str, str] = { def dashboard_module(cell: Cell) -> str: - """Return the Grafana/reporting module for a registry cell.""" - if isinstance(cell, LlmCell): - if cell.subject_endpoint in CORE_LLM_ENDPOINTS: - return "Core LLMs" - return "Non-Core LLMs" + """The Grafana/reporting module for a cell, decided from its id. LLM cells split + into Core vs Non-Core on the endpoint segment; every other module maps by prefix.""" + if cell.module == "llm": + endpoint = cell.id.split(".")[1] + return "Core LLMs" if endpoint in CORE_LLM_ENDPOINTS else "Non-Core LLMs" return PREFIX_ROLLUP[cell.module] diff --git a/tests/e2e/coverage_registry/test_collector.py b/tests/e2e/coverage_registry/test_collector.py index 079ee215866..587083f6389 100644 --- a/tests/e2e/coverage_registry/test_collector.py +++ b/tests/e2e/coverage_registry/test_collector.py @@ -1,8 +1,8 @@ """Tests for the coverage-registry tooling: pure logic plus a registry canary. -No `e2e` marker, so these run without a proxy. They exercise the coverage math and -the registry loader, and guard the checked-in registry against schema drift and -duplicate ids. +No `e2e` marker, so these run without a proxy. They exercise the coverage math, the +id grammar, the product-surface generator, and the overlay loader, and guard the +generated-plus-overlay denominator against drift. """ from __future__ import annotations @@ -10,6 +10,7 @@ from __future__ import annotations from pathlib import Path import pytest +from pydantic import ValidationError from coverage_registry.collector import ( compute_coverage, @@ -18,86 +19,71 @@ from coverage_registry.collector import ( render_loki, render_prometheus, ) +from coverage_registry.overlay import load_overlay +from coverage_registry.product_surface import ( + ModelEntry, + generate_llm_cell_ids, +) from coverage_registry.registry import load_registry from coverage_registry.schema import ( - GuardrailCell, - LlmCell, - LlmEndpoint, - LoggingCell, + Cell, + Module, Tier, + format_llm_id, loki_module_label, + parse_llm_id, ) +_CHAT_BASIC = "llm.chat_completions.openai.basic.nonstream.works" +_MESSAGES_BASIC = "llm.messages.anthropic.basic.nonstream.works" +_RESPONSES_BASIC = "llm.responses.openai.basic.nonstream.works" +_BATCHES = "llm.batches.openai.create.nonstream.works" +_REALTIME = "llm.realtime.openai.session.na.works" -def _llm( - cell_id: str, tier: Tier, subject_endpoint: LlmEndpoint = "chat_completions" -) -> LlmCell: - return LlmCell( - id=cell_id, - module="llm", - tier=tier, - assertions=("works",), - source="test", - subject_endpoint=subject_endpoint, - route="openai", - capability="basic", - streaming="nonstream", - ) + +def _cell(cell_id: str, tier: Tier, module: Module = "llm") -> Cell: + return Cell(id=cell_id, module=module, tier=tier) def test_compute_coverage_counts_covered_p0_and_gaps() -> None: - cells = (_llm("llm.a", Tier.P0), _llm("llm.b", Tier.P0), _llm("llm.c", Tier.P1)) - report = compute_coverage(cells, frozenset({"llm.a"})) + cells = ( + _cell(_CHAT_BASIC, Tier.P0), + _cell(_MESSAGES_BASIC, Tier.P0), + _cell(_RESPONSES_BASIC, Tier.P1), + ) + report = compute_coverage(cells, frozenset({_CHAT_BASIC})) assert (report.total, report.covered) == (3, 1) assert (report.p0_total, report.p0_covered) == (2, 1) - assert report.p0_gaps == ("llm.b",) + assert report.p0_gaps == (_MESSAGES_BASIC,) assert report.orphan_markers == () def test_orphan_marker_is_reported_not_counted() -> None: - cells = (_llm("llm.a", Tier.P0),) - report = compute_coverage(cells, frozenset({"llm.a", "llm.ghost"})) + cells = (_cell(_CHAT_BASIC, Tier.P0),) + report = compute_coverage(cells, frozenset({_CHAT_BASIC, "llm.ghost"})) assert report.covered == 1 assert report.orphan_markers == ("llm.ghost",) def test_logging_and_guardrail_roll_up_into_one_module() -> None: cells = ( - LoggingCell( - id="logging.x", - module="logging", - tier=Tier.P0, - assertions=("logs_spend",), - source="t", - event="success", - exercised_on=("chat_completions",), - ), - GuardrailCell( - id="guardrail.y", - module="guardrail", - tier=Tier.P1, - assertions=("blocks",), - source="t", - hook_point="pre_call", - exercised_on=("chat_completions",), - ), + _cell("logging.langfuse.success.logs_spend", Tier.P0, "logging"), + _cell("guardrail.presidio.pre_call.masks", Tier.P1, "guardrail"), ) report = compute_coverage(cells, frozenset()) - logging_and_guardrails = next( - m for m in report.modules if m.module == "Logging & Guardrails" - ) + logging_and_guardrails = next(m for m in report.modules if m.module == "Logging & Guardrails") assert logging_and_guardrails.total == 2 def test_llm_cells_roll_up_by_core_endpoint() -> None: cells = ( - _llm("llm.chat", Tier.P0, "chat_completions"), - _llm("llm.messages", Tier.P0, "messages"), - _llm("llm.responses", Tier.P1, "responses"), - _llm("llm.batches", Tier.P0, "batches"), - _llm("llm.realtime", Tier.P1, "realtime"), + _cell(_CHAT_BASIC, Tier.P0), + _cell(_MESSAGES_BASIC, Tier.P0), + _cell(_RESPONSES_BASIC, Tier.P1), + _cell(_BATCHES, Tier.P0), + _cell(_REALTIME, Tier.P1), ) - report = compute_coverage(cells, frozenset({"llm.chat", "llm.batches"})) + report = compute_coverage(cells, frozenset({_CHAT_BASIC, _BATCHES})) core = next(m for m in report.modules if m.module == "Core LLMs") non_core = next(m for m in report.modules if m.module == "Non-Core LLMs") @@ -113,8 +99,8 @@ def test_llm_cells_roll_up_by_core_endpoint() -> None: def test_text_render_uses_plain_coverage_language() -> None: report = compute_coverage( - (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), - frozenset({"llm.chat"}), + (_cell(_CHAT_BASIC, Tier.P0), _cell(_BATCHES, Tier.P0)), + frozenset({_CHAT_BASIC}), ) text = render(report) @@ -126,8 +112,8 @@ def test_text_render_uses_plain_coverage_language() -> None: def test_json_render_exposes_module_coverage_for_grafana_jobs() -> None: report = compute_coverage( - (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), - frozenset({"llm.chat"}), + (_cell(_CHAT_BASIC, Tier.P0), _cell(_BATCHES, Tier.P0)), + frozenset({_CHAT_BASIC}), ) payload = render_json(report) @@ -139,8 +125,8 @@ def test_json_render_exposes_module_coverage_for_grafana_jobs() -> None: def test_prometheus_render_exposes_module_coverage_timeseries() -> None: report = compute_coverage( - (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), - frozenset({"llm.chat"}), + (_cell(_CHAT_BASIC, Tier.P0), _cell(_BATCHES, Tier.P0)), + frozenset({_CHAT_BASIC}), ) metrics = render_prometheus(report) @@ -153,42 +139,94 @@ def test_prometheus_render_exposes_module_coverage_timeseries() -> None: def test_loki_render_exposes_exact_stdout_lines_for_loki() -> None: report = compute_coverage( - (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), - frozenset({"llm.chat"}), + (_cell(_CHAT_BASIC, Tier.P0), _cell(_BATCHES, Tier.P0)), + frozenset({_CHAT_BASIC}), ) lines = render_loki(report).splitlines() assert len(lines) == 1 + len(report.modules) assert lines[0] == "COVERAGE_TOTAL percent=50.0 covered=1 total=2" - assert ( - lines[1] == "COVERAGE_MODULE module=core_llms percent=100.0 covered=1 total=1" - ) - assert ( - lines[2] == "COVERAGE_MODULE module=non_core_llms percent=0.0 covered=0 total=1" - ) + assert lines[1] == "COVERAGE_MODULE module=core_llms percent=100.0 covered=1 total=1" + assert lines[2] == "COVERAGE_MODULE module=non_core_llms percent=0.0 covered=0 total=1" assert [line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:]] == [ loki_module_label(module.module) for module in report.modules ] - assert all( - " " not in line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:] + assert all(" " not in line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:]) + + +def test_parse_llm_id_round_trips_and_rejects_non_vocab() -> None: + parsed = parse_llm_id(_CHAT_BASIC) + assert parsed is not None + assert (parsed.endpoint, parsed.route, parsed.capability, parsed.streaming) == ( + "chat_completions", + "openai", + "basic", + "nonstream", ) + assert ( + format_llm_id( + parsed.endpoint, + parsed.route, + parsed.capability, + parsed.streaming, + parsed.assertion, + ) + == _CHAT_BASIC + ) + assert parse_llm_id("llm.chat_completions.openai.not_a_capability.nonstream.works") is None + assert parse_llm_id("mgmt.key.generate.persists") is None -def test_real_registry_loads_and_ids_are_unique() -> None: +def test_generator_gates_flagged_capabilities_on_model_metadata() -> None: + entries = ( + ModelEntry(litellm_provider="openai", mode="chat", supports_function_calling=True), + ModelEntry(litellm_provider="anthropic", mode="chat"), + ) + generated = generate_llm_cell_ids(entries) + + assert "llm.chat_completions.openai.basic.nonstream.works" in generated + assert "llm.chat_completions.anthropic.basic.nonstream.works" in generated + assert "llm.chat_completions.openai.tool_use.nonstream.works" in generated + assert "llm.chat_completions.anthropic.tool_use.nonstream.works" not in generated + + +def test_generator_leaves_messages_matrix_ungated() -> None: + entries = (ModelEntry(litellm_provider="anthropic", mode="chat"),) + generated = generate_llm_cell_ids(entries) + + assert "llm.messages.anthropic.tool_use.nonstream.works" in generated + assert "llm.messages.anthropic.web_search.nonstream.works" in generated + assert "llm.messages.anthropic.service_tier.nonstream.works" not in generated + + +def test_real_registry_is_generated_superset_of_overlay_and_ids_are_unique() -> None: cells = load_registry() - ids = [c.id for c in cells] + ids = tuple(c.id for c in cells) assert len(cells) > 250 assert len(ids) == len(set(ids)) - assert any(c.id == "logging.prometheus.success.exports_metric" for c in cells) + + denominator = frozenset(ids) + assert generate_llm_cell_ids() <= denominator + assert frozenset(load_overlay()) <= denominator + assert "logging.prometheus.success.exports_metric" in denominator -def test_load_registry_rejects_duplicate_ids(tmp_path: Path) -> None: - row = ( - "- {id: llm.dup, module: llm, tier: P0, assertions: [works], source: t, " - "subject_endpoint: chat_completions, route: openai, capability: basic, streaming: nonstream}\n" - ) - (tmp_path / "a.yaml").write_text(row) - (tmp_path / "b.yaml").write_text(row) - with pytest.raises(ValueError, match="duplicate cell ids"): - load_registry(tmp_path) +def test_generated_cell_absent_from_overlay_defaults_to_p2(tmp_path: Path) -> None: + overlay = tmp_path / "overlay.yaml" + overlay.write_text("logging.custom.success.logs_spend: {tier: P0}\n") + cells = load_registry(overlay) + by_id = {c.id: c for c in cells} + + curated = by_id["logging.custom.success.logs_spend"] + assert curated.tier is Tier.P0 + + generated_id = "llm.chat_completions.openai.basic.nonstream.works" + assert by_id[generated_id].tier is Tier.P2 + + +def test_load_registry_rejects_unknown_module_prefix(tmp_path: Path) -> None: + overlay = tmp_path / "overlay.yaml" + overlay.write_text("nope.some.cell: {tier: P0}\n") + with pytest.raises(ValidationError): + load_registry(overlay)