diff --git a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py index 17411464050..c56f17ada8d 100644 --- a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py +++ b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -20,6 +20,7 @@ from tests.claude_code.matrix_builder import ( ResultsError, build_from_paths, build_matrix, + find_regressions, load_manifest, load_results, ) @@ -90,6 +91,66 @@ def test_build_matrix_any_fail_makes_cell_fail(): assert cell["error"] == "[claude-opus-4-7] timeout" +def _single_cell_matrix(results): + """Build a 1x1 matrix and return its only cell. Helper for the + not_applicable aggregation tests below.""" + manifest = { + "schema_version": "1", + "providers": ["vertex_ai"], + "features": [{"id": "f", "name": "F"}], + } + matrix = build_matrix( + manifest=manifest, + results=[ + {"feature_id": "f", "provider": "vertex_ai", "result": r} for r in results + ], + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + return matrix["features"][0]["providers"]["vertex_ai"] + + +def test_build_matrix_not_applicable_is_neutral_when_others_pass(): + """A tier that doesn't support the feature (`not_applicable`) must not + drag down a cell whose supported tiers all pass. Models the real + Vertex AI count_tokens case: Haiku 4.5 is unsupported, Sonnet/Opus + pass → the cell is green.""" + cell = _single_cell_matrix( + [ + {"status": "not_applicable", "reason": "[haiku] not supported"}, + {"status": "pass"}, + {"status": "pass"}, + ] + ) + assert cell == {"status": "pass"} + + +def test_build_matrix_all_not_applicable_makes_cell_not_applicable(): + """If *every* tier is not_applicable, the cell is not_applicable and + surfaces the first reason.""" + cell = _single_cell_matrix( + [ + {"status": "not_applicable", "reason": "first reason"}, + {"status": "not_applicable", "reason": "second reason"}, + ] + ) + assert cell == {"status": "not_applicable", "reason": "first reason"} + + +def test_build_matrix_fail_beats_not_applicable(): + """A genuine failure still reds the cell even when another tier is + not_applicable — failures win over the neutral skip.""" + cell = _single_cell_matrix( + [ + {"status": "not_applicable", "reason": "[haiku] not supported"}, + {"status": "fail", "error": "[opus] regression"}, + ] + ) + assert cell["status"] == "fail" + assert cell["error"] == "[opus] regression" + + def test_build_matrix_fills_not_tested_for_missing_cells(): manifest = { "schema_version": "1", @@ -291,6 +352,137 @@ def test_build_matrix_1x5_grid_one_failing_model_breaks_cell(): assert "claude-opus-4-7-bedrock-invoke" in cell["error"] +def _matrix(cells, *, names=None): + """Build a minimal matrix dict from a {(feature_id, provider): status} + or {(feature_id, provider): cell_dict} mapping. Helper for the + find_regressions tests below.""" + names = names or {} + features = {} + for (feature_id, provider), value in cells.items(): + cell = {"status": value} if isinstance(value, str) else dict(value) + features.setdefault(feature_id, {})[provider] = cell + return { + "features": [ + { + "id": feature_id, + "name": names.get(feature_id, feature_id.upper()), + "providers": providers, + } + for feature_id, providers in features.items() + ] + } + + +def test_find_regressions_flags_pass_to_fail(): + old = _matrix({("vision", "anthropic"): "pass"}) + new = _matrix( + {("vision", "anthropic"): {"status": "fail", "error": "credit balance too low"}} + ) + regressions = find_regressions(old, new) + assert len(regressions) == 1 + r = regressions[0] + assert r["feature_id"] == "vision" + assert r["provider"] == "anthropic" + assert r["old_status"] == "pass" + assert r["new_status"] == "fail" + assert r["error"] == "credit balance too low" + + +def test_find_regressions_ignores_red_to_red(): + """An already-failing cell that stays failing is NOT a regression — a + provider that's independently broken (e.g. out of credits) must not + block the daily auto-merge forever.""" + old = _matrix({("vision", "anthropic"): "fail"}) + new = _matrix({("vision", "anthropic"): "fail"}) + assert find_regressions(old, new) == [] + + +def test_find_regressions_ignores_improvements_and_steady_green(): + old = _matrix( + { + ("vision", "anthropic"): "fail", # red -> green + ("tool_use", "azure"): "pass", # green -> green + } + ) + new = _matrix( + { + ("vision", "anthropic"): "pass", + ("tool_use", "azure"): "pass", + } + ) + assert find_regressions(old, new) == [] + + +def test_find_regressions_ignores_green_to_grey(): + """green→not_tested / green→not_applicable are degradations but not + *red* regressions; we deliberately don't block on them.""" + old = _matrix( + { + ("vision", "azure"): "pass", + ("tool_use", "azure"): "pass", + } + ) + new = _matrix( + { + ("vision", "azure"): "not_tested", + ("tool_use", "azure"): {"status": "not_applicable", "reason": "skip"}, + } + ) + assert find_regressions(old, new) == [] + + +def test_find_regressions_ignores_new_cells_without_baseline(): + """A cell only present in the new matrix (new feature/provider) has no + baseline, so a fail there can't be a regression.""" + old = _matrix({("vision", "anthropic"): "pass"}) + new = _matrix( + { + ("vision", "anthropic"): "pass", + ("brand_new_feature", "anthropic"): "fail", + } + ) + assert find_regressions(old, new) == [] + + +def test_find_regressions_matches_by_id_not_name(): + """Renaming a feature's display name must not hide a regression: cells + are matched on the stable id.""" + old = _matrix({("thinking", "anthropic"): "pass"}, names={"thinking": "Old Name"}) + new = _matrix( + {("thinking", "anthropic"): "fail"}, names={"thinking": "Totally New Name"} + ) + regressions = find_regressions(old, new) + assert len(regressions) == 1 + assert regressions[0]["feature_id"] == "thinking" + assert regressions[0]["feature_name"] == "Totally New Name" + + +def test_find_regressions_reports_multiple_sorted(): + old = _matrix( + { + ("vision", "anthropic"): "pass", + ("tool_use", "anthropic"): "pass", + ("vision", "azure"): "pass", + } + ) + new = _matrix( + { + ("vision", "anthropic"): "fail", + ("tool_use", "anthropic"): "fail", + ("vision", "azure"): "pass", # stays green + } + ) + regressions = find_regressions(old, new) + keys = [(r["feature_id"], r["provider"]) for r in regressions] + assert keys == [("tool_use", "anthropic"), ("vision", "anthropic")] + + +def test_find_regressions_empty_old_matrix_is_safe(): + """No baseline at all (first publish) yields no regressions.""" + new = _matrix({("vision", "anthropic"): "fail"}) + assert find_regressions({}, new) == [] + + def test_build_from_paths_writes_output(tmp_path): out = tmp_path / "compatibility-matrix.json" matrix = build_from_paths( diff --git a/tests/claude_code/count_tokens/test_anthropic.py b/tests/claude_code/count_tokens/test_anthropic.py index 1ca35bdb860..194b9a40cd1 100644 --- a/tests/claude_code/count_tokens/test_anthropic.py +++ b/tests/claude_code/count_tokens/test_anthropic.py @@ -43,6 +43,7 @@ import pytest from tests.claude_code.http_probe import ( assert_count_tokens_shape, + count_tokens_unsupported_reason, probe_count_tokens, ) @@ -78,9 +79,18 @@ def test_count_tokens_anthropic(compat_result): failures = [] for model in ANTHROPIC_MODELS: - result = probe_count_tokens( - base_url=base_url, api_key=api_key, model=model - ) + result = probe_count_tokens(base_url=base_url, api_key=api_key, model=model) + + unsupported = count_tokens_unsupported_reason(result) + if unsupported is not None: + compat_result.add( + { + "status": "not_applicable", + "reason": f"[{model}] {unsupported}", + } + ) + continue + shape_error = assert_count_tokens_shape(result) if shape_error is not None: error = f"[{model}] count_tokens probe failed: {shape_error}" diff --git a/tests/claude_code/count_tokens/test_azure.py b/tests/claude_code/count_tokens/test_azure.py index 6e67d36ce50..a384004be57 100644 --- a/tests/claude_code/count_tokens/test_azure.py +++ b/tests/claude_code/count_tokens/test_azure.py @@ -43,6 +43,7 @@ import pytest from tests.claude_code.http_probe import ( assert_count_tokens_shape, + count_tokens_unsupported_reason, probe_count_tokens, ) @@ -78,9 +79,18 @@ def test_count_tokens_azure(compat_result): failures = [] for model in AZURE_MODELS: - result = probe_count_tokens( - base_url=base_url, api_key=api_key, model=model - ) + result = probe_count_tokens(base_url=base_url, api_key=api_key, model=model) + + unsupported = count_tokens_unsupported_reason(result) + if unsupported is not None: + compat_result.add( + { + "status": "not_applicable", + "reason": f"[{model}] {unsupported}", + } + ) + continue + shape_error = assert_count_tokens_shape(result) if shape_error is not None: error = f"[{model}] count_tokens probe failed: {shape_error}" diff --git a/tests/claude_code/count_tokens/test_bedrock_converse.py b/tests/claude_code/count_tokens/test_bedrock_converse.py index 04e7a5c71d0..482bd739b6a 100644 --- a/tests/claude_code/count_tokens/test_bedrock_converse.py +++ b/tests/claude_code/count_tokens/test_bedrock_converse.py @@ -43,6 +43,7 @@ import pytest from tests.claude_code.http_probe import ( assert_count_tokens_shape, + count_tokens_unsupported_reason, probe_count_tokens, ) @@ -78,9 +79,18 @@ def test_count_tokens_bedrock_converse(compat_result): failures = [] for model in BEDROCK_CONVERSE_MODELS: - result = probe_count_tokens( - base_url=base_url, api_key=api_key, model=model - ) + result = probe_count_tokens(base_url=base_url, api_key=api_key, model=model) + + unsupported = count_tokens_unsupported_reason(result) + if unsupported is not None: + compat_result.add( + { + "status": "not_applicable", + "reason": f"[{model}] {unsupported}", + } + ) + continue + shape_error = assert_count_tokens_shape(result) if shape_error is not None: error = f"[{model}] count_tokens probe failed: {shape_error}" diff --git a/tests/claude_code/count_tokens/test_bedrock_invoke.py b/tests/claude_code/count_tokens/test_bedrock_invoke.py index f0874919fbc..9634b187871 100644 --- a/tests/claude_code/count_tokens/test_bedrock_invoke.py +++ b/tests/claude_code/count_tokens/test_bedrock_invoke.py @@ -43,6 +43,7 @@ import pytest from tests.claude_code.http_probe import ( assert_count_tokens_shape, + count_tokens_unsupported_reason, probe_count_tokens, ) @@ -78,9 +79,18 @@ def test_count_tokens_bedrock_invoke(compat_result): failures = [] for model in BEDROCK_INVOKE_MODELS: - result = probe_count_tokens( - base_url=base_url, api_key=api_key, model=model - ) + result = probe_count_tokens(base_url=base_url, api_key=api_key, model=model) + + unsupported = count_tokens_unsupported_reason(result) + if unsupported is not None: + compat_result.add( + { + "status": "not_applicable", + "reason": f"[{model}] {unsupported}", + } + ) + continue + shape_error = assert_count_tokens_shape(result) if shape_error is not None: error = f"[{model}] count_tokens probe failed: {shape_error}" diff --git a/tests/claude_code/count_tokens/test_vertex_ai.py b/tests/claude_code/count_tokens/test_vertex_ai.py index fe84372cc41..6e960f7dfa4 100644 --- a/tests/claude_code/count_tokens/test_vertex_ai.py +++ b/tests/claude_code/count_tokens/test_vertex_ai.py @@ -43,6 +43,7 @@ import pytest from tests.claude_code.http_probe import ( assert_count_tokens_shape, + count_tokens_unsupported_reason, probe_count_tokens, ) @@ -78,9 +79,18 @@ def test_count_tokens_vertex_ai(compat_result): failures = [] for model in VERTEX_AI_MODELS: - result = probe_count_tokens( - base_url=base_url, api_key=api_key, model=model - ) + result = probe_count_tokens(base_url=base_url, api_key=api_key, model=model) + + unsupported = count_tokens_unsupported_reason(result) + if unsupported is not None: + compat_result.add( + { + "status": "not_applicable", + "reason": f"[{model}] {unsupported}", + } + ) + continue + shape_error = assert_count_tokens_shape(result) if shape_error is not None: error = f"[{model}] count_tokens probe failed: {shape_error}" diff --git a/tests/claude_code/cron_vm/README.md b/tests/claude_code/cron_vm/README.md index 7c05be00cf2..eb569f01fce 100644 --- a/tests/claude_code/cron_vm/README.md +++ b/tests/claude_code/cron_vm/README.md @@ -24,6 +24,7 @@ Action. Trade-offs: | --- | --- | | `run_daily.sh` | The actual cron job. Resolves versions, updates the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR. | | `build_matrix.py` | Tiny Python CLI that wraps `tests.claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. | +| `check_regressions.py` | Tiny Python CLI that wraps `tests.claude_code.matrix_builder.find_regressions`. Diffs the freshly built matrix against the currently-published one and exits `3` if any cell flipped green→red, which gates auto-merge. | | `litellm-compat-matrix.service` | systemd oneshot that invokes `run_daily.sh`. | | `litellm-compat-matrix.timer` | `OnCalendar=*-*-* 06:00:00 UTC`, `Persistent=true`. | | `litellm-compat-matrix.env.example` | Template for `/etc/litellm-compat-matrix.env`. | @@ -52,10 +53,24 @@ Action. Trade-offs: 7. **Opens or updates a docs PR**: `gh repo clone` of `litellm-docs` into a tempdir, deterministic head branch (`compat-matrix/--`), - `--force-with-lease` push, `gh pr create`. A re-run on the same - day fast-forwards the existing branch and `gh pr create` no-ops - ("a pull request for branch ... already exists" is treated as - success). + `--force` push **directly to `BerriAI/litellm-docs`** (the + `mateo-berri` token has write access, so this is a same-repo branch, + not a fork), `gh pr create`. A re-run on the same day fast-forwards + the existing branch and `gh pr create` no-ops ("a pull request for + branch ... already exists" is treated as success). These PRs are no + longer gated on a second human review. +8. **Gates auto-merge on a regression check**: before enabling + auto-merge, `check_regressions.py` diffs the new matrix against the + one currently on `main`. Auto-merge (`gh pr merge --auto --squash`) + is only enabled when **no cell flipped green→red** — i.e. every + transition is red→green, green→green, or red→red. A pre-existing red + cell (e.g. a provider that's out of API credits) is `red→red` and + does **not** block; only a `pass`→`fail` flip does. When a regression + is detected the PR is still opened/updated (with a warning banner + naming the offending cells) but auto-merge is left **off** — and any + auto-merge a prior same-day run enabled is explicitly disabled — so a + human reviews before it lands on the public table. The check fails + *closed*: if it errors, auto-merge is withheld. ## One-time VM setup @@ -129,9 +144,13 @@ sudo systemctl disable --now litellm-compat-matrix.timer - **`uv sync --frozen` requires the resolved tag to be tagged on GitHub.** If the latest stable release was made but not pushed as a git tag, the `git checkout` step fails. Push the tag, then rerun. -- **`gh auth` token rotation is your problem.** The cron does not - refresh the token; if the bot account's PAT expires the run will - fail at `gh repo clone` with a 401. Re-run `gh auth login`. +- **`GITHUB_TOKEN` rotation is your problem.** The cron does not + refresh the token; if `mateo-berri`'s PAT in + `/etc/litellm-compat-matrix.env` expires, the run fails at the + `git push`/`gh pr create` step with a 401 ("Bad credentials" / + "Authentication failed"). Mint a fresh PAT and update the env file. + The token needs write access to `BerriAI/litellm-docs` (classic + `repo` scope, or fine-grained Contents:RW + Pull requests:RW). - **First run after upgrading the Claude Code CLI is the riskiest one.** If the new CLI changes its wire format the matrix run can produce systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI diff --git a/tests/claude_code/cron_vm/check_regressions.py b/tests/claude_code/cron_vm/check_regressions.py new file mode 100644 index 00000000000..af8ee98b9c5 --- /dev/null +++ b/tests/claude_code/cron_vm/check_regressions.py @@ -0,0 +1,75 @@ +"""CLI: detect green→red regressions between the published matrix and a +freshly built one, so `run_daily.sh` can decide whether to enable +auto-merge on the daily docs PR. + +All real logic lives in `tests.claude_code.matrix_builder.find_regressions` +(unit-tested under `_builder_unit_tests/`); this file only does the I/O and +maps the result onto an exit code the bash caller can branch on. + +Exit codes (the bash gate depends on these exact values): + + 0 no green→red regressions -> safe to auto-merge + 3 one or more green→red regressions -> do NOT auto-merge (human review) + 2 argparse/usage error (argparse default) + +The `--old` file is allowed to be missing: on the first-ever publish there +is no baseline to regress against, so we exit 0. + +Invoked from the cron worktree (`cd`'d in by run_daily.sh) so the +`tests.claude_code` package import resolves, mirroring build_matrix.py. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from tests.claude_code.matrix_builder import find_regressions + +REGRESSION_EXIT = 3 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--old", + type=Path, + required=True, + help="currently published matrix JSON (may be absent on first publish)", + ) + parser.add_argument( + "--new", + type=Path, + required=True, + help="freshly built matrix JSON", + ) + args = parser.parse_args() + + if not args.old.exists(): + print( + "no published matrix to compare against " + "(first publish); treating as no regressions" + ) + return 0 + + old_matrix = json.loads(args.old.read_text()) + new_matrix = json.loads(args.new.read_text()) + + regressions = find_regressions(old_matrix, new_matrix) + if not regressions: + print("no green->red regressions detected") + return 0 + + print(f"detected {len(regressions)} green->red regression(s):") + for r in regressions: + line = f" - {r['feature_name']} [{r['provider']}]: pass -> fail" + if r["error"]: + line += f" ({r['error'][:160]})" + print(line) + return REGRESSION_EXIT + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/claude_code/cron_vm/litellm-compat-matrix.env.example b/tests/claude_code/cron_vm/litellm-compat-matrix.env.example index 11633810533..e8186f93021 100644 --- a/tests/claude_code/cron_vm/litellm-compat-matrix.env.example +++ b/tests/claude_code/cron_vm/litellm-compat-matrix.env.example @@ -27,18 +27,16 @@ VERTEXAI_LOCATION=global AZURE_FOUNDRY_API_KEY= AZURE_FOUNDRY_API_BASE= -# REQUIRED for publishing: PAT for the `agent-shin` user, used to push -# the daily compat-matrix branch to its fork (agent-shin/litellm-docs) -# and open the cross-repo PR against BerriAI/litellm-docs. Scopes: -# classic `repo` + `workflow`, or fine-grained on agent-shin/litellm-docs -# with Contents:RW + Pull requests:RW + Workflows:RW. +# REQUIRED for publishing: PAT for the `mateo-berri` user, who has write +# access on BerriAI/litellm-docs. Used to (a) resolve the latest stable +# release, (b) push the daily compat-matrix branch directly to +# BerriAI/litellm-docs, (c) open the same-repo PR, and (d) enable +# squash auto-merge on it. Scopes: classic `repo` + `workflow`, or +# fine-grained on BerriAI/litellm-docs with Contents:RW + Pull +# requests:RW + Workflows:RW. # Skip by setting SKIP_PUBLISH=1 (publishes nothing; only writes the # matrix JSON locally). -AGENT_SHIN_GITHUB_TOKEN= - -# Optional: lifts the unauthenticated rate limit on the GitHub Releases -# API used by `resolver.py`. Any token works (read-only). Not required. -# GITHUB_TOKEN= +GITHUB_TOKEN= # Optional overrides; defaults are sensible for the cron VM. # PROXY_PORT=4100 @@ -46,5 +44,4 @@ AGENT_SHIN_GITHUB_TOKEN= # DOCS_REPO=BerriAI/litellm-docs # DOCS_BRANCH=main # DOCS_TARGET_PATH=src/data/compatibility-matrix.json -# FORK_OWNER=agent-shin -# FORK_REPO=agent-shin/litellm-docs +# AUTO_MERGE_METHOD=squash diff --git a/tests/claude_code/http_probe.py b/tests/claude_code/http_probe.py index 5628f7af3d9..83ff1166d2e 100644 --- a/tests/claude_code/http_probe.py +++ b/tests/claude_code/http_probe.py @@ -263,3 +263,42 @@ def assert_count_tokens_shape(result: ProbeResult) -> Optional[str]: if tokens <= 0: return f"input_tokens must be positive; got {tokens}" return None + + +# Upstream signal that a model simply does not offer token counting, as +# opposed to a transport/transform regression. Some providers return a +# 400 invalid_request_error whose message says the model "is not +# supported for token counting" (observed on Vertex AI for Haiku 4.5, +# request_id req_vrtx_...). That's a capability gap, not a failure, so +# callers should record the tier as `not_applicable` rather than `fail`. +_COUNT_TOKENS_UNSUPPORTED_MARKER = "not supported for token counting" + + +def count_tokens_unsupported_reason(result: ProbeResult) -> Optional[str]: + """Return a human-readable reason if `result` is a clean upstream + "this model does not support token counting" signal, else None. + + Only a 400 whose body carries the unsupported marker qualifies; every + other non-200 (auth, transform 500s, transport errors) stays a real + failure that should flip the cell red via `assert_count_tokens_shape`. + """ + if result.status_code != 400: + return None + if _COUNT_TOKENS_UNSUPPORTED_MARKER not in (result.body or "").lower(): + return None + + # Prefer the upstream message verbatim when we can find it; the body + # shape is `{"detail": {"error": {"message": ...}}}` for the proxy's + # Anthropic-passthrough errors, with a couple of common fallbacks. + payload = result.payload if isinstance(result.payload, Mapping) else {} + detail = payload.get("detail") + error = None + if isinstance(detail, Mapping): + error = detail.get("error") + if not isinstance(error, Mapping): + error = ( + payload.get("error") if isinstance(payload.get("error"), Mapping) else None + ) + message = error.get("message") if isinstance(error, Mapping) else None + + return str(message) if message else "model does not support token counting" diff --git a/tests/claude_code/long_context_1m/test_anthropic.py b/tests/claude_code/long_context_1m/test_anthropic.py index e331a377fb6..017b8947ebd 100644 --- a/tests/claude_code/long_context_1m/test_anthropic.py +++ b/tests/claude_code/long_context_1m/test_anthropic.py @@ -29,14 +29,13 @@ Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus. Daily cost across all five providers (this row only): ~$19. Haiku 4.5 is intentionally omitted: it does not support 1M context -(its window is 200k). Reporting `not_applicable` for Haiku would -flip the entire cell to `not_applicable`, hiding genuine 1M -regressions on Sonnet/Opus; instead we exclude Haiku from the model -list entirely and let the matrix's per-cell aggregator green the -cell on Sonnet + Opus passing. This is the one row where the "all -three tiers must pass" rule is relaxed; it's relaxed structurally -(via the model list), not semantically (via not_applicable), so the -matrix builder stays unmodified. +(its window is 200k), so there's no point spending on a tier that +can't pass. We exclude Haiku from the model list and let the matrix's +per-cell aggregator green the cell on Sonnet + Opus passing. The +aggregator treats `not_applicable` as neutral, so reporting Haiku as +not_applicable would green the cell too; we skip the call outright to +save the spend. This is one of the rows where the "all three tiers +must pass" rule is relaxed for a tier that can't support the feature. The prompt is delivered via subprocess stdin rather than a positional argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt diff --git a/tests/claude_code/long_context_1m/test_azure.py b/tests/claude_code/long_context_1m/test_azure.py index 8119a15d588..b0cc2775e5a 100644 --- a/tests/claude_code/long_context_1m/test_azure.py +++ b/tests/claude_code/long_context_1m/test_azure.py @@ -29,14 +29,13 @@ Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus. Daily cost across all five providers (this row only): ~$19. Haiku 4.5 is intentionally omitted: it does not support 1M context -(its window is 200k). Reporting `not_applicable` for Haiku would -flip the entire cell to `not_applicable`, hiding genuine 1M -regressions on Sonnet/Opus; instead we exclude Haiku from the model -list entirely and let the matrix's per-cell aggregator green the -cell on Sonnet + Opus passing. This is the one row where the "all -three tiers must pass" rule is relaxed; it's relaxed structurally -(via the model list), not semantically (via not_applicable), so the -matrix builder stays unmodified. +(its window is 200k), so there's no point spending on a tier that +can't pass. We exclude Haiku from the model list and let the matrix's +per-cell aggregator green the cell on Sonnet + Opus passing. The +aggregator treats `not_applicable` as neutral, so reporting Haiku as +not_applicable would green the cell too; we skip the call outright to +save the spend. This is one of the rows where the "all three tiers +must pass" rule is relaxed for a tier that can't support the feature. The prompt is delivered via subprocess stdin rather than a positional argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt diff --git a/tests/claude_code/long_context_1m/test_bedrock_converse.py b/tests/claude_code/long_context_1m/test_bedrock_converse.py index 4fc335b2aa1..8623ebe7e6f 100644 --- a/tests/claude_code/long_context_1m/test_bedrock_converse.py +++ b/tests/claude_code/long_context_1m/test_bedrock_converse.py @@ -29,14 +29,13 @@ Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus. Daily cost across all five providers (this row only): ~$19. Haiku 4.5 is intentionally omitted: it does not support 1M context -(its window is 200k). Reporting `not_applicable` for Haiku would -flip the entire cell to `not_applicable`, hiding genuine 1M -regressions on Sonnet/Opus; instead we exclude Haiku from the model -list entirely and let the matrix's per-cell aggregator green the -cell on Sonnet + Opus passing. This is the one row where the "all -three tiers must pass" rule is relaxed; it's relaxed structurally -(via the model list), not semantically (via not_applicable), so the -matrix builder stays unmodified. +(its window is 200k), so there's no point spending on a tier that +can't pass. We exclude Haiku from the model list and let the matrix's +per-cell aggregator green the cell on Sonnet + Opus passing. The +aggregator treats `not_applicable` as neutral, so reporting Haiku as +not_applicable would green the cell too; we skip the call outright to +save the spend. This is one of the rows where the "all three tiers +must pass" rule is relaxed for a tier that can't support the feature. The prompt is delivered via subprocess stdin rather than a positional argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt diff --git a/tests/claude_code/long_context_1m/test_bedrock_invoke.py b/tests/claude_code/long_context_1m/test_bedrock_invoke.py index 0b684fb2a46..6d9d23881d1 100644 --- a/tests/claude_code/long_context_1m/test_bedrock_invoke.py +++ b/tests/claude_code/long_context_1m/test_bedrock_invoke.py @@ -29,14 +29,13 @@ Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus. Daily cost across all five providers (this row only): ~$19. Haiku 4.5 is intentionally omitted: it does not support 1M context -(its window is 200k). Reporting `not_applicable` for Haiku would -flip the entire cell to `not_applicable`, hiding genuine 1M -regressions on Sonnet/Opus; instead we exclude Haiku from the model -list entirely and let the matrix's per-cell aggregator green the -cell on Sonnet + Opus passing. This is the one row where the "all -three tiers must pass" rule is relaxed; it's relaxed structurally -(via the model list), not semantically (via not_applicable), so the -matrix builder stays unmodified. +(its window is 200k), so there's no point spending on a tier that +can't pass. We exclude Haiku from the model list and let the matrix's +per-cell aggregator green the cell on Sonnet + Opus passing. The +aggregator treats `not_applicable` as neutral, so reporting Haiku as +not_applicable would green the cell too; we skip the call outright to +save the spend. This is one of the rows where the "all three tiers +must pass" rule is relaxed for a tier that can't support the feature. The prompt is delivered via subprocess stdin rather than a positional argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt diff --git a/tests/claude_code/long_context_1m/test_vertex_ai.py b/tests/claude_code/long_context_1m/test_vertex_ai.py index 1afb7d437ab..be98bceb360 100644 --- a/tests/claude_code/long_context_1m/test_vertex_ai.py +++ b/tests/claude_code/long_context_1m/test_vertex_ai.py @@ -29,14 +29,13 @@ Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus. Daily cost across all five providers (this row only): ~$19. Haiku 4.5 is intentionally omitted: it does not support 1M context -(its window is 200k). Reporting `not_applicable` for Haiku would -flip the entire cell to `not_applicable`, hiding genuine 1M -regressions on Sonnet/Opus; instead we exclude Haiku from the model -list entirely and let the matrix's per-cell aggregator green the -cell on Sonnet + Opus passing. This is the one row where the "all -three tiers must pass" rule is relaxed; it's relaxed structurally -(via the model list), not semantically (via not_applicable), so the -matrix builder stays unmodified. +(its window is 200k), so there's no point spending on a tier that +can't pass. We exclude Haiku from the model list and let the matrix's +per-cell aggregator green the cell on Sonnet + Opus passing. The +aggregator treats `not_applicable` as neutral, so reporting Haiku as +not_applicable would green the cell too; we skip the call outright to +save the spend. This is one of the rows where the "all three tiers +must pass" rule is relaxed for a tier that can't support the feature. The prompt is delivered via subprocess stdin rather than a positional argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt diff --git a/tests/claude_code/matrix_builder.py b/tests/claude_code/matrix_builder.py index b9bf8ecc598..33db6f3d871 100644 --- a/tests/claude_code/matrix_builder.py +++ b/tests/claude_code/matrix_builder.py @@ -131,8 +131,13 @@ def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: Order of precedence (most informative wins): - Any `fail` → cell is `fail` with the first failure's error. - - `not_applicable` → cell is `not_applicable` with the reason. - - `pass` → cell is `pass`. + - `not_applicable` is *neutral*: a tier whose model/provider + genuinely doesn't support the feature is dropped, and the cell + is decided by the remaining tiers. This keeps a cell green when + the tiers that *do* support the feature all pass (e.g. Vertex AI + count_tokens passes on Sonnet/Opus but Haiku 4.5 is unsupported). + - All remaining tiers `pass` → cell is `pass`. + - Every tier was `not_applicable` → cell is `not_applicable`. - empty / nothing recognized → `not_tested`. """ if not results: @@ -142,19 +147,109 @@ def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: if r.get("status") == "fail": return {"status": "fail", "error": str(r.get("error", "test failed"))} - for r in results: - if r.get("status") == "not_applicable": - return { - "status": "not_applicable", - "reason": str(r.get("reason", "not applicable")), - } + # `not_applicable` tiers are neutral — drop them and let the tiers + # that actually exercise the feature decide the cell. + considered = [r for r in results if r.get("status") != "not_applicable"] - if all(r.get("status") == "pass" for r in results): + if not considered: + # Nothing left means every tier was not_applicable → the whole + # cell is genuinely not_applicable; surface the first reason. + reason = next( + ( + str(r.get("reason", "not applicable")) + for r in results + if r.get("status") == "not_applicable" + ), + "not applicable", + ) + return {"status": "not_applicable", "reason": reason} + + if all(r.get("status") == "pass" for r in considered): return {"status": "pass"} return {"status": "not_tested"} +def _index_cells(matrix: Mapping[str, Any]) -> Dict[tuple, Dict[str, Any]]: + """Map ``(feature_id, provider) -> cell dict`` for a built matrix. + + Cells are keyed by the *stable* feature ``id`` (not the display + ``name``, which can be reworded without changing the underlying row) + and the provider key, so two matrices built at different times line up + even if feature names drift. + """ + out: Dict[tuple, Dict[str, Any]] = {} + for feature in matrix.get("features", []) or []: + if not isinstance(feature, Mapping): + continue + feature_id = feature.get("id") + if not feature_id: + continue + providers = feature.get("providers", {}) or {} + if not isinstance(providers, Mapping): + continue + for provider, cell in providers.items(): + if isinstance(cell, Mapping): + out[(feature_id, provider)] = dict(cell) + return out + + +def find_regressions( + old_matrix: Mapping[str, Any], + new_matrix: Mapping[str, Any], +) -> List[Dict[str, str]]: + """Return the cells that flipped green→red (``pass`` → ``fail``). + + A *regression* is defined strictly: a cell that was ``pass`` in + ``old_matrix`` and is ``fail`` in ``new_matrix``. Every other + transition is intentionally *not* a regression: + + * ``red → green`` / ``green → green`` — the happy path. + * ``red → red`` — a cell that is *already* failing for an unrelated + reason (e.g. Anthropic out of API credits) must not block + publishing, otherwise the daily PR would never auto-merge until + that independent issue is fixed. + * ``green → not_tested`` / ``green → not_applicable`` — a cell going + grey is a degradation but not a *red* regression; treating a + skipped/flaky run as a hard block would create false positives. + + Cells present only in ``new_matrix`` (a newly added feature or + provider) have no baseline and therefore cannot be regressions. + + Each returned item is a flat str→str mapping so callers (the cron's + ``check_regressions.py``) can render it without further lookups: + ``feature_id``, ``feature_name``, ``provider``, ``old_status``, + ``new_status``, ``error``. + """ + old_cells = _index_cells(old_matrix) + feature_names = { + f.get("id"): str(f.get("name", f.get("id"))) + for f in new_matrix.get("features", []) or [] + if isinstance(f, Mapping) and f.get("id") + } + + regressions: List[Dict[str, str]] = [] + for (feature_id, provider), new_cell in sorted( + _index_cells(new_matrix).items(), key=lambda kv: (kv[0][0], kv[0][1]) + ): + if new_cell.get("status") != "fail": + continue + old_cell = old_cells.get((feature_id, provider)) + if old_cell is None or old_cell.get("status") != "pass": + continue + regressions.append( + { + "feature_id": str(feature_id), + "feature_name": feature_names.get(feature_id, str(feature_id)), + "provider": str(provider), + "old_status": "pass", + "new_status": "fail", + "error": str(new_cell.get("error", "")), + } + ) + return regressions + + def build_from_paths( *, manifest_path: Path,