chore(claude_code): commit remaining deployed working-tree state

Everything under tests/claude_code/ that the daily cron run shims into
the stable-tag worktree but that existed only on the cron VM's disk:
check_regressions.py (required by run_daily.sh's auto-merge gate, was
untracked), the count_tokens and long_context_1m test refinements,
http_probe/matrix_builder updates, and the cron_vm README + env
example matching the deployed direct-publish flow.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

tests/claude_code/_builder_unit_tests/test_matrix_builder.py

@@ -20,6 +20,7 @@ from tests.claude_code.matrix_builder import (
     ResultsError,
     build_from_paths,
     build_matrix,
+    find_regressions,
     load_manifest,
     load_results,
 )
@@ -90,6 +91,66 @@ def test_build_matrix_any_fail_makes_cell_fail():
     assert cell["error"] == "[claude-opus-4-7] timeout"
 
 
+def _single_cell_matrix(results):
+    """Build a 1x1 matrix and return its only cell. Helper for the
+    not_applicable aggregation tests below."""
+    manifest = {
+        "schema_version": "1",
+        "providers": ["vertex_ai"],
+        "features": [{"id": "f", "name": "F"}],
+    }
+    matrix = build_matrix(
+        manifest=manifest,
+        results=[
+            {"feature_id": "f", "provider": "vertex_ai", "result": r} for r in results
+        ],
+        litellm_version="v",
+        claude_code_version="c",
+        generated_at="t",
+    )
+    return matrix["features"][0]["providers"]["vertex_ai"]
+
+
+def test_build_matrix_not_applicable_is_neutral_when_others_pass():
+    """A tier that doesn't support the feature (`not_applicable`) must not
+    drag down a cell whose supported tiers all pass. Models the real
+    Vertex AI count_tokens case: Haiku 4.5 is unsupported, Sonnet/Opus
+    pass → the cell is green."""
+    cell = _single_cell_matrix(
+        [
+            {"status": "not_applicable", "reason": "[haiku] not supported"},
+            {"status": "pass"},
+            {"status": "pass"},
+        ]
+    )
+    assert cell == {"status": "pass"}
+
+
+def test_build_matrix_all_not_applicable_makes_cell_not_applicable():
+    """If *every* tier is not_applicable, the cell is not_applicable and
+    surfaces the first reason."""
+    cell = _single_cell_matrix(
+        [
+            {"status": "not_applicable", "reason": "first reason"},
+            {"status": "not_applicable", "reason": "second reason"},
+        ]
+    )
+    assert cell == {"status": "not_applicable", "reason": "first reason"}
+
+
+def test_build_matrix_fail_beats_not_applicable():
+    """A genuine failure still reds the cell even when another tier is
+    not_applicable — failures win over the neutral skip."""
+    cell = _single_cell_matrix(
+        [
+            {"status": "not_applicable", "reason": "[haiku] not supported"},
+            {"status": "fail", "error": "[opus] regression"},
+        ]
+    )
+    assert cell["status"] == "fail"
+    assert cell["error"] == "[opus] regression"
+
+
 def test_build_matrix_fills_not_tested_for_missing_cells():
     manifest = {
         "schema_version": "1",
@@ -291,6 +352,137 @@ def test_build_matrix_1x5_grid_one_failing_model_breaks_cell():
     assert "claude-opus-4-7-bedrock-invoke" in cell["error"]
 
 
+def _matrix(cells, *, names=None):
+    """Build a minimal matrix dict from a {(feature_id, provider): status}
+    or {(feature_id, provider): cell_dict} mapping. Helper for the
+    find_regressions tests below."""
+    names = names or {}
+    features = {}
+    for (feature_id, provider), value in cells.items():
+        cell = {"status": value} if isinstance(value, str) else dict(value)
+        features.setdefault(feature_id, {})[provider] = cell
+    return {
+        "features": [
+            {
+                "id": feature_id,
+                "name": names.get(feature_id, feature_id.upper()),
+                "providers": providers,
+            }
+            for feature_id, providers in features.items()
+        ]
+    }
+
+
+def test_find_regressions_flags_pass_to_fail():
+    old = _matrix({("vision", "anthropic"): "pass"})
+    new = _matrix(
+        {("vision", "anthropic"): {"status": "fail", "error": "credit balance too low"}}
+    )
+    regressions = find_regressions(old, new)
+    assert len(regressions) == 1
+    r = regressions[0]
+    assert r["feature_id"] == "vision"
+    assert r["provider"] == "anthropic"
+    assert r["old_status"] == "pass"
+    assert r["new_status"] == "fail"
+    assert r["error"] == "credit balance too low"
+
+
+def test_find_regressions_ignores_red_to_red():
+    """An already-failing cell that stays failing is NOT a regression — a
+    provider that's independently broken (e.g. out of credits) must not
+    block the daily auto-merge forever."""
+    old = _matrix({("vision", "anthropic"): "fail"})
+    new = _matrix({("vision", "anthropic"): "fail"})
+    assert find_regressions(old, new) == []
+
+
+def test_find_regressions_ignores_improvements_and_steady_green():
+    old = _matrix(
+        {
+            ("vision", "anthropic"): "fail",  # red -> green
+            ("tool_use", "azure"): "pass",  # green -> green
+        }
+    )
+    new = _matrix(
+        {
+            ("vision", "anthropic"): "pass",
+            ("tool_use", "azure"): "pass",
+        }
+    )
+    assert find_regressions(old, new) == []
+
+
+def test_find_regressions_ignores_green_to_grey():
+    """green→not_tested / green→not_applicable are degradations but not
+    *red* regressions; we deliberately don't block on them."""
+    old = _matrix(
+        {
+            ("vision", "azure"): "pass",
+            ("tool_use", "azure"): "pass",
+        }
+    )
+    new = _matrix(
+        {
+            ("vision", "azure"): "not_tested",
+            ("tool_use", "azure"): {"status": "not_applicable", "reason": "skip"},
+        }
+    )
+    assert find_regressions(old, new) == []
+
+
+def test_find_regressions_ignores_new_cells_without_baseline():
+    """A cell only present in the new matrix (new feature/provider) has no
+    baseline, so a fail there can't be a regression."""
+    old = _matrix({("vision", "anthropic"): "pass"})
+    new = _matrix(
+        {
+            ("vision", "anthropic"): "pass",
+            ("brand_new_feature", "anthropic"): "fail",
+        }
+    )
+    assert find_regressions(old, new) == []
+
+
+def test_find_regressions_matches_by_id_not_name():
+    """Renaming a feature's display name must not hide a regression: cells
+    are matched on the stable id."""
+    old = _matrix({("thinking", "anthropic"): "pass"}, names={"thinking": "Old Name"})
+    new = _matrix(
+        {("thinking", "anthropic"): "fail"}, names={"thinking": "Totally New Name"}
+    )
+    regressions = find_regressions(old, new)
+    assert len(regressions) == 1
+    assert regressions[0]["feature_id"] == "thinking"
+    assert regressions[0]["feature_name"] == "Totally New Name"
+
+
+def test_find_regressions_reports_multiple_sorted():
+    old = _matrix(
+        {
+            ("vision", "anthropic"): "pass",
+            ("tool_use", "anthropic"): "pass",
+            ("vision", "azure"): "pass",
+        }
+    )
+    new = _matrix(
+        {
+            ("vision", "anthropic"): "fail",
+            ("tool_use", "anthropic"): "fail",
+            ("vision", "azure"): "pass",  # stays green
+        }
+    )
+    regressions = find_regressions(old, new)
+    keys = [(r["feature_id"], r["provider"]) for r in regressions]
+    assert keys == [("tool_use", "anthropic"), ("vision", "anthropic")]
+
+
+def test_find_regressions_empty_old_matrix_is_safe():
+    """No baseline at all (first publish) yields no regressions."""
+    new = _matrix({("vision", "anthropic"): "fail"})
+    assert find_regressions({}, new) == []
+
+
 def test_build_from_paths_writes_output(tmp_path):
     out = tmp_path / "compatibility-matrix.json"
     matrix = build_from_paths(

tests/claude_code/count_tokens/test_anthropic.py

@@ -43,6 +43,7 @@ import pytest
 
 from tests.claude_code.http_probe import (
     assert_count_tokens_shape,
+    count_tokens_unsupported_reason,
     probe_count_tokens,
 )
 
@@ -78,9 +79,18 @@ def test_count_tokens_anthropic(compat_result):
 
     failures = []
     for model in ANTHROPIC_MODELS:
-        result = probe_count_tokens(
-            base_url=base_url, api_key=api_key, model=model
-        )
+        result = probe_count_tokens(base_url=base_url, api_key=api_key, model=model)
+
+        unsupported = count_tokens_unsupported_reason(result)
+        if unsupported is not None:
+            compat_result.add(
+                {
+                    "status": "not_applicable",
+                    "reason": f"[{model}] {unsupported}",
+                }
+            )
+            continue
+
         shape_error = assert_count_tokens_shape(result)
         if shape_error is not None:
             error = f"[{model}] count_tokens probe failed: {shape_error}"

tests/claude_code/count_tokens/test_azure.py

@@ -43,6 +43,7 @@ import pytest
 
 from tests.claude_code.http_probe import (
     assert_count_tokens_shape,
+    count_tokens_unsupported_reason,
     probe_count_tokens,
 )
 
@@ -78,9 +79,18 @@ def test_count_tokens_azure(compat_result):
 
     failures = []
     for model in AZURE_MODELS:
-        result = probe_count_tokens(
-            base_url=base_url, api_key=api_key, model=model
-        )
+        result = probe_count_tokens(base_url=base_url, api_key=api_key, model=model)
+
+        unsupported = count_tokens_unsupported_reason(result)
+        if unsupported is not None:
+            compat_result.add(
+                {
+                    "status": "not_applicable",
+                    "reason": f"[{model}] {unsupported}",
+                }
+            )
+            continue
+
         shape_error = assert_count_tokens_shape(result)
         if shape_error is not None:
             error = f"[{model}] count_tokens probe failed: {shape_error}"

tests/claude_code/count_tokens/test_bedrock_converse.py

@@ -43,6 +43,7 @@ import pytest
 
 from tests.claude_code.http_probe import (
     assert_count_tokens_shape,
+    count_tokens_unsupported_reason,
     probe_count_tokens,
 )
 
@@ -78,9 +79,18 @@ def test_count_tokens_bedrock_converse(compat_result):
 
     failures = []
     for model in BEDROCK_CONVERSE_MODELS:
-        result = probe_count_tokens(
-            base_url=base_url, api_key=api_key, model=model
-        )
+        result = probe_count_tokens(base_url=base_url, api_key=api_key, model=model)
+
+        unsupported = count_tokens_unsupported_reason(result)
+        if unsupported is not None:
+            compat_result.add(
+                {
+                    "status": "not_applicable",
+                    "reason": f"[{model}] {unsupported}",
+                }
+            )
+            continue
+
         shape_error = assert_count_tokens_shape(result)
         if shape_error is not None:
             error = f"[{model}] count_tokens probe failed: {shape_error}"

tests/claude_code/count_tokens/test_bedrock_invoke.py

@@ -43,6 +43,7 @@ import pytest
 
 from tests.claude_code.http_probe import (
     assert_count_tokens_shape,
+    count_tokens_unsupported_reason,
     probe_count_tokens,
 )
 
@@ -78,9 +79,18 @@ def test_count_tokens_bedrock_invoke(compat_result):
 
     failures = []
     for model in BEDROCK_INVOKE_MODELS:
-        result = probe_count_tokens(
-            base_url=base_url, api_key=api_key, model=model
-        )
+        result = probe_count_tokens(base_url=base_url, api_key=api_key, model=model)
+
+        unsupported = count_tokens_unsupported_reason(result)
+        if unsupported is not None:
+            compat_result.add(
+                {
+                    "status": "not_applicable",
+                    "reason": f"[{model}] {unsupported}",
+                }
+            )
+            continue
+
         shape_error = assert_count_tokens_shape(result)
         if shape_error is not None:
             error = f"[{model}] count_tokens probe failed: {shape_error}"

tests/claude_code/count_tokens/test_vertex_ai.py

@@ -43,6 +43,7 @@ import pytest
 
 from tests.claude_code.http_probe import (
     assert_count_tokens_shape,
+    count_tokens_unsupported_reason,
     probe_count_tokens,
 )
 
@@ -78,9 +79,18 @@ def test_count_tokens_vertex_ai(compat_result):
 
     failures = []
     for model in VERTEX_AI_MODELS:
-        result = probe_count_tokens(
-            base_url=base_url, api_key=api_key, model=model
-        )
+        result = probe_count_tokens(base_url=base_url, api_key=api_key, model=model)
+
+        unsupported = count_tokens_unsupported_reason(result)
+        if unsupported is not None:
+            compat_result.add(
+                {
+                    "status": "not_applicable",
+                    "reason": f"[{model}] {unsupported}",
+                }
+            )
+            continue
+
         shape_error = assert_count_tokens_shape(result)
         if shape_error is not None:
             error = f"[{model}] count_tokens probe failed: {shape_error}"

tests/claude_code/cron_vm/README.md

@@ -24,6 +24,7 @@ Action. Trade-offs:
 | --- | --- |
 | `run_daily.sh` | The actual cron job. Resolves versions, updates the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR. |
 | `build_matrix.py` | Tiny Python CLI that wraps `tests.claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. |
+| `check_regressions.py` | Tiny Python CLI that wraps `tests.claude_code.matrix_builder.find_regressions`. Diffs the freshly built matrix against the currently-published one and exits `3` if any cell flipped green→red, which gates auto-merge. |
 | `litellm-compat-matrix.service` | systemd oneshot that invokes `run_daily.sh`. |
 | `litellm-compat-matrix.timer` | `OnCalendar=*-*-* 06:00:00 UTC`, `Persistent=true`. |
 | `litellm-compat-matrix.env.example` | Template for `/etc/litellm-compat-matrix.env`. |
@@ -52,10 +53,24 @@ Action. Trade-offs:
 7. **Opens or updates a docs PR**: `gh repo clone` of `litellm-docs`
    into a tempdir, deterministic head branch
    (`compat-matrix/<litellm-version>-<claude-code-version>-<UTC-date>`),
-   `--force-with-lease` push, `gh pr create`. A re-run on the same
-   day fast-forwards the existing branch and `gh pr create` no-ops
-   ("a pull request for branch ... already exists" is treated as
-   success).
+   `--force` push **directly to `BerriAI/litellm-docs`** (the
+   `mateo-berri` token has write access, so this is a same-repo branch,
+   not a fork), `gh pr create`. A re-run on the same day fast-forwards
+   the existing branch and `gh pr create` no-ops ("a pull request for
+   branch ... already exists" is treated as success). These PRs are no
+   longer gated on a second human review.
+8. **Gates auto-merge on a regression check**: before enabling
+   auto-merge, `check_regressions.py` diffs the new matrix against the
+   one currently on `main`. Auto-merge (`gh pr merge --auto --squash`)
+   is only enabled when **no cell flipped green→red** — i.e. every
+   transition is red→green, green→green, or red→red. A pre-existing red
+   cell (e.g. a provider that's out of API credits) is `red→red` and
+   does **not** block; only a `pass`→`fail` flip does. When a regression
+   is detected the PR is still opened/updated (with a warning banner
+   naming the offending cells) but auto-merge is left **off** — and any
+   auto-merge a prior same-day run enabled is explicitly disabled — so a
+   human reviews before it lands on the public table. The check fails
+   *closed*: if it errors, auto-merge is withheld.
 
 ## One-time VM setup
 
@@ -129,9 +144,13 @@ sudo systemctl disable --now litellm-compat-matrix.timer
 - **`uv sync --frozen` requires the resolved tag to be tagged on
   GitHub.** If the latest stable release was made but not pushed as a
   git tag, the `git checkout` step fails. Push the tag, then rerun.
-- **`gh auth` token rotation is your problem.** The cron does not
-  refresh the token; if the bot account's PAT expires the run will
-  fail at `gh repo clone` with a 401. Re-run `gh auth login`.
+- **`GITHUB_TOKEN` rotation is your problem.** The cron does not
+  refresh the token; if `mateo-berri`'s PAT in
+  `/etc/litellm-compat-matrix.env` expires, the run fails at the
+  `git push`/`gh pr create` step with a 401 ("Bad credentials" /
+  "Authentication failed"). Mint a fresh PAT and update the env file.
+  The token needs write access to `BerriAI/litellm-docs` (classic
+  `repo` scope, or fine-grained Contents:RW + Pull requests:RW).
 - **First run after upgrading the Claude Code CLI is the riskiest one.**
   If the new CLI changes its wire format the matrix run can produce
   systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI

tests/claude_code/cron_vm/check_regressions.py

@@ -0,0 +1,75 @@
+"""CLI: detect green→red regressions between the published matrix and a
+freshly built one, so `run_daily.sh` can decide whether to enable
+auto-merge on the daily docs PR.
+
+All real logic lives in `tests.claude_code.matrix_builder.find_regressions`
+(unit-tested under `_builder_unit_tests/`); this file only does the I/O and
+maps the result onto an exit code the bash caller can branch on.
+
+Exit codes (the bash gate depends on these exact values):
+
+  0  no green→red regressions          -> safe to auto-merge
+  3  one or more green→red regressions  -> do NOT auto-merge (human review)
+  2  argparse/usage error (argparse default)
+
+The `--old` file is allowed to be missing: on the first-ever publish there
+is no baseline to regress against, so we exit 0.
+
+Invoked from the cron worktree (`cd`'d in by run_daily.sh) so the
+`tests.claude_code` package import resolves, mirroring build_matrix.py.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from tests.claude_code.matrix_builder import find_regressions
+
+REGRESSION_EXIT = 3
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--old",
+        type=Path,
+        required=True,
+        help="currently published matrix JSON (may be absent on first publish)",
+    )
+    parser.add_argument(
+        "--new",
+        type=Path,
+        required=True,
+        help="freshly built matrix JSON",
+    )
+    args = parser.parse_args()
+
+    if not args.old.exists():
+        print(
+            "no published matrix to compare against "
+            "(first publish); treating as no regressions"
+        )
+        return 0
+
+    old_matrix = json.loads(args.old.read_text())
+    new_matrix = json.loads(args.new.read_text())
+
+    regressions = find_regressions(old_matrix, new_matrix)
+    if not regressions:
+        print("no green->red regressions detected")
+        return 0
+
+    print(f"detected {len(regressions)} green->red regression(s):")
+    for r in regressions:
+        line = f"  - {r['feature_name']} [{r['provider']}]: pass -> fail"
+        if r["error"]:
+            line += f" ({r['error'][:160]})"
+        print(line)
+    return REGRESSION_EXIT
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/claude_code/cron_vm/litellm-compat-matrix.env.example

@@ -27,18 +27,16 @@ VERTEXAI_LOCATION=global
 AZURE_FOUNDRY_API_KEY=
 AZURE_FOUNDRY_API_BASE=
 
-# REQUIRED for publishing: PAT for the `agent-shin` user, used to push
-# the daily compat-matrix branch to its fork (agent-shin/litellm-docs)
-# and open the cross-repo PR against BerriAI/litellm-docs. Scopes:
-# classic `repo` + `workflow`, or fine-grained on agent-shin/litellm-docs
-# with Contents:RW + Pull requests:RW + Workflows:RW.
+# REQUIRED for publishing: PAT for the `mateo-berri` user, who has write
+# access on BerriAI/litellm-docs. Used to (a) resolve the latest stable
+# release, (b) push the daily compat-matrix branch directly to
+# BerriAI/litellm-docs, (c) open the same-repo PR, and (d) enable
+# squash auto-merge on it. Scopes: classic `repo` + `workflow`, or
+# fine-grained on BerriAI/litellm-docs with Contents:RW + Pull
+# requests:RW + Workflows:RW.
 # Skip by setting SKIP_PUBLISH=1 (publishes nothing; only writes the
 # matrix JSON locally).
-AGENT_SHIN_GITHUB_TOKEN=
-
-# Optional: lifts the unauthenticated rate limit on the GitHub Releases
-# API used by `resolver.py`. Any token works (read-only). Not required.
-# GITHUB_TOKEN=
+GITHUB_TOKEN=
 
 # Optional overrides; defaults are sensible for the cron VM.
 # PROXY_PORT=4100
@@ -46,5 +44,4 @@ AGENT_SHIN_GITHUB_TOKEN=
 # DOCS_REPO=BerriAI/litellm-docs
 # DOCS_BRANCH=main
 # DOCS_TARGET_PATH=src/data/compatibility-matrix.json
-# FORK_OWNER=agent-shin
-# FORK_REPO=agent-shin/litellm-docs
+# AUTO_MERGE_METHOD=squash

tests/claude_code/http_probe.py

@@ -263,3 +263,42 @@ def assert_count_tokens_shape(result: ProbeResult) -> Optional[str]:
     if tokens <= 0:
         return f"input_tokens must be positive; got {tokens}"
     return None
+
+
+# Upstream signal that a model simply does not offer token counting, as
+# opposed to a transport/transform regression. Some providers return a
+# 400 invalid_request_error whose message says the model "is not
+# supported for token counting" (observed on Vertex AI for Haiku 4.5,
+# request_id req_vrtx_...). That's a capability gap, not a failure, so
+# callers should record the tier as `not_applicable` rather than `fail`.
+_COUNT_TOKENS_UNSUPPORTED_MARKER = "not supported for token counting"
+
+
+def count_tokens_unsupported_reason(result: ProbeResult) -> Optional[str]:
+    """Return a human-readable reason if `result` is a clean upstream
+    "this model does not support token counting" signal, else None.
+
+    Only a 400 whose body carries the unsupported marker qualifies; every
+    other non-200 (auth, transform 500s, transport errors) stays a real
+    failure that should flip the cell red via `assert_count_tokens_shape`.
+    """
+    if result.status_code != 400:
+        return None
+    if _COUNT_TOKENS_UNSUPPORTED_MARKER not in (result.body or "").lower():
+        return None
+
+    # Prefer the upstream message verbatim when we can find it; the body
+    # shape is `{"detail": {"error": {"message": ...}}}` for the proxy's
+    # Anthropic-passthrough errors, with a couple of common fallbacks.
+    payload = result.payload if isinstance(result.payload, Mapping) else {}
+    detail = payload.get("detail")
+    error = None
+    if isinstance(detail, Mapping):
+        error = detail.get("error")
+    if not isinstance(error, Mapping):
+        error = (
+            payload.get("error") if isinstance(payload.get("error"), Mapping) else None
+        )
+    message = error.get("message") if isinstance(error, Mapping) else None
+
+    return str(message) if message else "model does not support token counting"

tests/claude_code/long_context_1m/test_anthropic.py

@@ -29,14 +29,13 @@ Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus.
 Daily cost across all five providers (this row only): ~$19.
 
 Haiku 4.5 is intentionally omitted: it does not support 1M context
-(its window is 200k). Reporting `not_applicable` for Haiku would
-flip the entire cell to `not_applicable`, hiding genuine 1M
-regressions on Sonnet/Opus; instead we exclude Haiku from the model
-list entirely and let the matrix's per-cell aggregator green the
-cell on Sonnet + Opus passing. This is the one row where the "all
-three tiers must pass" rule is relaxed; it's relaxed structurally
-(via the model list), not semantically (via not_applicable), so the
-matrix builder stays unmodified.
+(its window is 200k), so there's no point spending on a tier that
+can't pass. We exclude Haiku from the model list and let the matrix's
+per-cell aggregator green the cell on Sonnet + Opus passing. The
+aggregator treats `not_applicable` as neutral, so reporting Haiku as
+not_applicable would green the cell too; we skip the call outright to
+save the spend. This is one of the rows where the "all three tiers
+must pass" rule is relaxed for a tier that can't support the feature.
 
 The prompt is delivered via subprocess stdin rather than a positional
 argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt

tests/claude_code/long_context_1m/test_azure.py

@@ -29,14 +29,13 @@ Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus.
 Daily cost across all five providers (this row only): ~$19.
 
 Haiku 4.5 is intentionally omitted: it does not support 1M context
-(its window is 200k). Reporting `not_applicable` for Haiku would
-flip the entire cell to `not_applicable`, hiding genuine 1M
-regressions on Sonnet/Opus; instead we exclude Haiku from the model
-list entirely and let the matrix's per-cell aggregator green the
-cell on Sonnet + Opus passing. This is the one row where the "all
-three tiers must pass" rule is relaxed; it's relaxed structurally
-(via the model list), not semantically (via not_applicable), so the
-matrix builder stays unmodified.
+(its window is 200k), so there's no point spending on a tier that
+can't pass. We exclude Haiku from the model list and let the matrix's
+per-cell aggregator green the cell on Sonnet + Opus passing. The
+aggregator treats `not_applicable` as neutral, so reporting Haiku as
+not_applicable would green the cell too; we skip the call outright to
+save the spend. This is one of the rows where the "all three tiers
+must pass" rule is relaxed for a tier that can't support the feature.
 
 The prompt is delivered via subprocess stdin rather than a positional
 argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt

tests/claude_code/long_context_1m/test_bedrock_converse.py

@@ -29,14 +29,13 @@ Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus.
 Daily cost across all five providers (this row only): ~$19.
 
 Haiku 4.5 is intentionally omitted: it does not support 1M context
-(its window is 200k). Reporting `not_applicable` for Haiku would
-flip the entire cell to `not_applicable`, hiding genuine 1M
-regressions on Sonnet/Opus; instead we exclude Haiku from the model
-list entirely and let the matrix's per-cell aggregator green the
-cell on Sonnet + Opus passing. This is the one row where the "all
-three tiers must pass" rule is relaxed; it's relaxed structurally
-(via the model list), not semantically (via not_applicable), so the
-matrix builder stays unmodified.
+(its window is 200k), so there's no point spending on a tier that
+can't pass. We exclude Haiku from the model list and let the matrix's
+per-cell aggregator green the cell on Sonnet + Opus passing. The
+aggregator treats `not_applicable` as neutral, so reporting Haiku as
+not_applicable would green the cell too; we skip the call outright to
+save the spend. This is one of the rows where the "all three tiers
+must pass" rule is relaxed for a tier that can't support the feature.
 
 The prompt is delivered via subprocess stdin rather than a positional
 argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt

tests/claude_code/long_context_1m/test_bedrock_invoke.py

@@ -29,14 +29,13 @@ Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus.
 Daily cost across all five providers (this row only): ~$19.
 
 Haiku 4.5 is intentionally omitted: it does not support 1M context
-(its window is 200k). Reporting `not_applicable` for Haiku would
-flip the entire cell to `not_applicable`, hiding genuine 1M
-regressions on Sonnet/Opus; instead we exclude Haiku from the model
-list entirely and let the matrix's per-cell aggregator green the
-cell on Sonnet + Opus passing. This is the one row where the "all
-three tiers must pass" rule is relaxed; it's relaxed structurally
-(via the model list), not semantically (via not_applicable), so the
-matrix builder stays unmodified.
+(its window is 200k), so there's no point spending on a tier that
+can't pass. We exclude Haiku from the model list and let the matrix's
+per-cell aggregator green the cell on Sonnet + Opus passing. The
+aggregator treats `not_applicable` as neutral, so reporting Haiku as
+not_applicable would green the cell too; we skip the call outright to
+save the spend. This is one of the rows where the "all three tiers
+must pass" rule is relaxed for a tier that can't support the feature.
 
 The prompt is delivered via subprocess stdin rather than a positional
 argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt

tests/claude_code/long_context_1m/test_vertex_ai.py

@@ -29,14 +29,13 @@ Per-cell cost at 210k input tokens: ~$0.63 Sonnet, ~$3.15 Opus.
 Daily cost across all five providers (this row only): ~$19.
 
 Haiku 4.5 is intentionally omitted: it does not support 1M context
-(its window is 200k). Reporting `not_applicable` for Haiku would
-flip the entire cell to `not_applicable`, hiding genuine 1M
-regressions on Sonnet/Opus; instead we exclude Haiku from the model
-list entirely and let the matrix's per-cell aggregator green the
-cell on Sonnet + Opus passing. This is the one row where the "all
-three tiers must pass" rule is relaxed; it's relaxed structurally
-(via the model list), not semantically (via not_applicable), so the
-matrix builder stays unmodified.
+(its window is 200k), so there's no point spending on a tier that
+can't pass. We exclude Haiku from the model list and let the matrix's
+per-cell aggregator green the cell on Sonnet + Opus passing. The
+aggregator treats `not_applicable` as neutral, so reporting Haiku as
+not_applicable would green the cell too; we skip the call outright to
+save the spend. This is one of the rows where the "all three tiers
+must pass" rule is relaxed for a tier that can't support the feature.
 
 The prompt is delivered via subprocess stdin rather than a positional
 argument. ARG_MAX on Linux is typically 2MB and an 840KB prompt

tests/claude_code/matrix_builder.py

@@ -131,8 +131,13 @@ def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]:
 
     Order of precedence (most informative wins):
       - Any `fail` → cell is `fail` with the first failure's error.
-      - `not_applicable` → cell is `not_applicable` with the reason.
-      - `pass` → cell is `pass`.
+      - `not_applicable` is *neutral*: a tier whose model/provider
+        genuinely doesn't support the feature is dropped, and the cell
+        is decided by the remaining tiers. This keeps a cell green when
+        the tiers that *do* support the feature all pass (e.g. Vertex AI
+        count_tokens passes on Sonnet/Opus but Haiku 4.5 is unsupported).
+      - All remaining tiers `pass` → cell is `pass`.
+      - Every tier was `not_applicable` → cell is `not_applicable`.
       - empty / nothing recognized → `not_tested`.
     """
     if not results:
@@ -142,19 +147,109 @@ def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]:
         if r.get("status") == "fail":
             return {"status": "fail", "error": str(r.get("error", "test failed"))}
 
-    for r in results:
-        if r.get("status") == "not_applicable":
-            return {
-                "status": "not_applicable",
-                "reason": str(r.get("reason", "not applicable")),
-            }
+    # `not_applicable` tiers are neutral — drop them and let the tiers
+    # that actually exercise the feature decide the cell.
+    considered = [r for r in results if r.get("status") != "not_applicable"]
 
-    if all(r.get("status") == "pass" for r in results):
+    if not considered:
+        # Nothing left means every tier was not_applicable → the whole
+        # cell is genuinely not_applicable; surface the first reason.
+        reason = next(
+            (
+                str(r.get("reason", "not applicable"))
+                for r in results
+                if r.get("status") == "not_applicable"
+            ),
+            "not applicable",
+        )
+        return {"status": "not_applicable", "reason": reason}
+
+    if all(r.get("status") == "pass" for r in considered):
         return {"status": "pass"}
 
     return {"status": "not_tested"}
 
 
+def _index_cells(matrix: Mapping[str, Any]) -> Dict[tuple, Dict[str, Any]]:
+    """Map ``(feature_id, provider) -> cell dict`` for a built matrix.
+
+    Cells are keyed by the *stable* feature ``id`` (not the display
+    ``name``, which can be reworded without changing the underlying row)
+    and the provider key, so two matrices built at different times line up
+    even if feature names drift.
+    """
+    out: Dict[tuple, Dict[str, Any]] = {}
+    for feature in matrix.get("features", []) or []:
+        if not isinstance(feature, Mapping):
+            continue
+        feature_id = feature.get("id")
+        if not feature_id:
+            continue
+        providers = feature.get("providers", {}) or {}
+        if not isinstance(providers, Mapping):
+            continue
+        for provider, cell in providers.items():
+            if isinstance(cell, Mapping):
+                out[(feature_id, provider)] = dict(cell)
+    return out
+
+
+def find_regressions(
+    old_matrix: Mapping[str, Any],
+    new_matrix: Mapping[str, Any],
+) -> List[Dict[str, str]]:
+    """Return the cells that flipped green→red (``pass`` → ``fail``).
+
+    A *regression* is defined strictly: a cell that was ``pass`` in
+    ``old_matrix`` and is ``fail`` in ``new_matrix``. Every other
+    transition is intentionally *not* a regression:
+
+      * ``red → green`` / ``green → green`` — the happy path.
+      * ``red → red`` — a cell that is *already* failing for an unrelated
+        reason (e.g. Anthropic out of API credits) must not block
+        publishing, otherwise the daily PR would never auto-merge until
+        that independent issue is fixed.
+      * ``green → not_tested`` / ``green → not_applicable`` — a cell going
+        grey is a degradation but not a *red* regression; treating a
+        skipped/flaky run as a hard block would create false positives.
+
+    Cells present only in ``new_matrix`` (a newly added feature or
+    provider) have no baseline and therefore cannot be regressions.
+
+    Each returned item is a flat str→str mapping so callers (the cron's
+    ``check_regressions.py``) can render it without further lookups:
+    ``feature_id``, ``feature_name``, ``provider``, ``old_status``,
+    ``new_status``, ``error``.
+    """
+    old_cells = _index_cells(old_matrix)
+    feature_names = {
+        f.get("id"): str(f.get("name", f.get("id")))
+        for f in new_matrix.get("features", []) or []
+        if isinstance(f, Mapping) and f.get("id")
+    }
+
+    regressions: List[Dict[str, str]] = []
+    for (feature_id, provider), new_cell in sorted(
+        _index_cells(new_matrix).items(), key=lambda kv: (kv[0][0], kv[0][1])
+    ):
+        if new_cell.get("status") != "fail":
+            continue
+        old_cell = old_cells.get((feature_id, provider))
+        if old_cell is None or old_cell.get("status") != "pass":
+            continue
+        regressions.append(
+            {
+                "feature_id": str(feature_id),
+                "feature_name": feature_names.get(feature_id, str(feature_id)),
+                "provider": str(provider),
+                "old_status": "pass",
+                "new_status": "fail",
+                "error": str(new_cell.get("error", "")),
+            }
+        )
+    return regressions
+
+
 def build_from_paths(
     *,
     manifest_path: Path,