From 5d121fa697969bd69c2d4aa1616465f7721d462b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 19 May 2026 03:01:19 +0000 Subject: [PATCH] =?UTF-8?q?fix(matrix-builder):=20greptile=20=E2=80=94=20s?= =?UTF-8?q?urface=20pass=20over=20not=5Fapplicable=20when=20mixed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a cell aggregates per-model results across three tiers (Haiku/Sonnet/Opus), a mix of (pass, not_applicable) used to short-circuit to not_applicable on the first NA match, discarding the passing tiers from the published matrix. Treat not_applicable like not_tested when mixed with pass: only return not_applicable when every observed row is NA. Otherwise any pass surfaces as pass, so the cell answers 'does this feature work on this provider?' truthfully when at least one tier passes. Add two regression tests pinning the new precedence: - mixed pass + NA → pass - all NA → not_applicable (with first reason) --- .../test_matrix_builder.py | 72 +++++++++++++++++++ tests/claude_code/matrix_builder.py | 25 ++++--- 2 files changed, 87 insertions(+), 10 deletions(-) diff --git a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py index 2644e092e48..8e1eef17811 100644 --- a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py +++ b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -189,6 +189,78 @@ def test_build_matrix_all_not_tested_stays_not_tested(): assert matrix["features"][0]["providers"]["anthropic"] == {"status": "not_tested"} +def test_build_matrix_mixed_pass_and_not_applicable_surfaces_pass(): + """A `not_applicable` row mixed with `pass` rows must surface as + `pass`, not `not_applicable`. The published cell answers "does this + feature work on this provider?"; if any tier passes, the feature + works there. Discarding passing tiers because one tier is NA would + misrepresent the cell as unsupported. + """ + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + { + "feature_id": "f", + "provider": "anthropic", + "result": { + "status": "not_applicable", + "reason": "haiku does not support extended thinking", + }, + }, + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"} + + +def test_build_matrix_all_not_applicable_stays_not_applicable(): + """When every observed row is `not_applicable`, the cell remains + `not_applicable` and the first row's reason carries through to the + published matrix. + """ + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + { + "feature_id": "f", + "provider": "anthropic", + "result": { + "status": "not_applicable", + "reason": "feature unsupported on this provider", + }, + }, + { + "feature_id": "f", + "provider": "anthropic", + "result": {"status": "not_applicable", "reason": "ditto"}, + }, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["features"][0]["providers"]["anthropic"] == { + "status": "not_applicable", + "reason": "feature unsupported on this provider", + } + + def test_build_matrix_fills_not_tested_for_missing_cells(): manifest = { "schema_version": "1", diff --git a/tests/claude_code/matrix_builder.py b/tests/claude_code/matrix_builder.py index 8555805dab2..5641e488da2 100644 --- a/tests/claude_code/matrix_builder.py +++ b/tests/claude_code/matrix_builder.py @@ -133,8 +133,14 @@ def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: - Any `fail` → cell is `fail` with every failing model's error joined by `"; "` so a multi-tier breakage doesn't silently hide all but the first error from the published matrix. - - `not_applicable` → cell is `not_applicable` with the reason. - - `pass` → cell is `pass`. + - Any `pass` → cell is `pass`. A mix of (pass, not_applicable) — + e.g. a tier where the feature isn't supported alongside tiers + where it works — surfaces as `pass` so the published cell + reflects that the feature *does* work on this provider rather + than silently demoting it to `not_applicable` and discarding + the passing tiers. + - All `not_applicable` → cell is `not_applicable` with the first + row's reason. - empty / nothing recognized → `not_tested`. `not_tested` rows are treated as absent data: they're dropped before @@ -156,16 +162,15 @@ def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: errors = [str(r.get("error", "test failed")) for r in failures] return {"status": "fail", "error": "; ".join(errors)} - for r in observed: - if r.get("status") == "not_applicable": - return { - "status": "not_applicable", - "reason": str(r.get("reason", "not applicable")), - } - - if all(r.get("status") == "pass" for r in observed): + if any(r.get("status") == "pass" for r in observed): return {"status": "pass"} + if all(r.get("status") == "not_applicable" for r in observed): + return { + "status": "not_applicable", + "reason": str(observed[0].get("reason", "not applicable")), + } + return {"status": "not_tested"}