diff --git a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py index 2644e092e48..8e1eef17811 100644 --- a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py +++ b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -189,6 +189,78 @@ def test_build_matrix_all_not_tested_stays_not_tested(): assert matrix["features"][0]["providers"]["anthropic"] == {"status": "not_tested"} +def test_build_matrix_mixed_pass_and_not_applicable_surfaces_pass(): + """A `not_applicable` row mixed with `pass` rows must surface as + `pass`, not `not_applicable`. The published cell answers "does this + feature work on this provider?"; if any tier passes, the feature + works there. Discarding passing tiers because one tier is NA would + misrepresent the cell as unsupported. + """ + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + { + "feature_id": "f", + "provider": "anthropic", + "result": { + "status": "not_applicable", + "reason": "haiku does not support extended thinking", + }, + }, + {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"} + + +def test_build_matrix_all_not_applicable_stays_not_applicable(): + """When every observed row is `not_applicable`, the cell remains + `not_applicable` and the first row's reason carries through to the + published matrix. + """ + manifest = { + "schema_version": "1", + "providers": ["anthropic"], + "features": [{"id": "f", "name": "F"}], + } + results = [ + { + "feature_id": "f", + "provider": "anthropic", + "result": { + "status": "not_applicable", + "reason": "feature unsupported on this provider", + }, + }, + { + "feature_id": "f", + "provider": "anthropic", + "result": {"status": "not_applicable", "reason": "ditto"}, + }, + ] + matrix = build_matrix( + manifest=manifest, + results=results, + litellm_version="v", + claude_code_version="c", + generated_at="t", + ) + assert matrix["features"][0]["providers"]["anthropic"] == { + "status": "not_applicable", + "reason": "feature unsupported on this provider", + } + + def test_build_matrix_fills_not_tested_for_missing_cells(): manifest = { "schema_version": "1", diff --git a/tests/claude_code/matrix_builder.py b/tests/claude_code/matrix_builder.py index 8555805dab2..5641e488da2 100644 --- a/tests/claude_code/matrix_builder.py +++ b/tests/claude_code/matrix_builder.py @@ -133,8 +133,14 @@ def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: - Any `fail` → cell is `fail` with every failing model's error joined by `"; "` so a multi-tier breakage doesn't silently hide all but the first error from the published matrix. - - `not_applicable` → cell is `not_applicable` with the reason. - - `pass` → cell is `pass`. + - Any `pass` → cell is `pass`. A mix of (pass, not_applicable) — + e.g. a tier where the feature isn't supported alongside tiers + where it works — surfaces as `pass` so the published cell + reflects that the feature *does* work on this provider rather + than silently demoting it to `not_applicable` and discarding + the passing tiers. + - All `not_applicable` → cell is `not_applicable` with the first + row's reason. - empty / nothing recognized → `not_tested`. `not_tested` rows are treated as absent data: they're dropped before @@ -156,16 +162,15 @@ def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: errors = [str(r.get("error", "test failed")) for r in failures] return {"status": "fail", "error": "; ".join(errors)} - for r in observed: - if r.get("status") == "not_applicable": - return { - "status": "not_applicable", - "reason": str(r.get("reason", "not applicable")), - } - - if all(r.get("status") == "pass" for r in observed): + if any(r.get("status") == "pass" for r in observed): return {"status": "pass"} + if all(r.get("status") == "not_applicable" for r in observed): + return { + "status": "not_applicable", + "reason": str(observed[0].get("reason", "not applicable")), + } + return {"status": "not_tested"}