feat(rust-python-harness): wire existing e2e SDK tests into the matrix (#39463)

Adds chat_completions and transcription as SDK function columns, backed
by the existing rust_bridge test files. Adds a fourth strategy folder,
existing_e2e_test_sdk, that points at already-existing live-API SDK
tests (tests/ocr_tests/ as a whole folder, plus chat completion and
Whisper transcription tests) instead of writing new parity tests.
Extends selector_matches_node with trailing-slash folder selectors so
a whole test folder can back one matrix cell.

tests/rust-python-harness/README.md

@@ -8,14 +8,17 @@ The matrix always has these SDK columns:
 - `messages / amessages`
 - `responses / aresponses`
 - `count_tokens`
+- `chat_completions / acompletion`
+- `transcription / atranscription`
 
-The harness has three deliberately broad test-strategy folders:
+The harness has four deliberately broad test-strategy folders:
 
 | Strategy | Folder |
 | --- | --- |
 | Public SDK parity over generated and recorded inputs | [`e2e_fuzz_tests/`](e2e_fuzz_tests/) |
 | Focused tests of Rust-owned behavior | [`unit_tests_rust/`](unit_tests_rust/) |
 | Isolated transform and Python-to-Rust helper coverage | [`validate_sub_methods/`](validate_sub_methods/) |
+| Already-existing live-API SDK tests | [`existing_e2e_test_sdk/`](existing_e2e_test_sdk/) |
 
 ## Run it
 
@@ -112,7 +115,7 @@ The initial end-to-end entries deliberately show `◐`: the repository has Rust
 
 ## Attach parity tests
 
-Each of the three folders contains a concise `README.md` and a `strategy.json`. Add a pytest file or node ID to the appropriate SDK function's `selectors` list:
+Each of the four folders contains a concise `README.md` and a `strategy.json`. Add a pytest file or node ID to the appropriate SDK function's `selectors` list:
 
 ```json
 {
@@ -123,7 +126,7 @@ Each of the three folders contains a concise `README.md` and a `strategy.json`.
 }
 ```
 
-Selectors use the same syntax as pytest. A file selector aggregates every test in the file; a node selector can target one test or parametrized family. The runner deduplicates selectors, so one test may intentionally prove more than one cell without executing twice.
+Selectors use the same syntax as pytest. A file selector aggregates every test in the file; a node selector can target one test or parametrized family; a selector ending in `/` aggregates every test in that folder, recursively. The runner deduplicates selectors, so one test may intentionally prove more than one cell without executing twice.
 
 Use these coverage values:
 

tests/rust-python-harness/cli.py

@@ -6,14 +6,13 @@ from collections.abc import Sequence
 from pathlib import Path
 
 from .catalog import load_catalog
-from .models import HarnessCase, Strategy
+from .models import SDK_FUNCTIONS, HarnessCase, Strategy
 from .runner import run_pytest
 from .ui import make_dashboard
 from .strategies.unit_tests.mapping_validator import FunctionReport, build_function_report
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
 COVERAGE_ROOT = REPO_ROOT / "target" / "rust-python-harness"
-SDK_FUNCTION_CHOICES = ("ocr", "messages", "responses", "count_tokens")
 
 
 def _parser() -> argparse.ArgumentParser:
@@ -42,7 +41,7 @@ def _parser() -> argparse.ArgumentParser:
         action="append",
         default=[],
         dest="sdk_functions",
-        choices=SDK_FUNCTION_CHOICES,
+        choices=SDK_FUNCTIONS,
         help="run only this SDK function",
     )
     parser.add_argument(
@@ -110,7 +109,7 @@ def _interactive_filters(strategies: Sequence[Strategy]) -> tuple[set[str], set[
     )
     sdk_functions = _pick_values(
         "SDK functions",
-        [(name, name) for name in SDK_FUNCTION_CHOICES],
+        [(name, name) for name in SDK_FUNCTIONS],
     )
     return strategy_ids, sdk_functions
 
@@ -166,7 +165,7 @@ def _print_function_report(report: FunctionReport) -> None:
 
 
 def _validate_ledger(sdk_functions: set[str]) -> int:
-    functions = sdk_functions or set(SDK_FUNCTION_CHOICES)
+    functions = sdk_functions or set(SDK_FUNCTIONS)
     reports = tuple(build_function_report(function) for function in sorted(functions))
     for report in reports:
         _print_function_report(report)

tests/rust-python-harness/e2e_fuzz_tests/strategy.json

@@ -7,6 +7,8 @@
     "ocr": {"coverage": "partial", "selectors": ["tests/test_litellm/ocr/test_rust_bridge.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."},
     "messages": {"coverage": "partial", "selectors": ["tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."},
     "responses": {"coverage": "partial", "selectors": ["tests/test_litellm/responses/test_rust_bridge_websocket.py"], "note": "Covers the websocket bridge; full responses parity is still being added."},
-    "count_tokens": {"coverage": "planned", "selectors": [], "note": "No Rust count_tokens parity test is present yet."}
+    "count_tokens": {"coverage": "planned", "selectors": [], "note": "No Rust count_tokens parity test is present yet."},
+    "chat_completions": {"coverage": "partial", "selectors": ["tests/test_litellm/rust_bridge/test_chat_completions.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."},
+    "transcription": {"coverage": "partial", "selectors": ["tests/test_litellm/test_audio_transcription_rust_bridge.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."}
   }
 }

tests/rust-python-harness/existing_e2e_test_sdk/README.md

@@ -0,0 +1,3 @@
+# Existing e2e SDK tests
+
+Wires already-existing live-API SDK tests into the matrix instead of writing new parity tests. Selectors point at real test files and folders, such as `tests/ocr_tests/`, rather than individual node IDs, so future tests added to those folders are picked up automatically.

tests/rust-python-harness/existing_e2e_test_sdk/strategy.json

@@ -0,0 +1,14 @@
+{
+  "order": 40,
+  "id": "existing_e2e_test_sdk",
+  "label": "Existing e2e SDK tests",
+  "description": "Wire already-existing live-API SDK tests into the matrix instead of writing new parity tests.",
+  "functions": {
+    "ocr": {"coverage": "partial", "selectors": ["tests/ocr_tests/"], "note": "Existing live OCR provider tests; not yet a frozen Rust/Python oracle comparison."},
+    "messages": {"coverage": "planned", "selectors": []},
+    "responses": {"coverage": "planned", "selectors": []},
+    "count_tokens": {"coverage": "planned", "selectors": []},
+    "chat_completions": {"coverage": "partial", "selectors": ["tests/llm_translation/test_anthropic_completion.py", "tests/llm_translation/test_bedrock_completion.py"], "note": "Existing live chat completion tests for providers with confirmed Rust bridge regressions."},
+    "transcription": {"coverage": "partial", "selectors": ["tests/audio_tests/test_whisper.py"], "note": "Existing live Whisper transcription test."}
+  }
+}

tests/rust-python-harness/models.py

@@ -33,7 +33,7 @@ class ConfidenceLevel(str, Enum):
     LOW = "LOW"
 
 
-SDK_FUNCTIONS = ("ocr", "messages", "responses", "count_tokens")
+SDK_FUNCTIONS = ("ocr", "messages", "responses", "count_tokens", "chat_completions", "transcription")
 
 
 @dataclass(frozen=True)

tests/rust-python-harness/runner.py

@@ -15,6 +15,8 @@ UpdateCallback = Callable[[HarnessRun], None]
 def selector_matches_node(selector: str, nodeid: str) -> bool:
     normalized_selector = selector.replace("\\", "/")
     normalized_nodeid = nodeid.replace("\\", "/")
+    if normalized_selector.endswith("/"):
+        return normalized_nodeid.startswith(normalized_selector)
     if "::" in normalized_selector:
         return normalized_nodeid == normalized_selector or normalized_nodeid.startswith(
             f"{normalized_selector}["

tests/rust-python-harness/ui.py

@@ -119,7 +119,7 @@ class RichDashboard(AbstractContextManager["RichDashboard"]):
 
         table = Table(box=box.ROUNDED, expand=True, title="Strategy × SDK function")
         table.add_column("Strategy", ratio=3)
-        for label in ("ocr/aocr", "messages", "responses", "count_tokens"):
+        for label in SDK_FUNCTIONS:
             table.add_column(label, justify="center", ratio=1)
         for strategy in self.strategies:
             cells = []

tests/rust-python-harness/unit_tests_rust/strategy.json

@@ -7,6 +7,8 @@
     "ocr": {"coverage": "planned", "selectors": []},
     "messages": {"coverage": "planned", "selectors": []},
     "responses": {"coverage": "planned", "selectors": []},
-    "count_tokens": {"coverage": "planned", "selectors": []}
+    "count_tokens": {"coverage": "planned", "selectors": []},
+    "chat_completions": {"coverage": "planned", "selectors": []},
+    "transcription": {"coverage": "planned", "selectors": []}
   }
 }

tests/rust-python-harness/validate_sub_methods/strategy.json

@@ -7,6 +7,8 @@
     "ocr": {"coverage": "planned", "selectors": []},
     "messages": {"coverage": "planned", "selectors": []},
     "responses": {"coverage": "planned", "selectors": []},
-    "count_tokens": {"coverage": "planned", "selectors": []}
+    "count_tokens": {"coverage": "planned", "selectors": []},
+    "chat_completions": {"coverage": "planned", "selectors": []},
+    "transcription": {"coverage": "planned", "selectors": []}
   }
 }

tests/test_rust_python_harness.py

@@ -66,13 +66,14 @@ def _manifest() -> dict[str, object]:
     }
 
 
-def test_should_load_the_three_harness_strategies_in_order() -> None:
+def test_should_load_the_four_harness_strategies_in_order() -> None:
     strategies = load_catalog()
 
     assert [strategy.id for strategy in strategies] == [
         "e2e_fuzz_tests",
         "unit_tests_rust",
         "validate_sub_methods",
+        "existing_e2e_test_sdk",
     ]
     assert all(
         tuple(case.sdk_function for case in strategy.cases) == SDK_FUNCTIONS
@@ -104,6 +105,8 @@ def test_should_reject_a_manifest_missing_an_sdk_function(tmp_path: Path) -> Non
             True,
         ),
         ("tests/test_parity.py::test_one", "tests/test_parity.py::test_two", False),
+        ("tests/ocr_tests/", "tests/ocr_tests/test_ocr_mistral.py::test_one", True),
+        ("tests/ocr_tests/", "tests/other_tests/test_ocr_mistral.py::test_one", False),
     ],
 )
 def test_should_match_pytest_file_and_node_selectors(
@@ -123,6 +126,13 @@ def test_should_only_return_selectors_whose_files_exist(tmp_path: Path) -> None:
     assert runnable_selectors((case,), tmp_path) == ("tests/test_parity.py",)
 
 
+def test_should_treat_an_existing_folder_selector_as_runnable(tmp_path: Path) -> None:
+    (tmp_path / "tests" / "ocr_tests").mkdir(parents=True)
+    case = _case(selectors=("tests/ocr_tests/",))
+
+    assert runnable_selectors((case,), tmp_path) == ("tests/ocr_tests/",)
+
+
 def test_should_mark_planned_and_not_applicable_cases_without_running() -> None:
     planned = CaseResult(case=_case(coverage=Coverage.PLANNED))
     not_applicable = CaseResult(case=_case(coverage=Coverage.NOT_APPLICABLE))
@@ -239,8 +249,8 @@ def test_should_report_confidence_for_each_sdk_section() -> None:
     }
 
     assert scores["responses"].verified_strategies == 1
-    assert scores["responses"].required_strategies == 3
-    assert scores["responses"].percentage == 33
+    assert scores["responses"].required_strategies == 4
+    assert scores["responses"].percentage == 25
     assert scores["responses"].level.value == "MEDIUM"
     assert scores["count_tokens"].percentage == 0
     assert scores["count_tokens"].level.value == "LOW"