test: simplify Rust Python parity harness structure

tests/rust-python-harness/README.md

@@ -9,34 +9,28 @@ The matrix always has these SDK columns:
 - `responses / aresponses`
 - `count_tokens`
 
-Each numbered section in the parity TDD owns one folder below [`strategies/`](strategies/):
+The harness has three deliberately broad test-strategy folders:
 
-| TDD section | Strategy folder |
+| Strategy | Folder |
 | --- | --- |
-| 1 | `end_to_end/` |
-| 2a | `transform_request/` |
-| 2b | `transform_response/` |
-| 2c | `transform_stream/` |
-| 3 | `cassettes/` |
-| 4 | `callbacks/` |
-| 5 | `manifest_coverage/` |
-| 6a | `dual_build_suite/` |
-| 6b | `shadow_mode/` |
+| Public SDK parity over generated and recorded inputs | [`e2e_fuzz_tests/`](e2e_fuzz_tests/) |
+| Focused tests of Rust-owned behavior | [`unit_tests_rust/`](unit_tests_rust/) |
+| Isolated transform and Python-to-Rust helper coverage | [`validate_sub_methods/`](validate_sub_methods/) |
 
 ## Run it
 
 From the repository root:
 
 ```bash
-poetry run python -m tests.rust_python_harness
+poetry run python -m tests.rust-python-harness
 ```
 
 The default runs every configured test once and updates all matching cells in real time. Narrow a run by strategy, SDK function, or both:
 
 ```bash
-poetry run python -m tests.rust_python_harness --strategy end_to_end
-poetry run python -m tests.rust_python_harness --function messages
-poetry run python -m tests.rust_python_harness --strategy end_to_end --function ocr
+poetry run python -m tests.rust-python-harness --strategy e2e_fuzz_tests
+poetry run python -m tests.rust-python-harness --function messages
+poetry run python -m tests.rust-python-harness --strategy validate_sub_methods --function ocr
 ```
 
 For a guided run, use the interactive picker. It asks which strategy rows and SDK
@@ -44,20 +38,23 @@ function columns to include, then hands the terminal to the live dashboard. It n
 captures keys while tests are running, so Ctrl-C and pytest debugging remain safe.
 
 ```bash
-poetry run python -m tests.rust_python_harness --interactive
+poetry run python -m tests.rust-python-harness --interactive
 ```
 
 Useful operator options:
 
 ```bash
 # Inspect coverage and pytest selectors without running anything.
-poetry run python -m tests.rust_python_harness --list
+poetry run python -m tests.rust-python-harness --list
 
 # Stable line-oriented output for CI logs or redirected output.
-poetry run python -m tests.rust_python_harness --plain
+poetry run python -m tests.rust-python-harness --plain
+
+# Measure Python reference lines exercised by this parity run and build an HTML heatmap.
+poetry run python -m tests.rust-python-harness --coverage
 
 # Forward pytest options. Use the equals form when the value begins with a dash.
-poetry run python -m tests.rust_python_harness --pytest-arg=-x
+poetry run python -m tests.rust-python-harness --pytest-arg=-x
 ```
 
 The process returns pytest's exit code. A configured selector that collects no test is also a failure. A planned cell has no selector yet and does not fail the run.
@@ -67,6 +64,32 @@ and prints the three slowest tests when the run ends. Each failure includes a fo
 `poetry run pytest ... -q` command. Redirected output and CI automatically use the
 line-oriented plain renderer; `--plain` lets you opt into it locally.
 
+Coverage reports are written outside the three strategy folders at
+`target/rust-python-harness/`. Open `python-html/index.html` to inspect executed and
+missing Python lines; `python.json` and `python.xml` are available for automation.
+Coverage is finalized after pytest exits, because worker processes must flush their
+data first.
+
+## Port coverage and confidence
+
+Treat these as separate signals instead of one ambiguous coverage percentage:
+
+| Signal | Tool | What it proves |
+| --- | --- | --- |
+| Python reference LOC | `coverage.py` / `pytest-cov` via `--coverage` | The mapped Python behavior ran |
+| Rust port LOC | `cargo-llvm-cov` | The mapped Rust implementation ran |
+| Parity contracts | This harness matrix | Python and Rust had the same observable behavior |
+
+`validate_sub_methods/` owns the future source-section inventory that maps a stable
+Python qualified symbol to its Rust symbol. That inventory is the denominator for
+per-function rollups; raw coverage for the entire LiteLLM repository would obscure
+the port's real gaps. `unit_tests_rust/` owns direct `cargo-llvm-cov` runs, while
+`e2e_fuzz_tests/` owns behavioral parity and fuzz-case counts. Keep Python, Rust, and
+parity percentages visible side by side and label section confidence High only when
+the mapped implementation exists, every required strategy passes, and both sides meet
+their LOC thresholds. Generated Rust LCOV/HTML and the combined index also belong in
+`target/rust-python-harness/`, not in a fourth strategy folder.
+
 ## Read the matrix
 
 | Mark | Meaning |
@@ -84,13 +107,13 @@ The initial end-to-end entries deliberately show `◐`: the repository has Rust
 
 ## Attach parity tests
 
-Every strategy folder contains a `strategy.json`. Add a pytest file or node ID to the appropriate SDK function's `selectors` list:
+Each of the three folders contains a concise `README.md` and a `strategy.json`. Add a pytest file or node ID to the appropriate SDK function's `selectors` list:
 
 ```json
 {
   "coverage": "complete",
   "selectors": [
-    "tests/rust_python_harness/strategies/transform_request/test_messages.py"
+    "tests/rust-python-harness/validate_sub_methods/test_messages.py"
   ]
 }
 ```

tests/rust-python-harness/catalog.py

@@ -6,7 +6,7 @@ from typing import Any
 
 from .models import Coverage, HarnessCase, SDK_FUNCTIONS, Strategy
 
-STRATEGIES_ROOT = Path(__file__).parent / "strategies"
+STRATEGIES_ROOT = Path(__file__).parent
 
 
 def _require_string(value: Any, field: str, source: Path) -> str:

tests/rust-python-harness/cli.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import importlib.util
 from collections.abc import Sequence
 from pathlib import Path
 
@@ -10,6 +11,7 @@ from .runner import run_pytest
 from .ui import make_dashboard
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
+COVERAGE_ROOT = REPO_ROOT / "target" / "rust-python-harness"
 
 
 def _parser() -> argparse.ArgumentParser:
@@ -46,6 +48,11 @@ def _parser() -> argparse.ArgumentParser:
         action="store_true",
         help="disable the interactive terminal dashboard",
     )
+    parser.add_argument(
+        "--coverage",
+        action="store_true",
+        help="write Python reference LOC reports (HTML, JSON, and XML)",
+    )
     parser.add_argument(
         "--pytest-arg",
         action="append",
@@ -56,6 +63,17 @@ def _parser() -> argparse.ArgumentParser:
     return parser
 
 
+def _coverage_pytest_args(output_root: Path = COVERAGE_ROOT) -> tuple[str, ...]:
+    output_root.mkdir(parents=True, exist_ok=True)
+    return (
+        "--cov=litellm",
+        "--cov-context=test",
+        f"--cov-report=json:{output_root / 'python.json'}",
+        f"--cov-report=xml:{output_root / 'python.xml'}",
+        f"--cov-report=html:{output_root / 'python-html'}",
+    )
+
+
 def _pick_values(
     title: str, options: Sequence[tuple[str, str]], input_fn=input
 ) -> set[str]:
@@ -115,6 +133,11 @@ def _print_catalog(strategies: Sequence[Strategy]) -> None:
 
 def main(argv: Sequence[str] | None = None) -> int:
     args = _parser().parse_args(argv)
+    if args.coverage and importlib.util.find_spec("pytest_cov") is None:
+        _parser().error(
+            "--coverage requires the project's pytest-cov dependency; run with "
+            "`poetry run python -m tests.rust-python-harness --coverage`"
+        )
     strategies = load_catalog()
     if args.list:
         _print_catalog(strategies)
@@ -136,12 +159,18 @@ def main(argv: Sequence[str] | None = None) -> int:
         strategy for strategy in strategies if strategy.id in selected_strategy_ids
     )
     dashboard = make_dashboard(visible_strategies, plain=args.plain)
+    pytest_args = [*args.pytest_arg]
+    if args.coverage:
+        pytest_args.extend(_coverage_pytest_args())
     with dashboard:
         exit_code, run = run_pytest(
             cases=cases,
             repo_root=REPO_ROOT,
             on_update=dashboard.update,
-            pytest_args=args.pytest_arg,
+            pytest_args=pytest_args,
         )
         dashboard.finish(run, exit_code)
+    if args.coverage and (COVERAGE_ROOT / "python.json").exists():
+        print(f"Python LOC heatmap: {COVERAGE_ROOT / 'python-html' / 'index.html'}")
+        print(f"Machine-readable coverage: {COVERAGE_ROOT / 'python.json'}")
     return exit_code

tests/rust-python-harness/e2e_fuzz_tests/README.md

@@ -0,0 +1,3 @@
+# End-to-end fuzz tests
+
+Runs the same SDK call through the Python and Rust paths using generated inputs and recorded provider responses. It compares public results, streams, callbacks, and exceptions to catch behavior differences a unit test can miss.

tests/rust-python-harness/e2e_fuzz_tests/strategy.json

@@ -1,12 +1,12 @@
 {
   "order": 10,
-  "id": "end_to_end",
-  "label": "1 · End-to-end public call",
-  "description": "Compare the observable SDK result or exception from the Python and Rust paths.",
+  "id": "e2e_fuzz_tests",
+  "label": "End-to-end fuzz tests",
+  "description": "Compare observable Python and Rust SDK behavior over generated and recorded inputs.",
   "functions": {
-    "ocr": {"coverage": "partial", "selectors": ["tests/test_litellm/ocr/test_rust_bridge.py"], "note": "Bridge coverage exists; frozen-oracle parity cases are still needed."},
-    "messages": {"coverage": "partial", "selectors": ["tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py"], "note": "Bridge coverage exists; frozen-oracle parity cases are still needed."},
-    "responses": {"coverage": "partial", "selectors": ["tests/test_litellm/responses/test_rust_bridge_websocket.py"], "note": "Covers the websocket bridge, not full responses/aresponses parity."},
+    "ocr": {"coverage": "partial", "selectors": ["tests/test_litellm/ocr/test_rust_bridge.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."},
+    "messages": {"coverage": "partial", "selectors": ["tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."},
+    "responses": {"coverage": "partial", "selectors": ["tests/test_litellm/responses/test_rust_bridge_websocket.py"], "note": "Covers the websocket bridge; full responses parity is still being added."},
     "count_tokens": {"coverage": "planned", "selectors": [], "note": "No Rust count_tokens parity test is present yet."}
   }
 }

tests/rust-python-harness/unit_tests_rust/README.md

@@ -0,0 +1,3 @@
+# Rust unit tests
+
+Holds focused Cargo tests for Rust-owned parsing, transforms, errors, and streaming behavior. These tests make failures fast to diagnose before the Python bridge or full SDK path is involved.

tests/rust-python-harness/unit_tests_rust/strategy.json

@@ -1,8 +1,8 @@
 {
   "order": 20,
-  "id": "transform_request",
-  "label": "2a · transform_request fuzz diff",
-  "description": "Compare normalized method, URL, headers, and body across generated request inputs.",
+  "id": "unit_tests_rust",
+  "label": "Rust unit tests",
+  "description": "Exercise Rust-owned behavior directly with focused unit tests.",
   "functions": {
     "ocr": {"coverage": "planned", "selectors": []},
     "messages": {"coverage": "planned", "selectors": []},

tests/rust-python-harness/validate_sub_methods/README.md

@@ -0,0 +1,3 @@
+# Validate sub-methods
+
+Checks each request, response, stream, and error-mapping sub-method independently across Python and Rust. It also validates that traced Python helpers have an explicit Rust implementation and parity test.

tests/rust-python-harness/validate_sub_methods/strategy.json

@@ -1,8 +1,8 @@
 {
   "order": 30,
-  "id": "transform_response",
-  "label": "2b · transform_response cassette diff",
-  "description": "Feed identical recorded status, headers, and body bytes into both transforms.",
+  "id": "validate_sub_methods",
+  "label": "Validate sub-methods",
+  "description": "Compare isolated transforms and verify Python-to-Rust helper coverage.",
   "functions": {
     "ocr": {"coverage": "planned", "selectors": []},
     "messages": {"coverage": "planned", "selectors": []},

tests/rust_python_harness/strategies/callbacks/strategy.json

@@ -1,12 +0,0 @@
-{
-  "order": 60,
-  "id": "callbacks",
-  "label": "4 · Callback and logging payload diff",
-  "description": "Compare callback arguments and StandardLoggingPayload with unstable fields scrubbed.",
-  "functions": {
-    "ocr": {"coverage": "planned", "selectors": []},
-    "messages": {"coverage": "planned", "selectors": []},
-    "responses": {"coverage": "planned", "selectors": []},
-    "count_tokens": {"coverage": "planned", "selectors": []}
-  }
-}

tests/rust_python_harness/strategies/cassettes/strategy.json

@@ -1,12 +0,0 @@
-{
-  "order": 50,
-  "id": "cassettes",
-  "label": "3 · Recorded provider cassettes",
-  "description": "Validate checked-in provider-byte recordings used by both implementations.",
-  "functions": {
-    "ocr": {"coverage": "planned", "selectors": []},
-    "messages": {"coverage": "planned", "selectors": []},
-    "responses": {"coverage": "planned", "selectors": []},
-    "count_tokens": {"coverage": "planned", "selectors": []}
-  }
-}

tests/rust_python_harness/strategies/dual_build_suite/strategy.json

@@ -1,12 +0,0 @@
-{
-  "order": 80,
-  "id": "dual_build_suite",
-  "label": "6a · Existing suite under both builds",
-  "description": "Compare pass, fail, and skip outcomes from the established suite under both flags.",
-  "functions": {
-    "ocr": {"coverage": "planned", "selectors": []},
-    "messages": {"coverage": "planned", "selectors": []},
-    "responses": {"coverage": "planned", "selectors": []},
-    "count_tokens": {"coverage": "planned", "selectors": []}
-  }
-}

tests/rust_python_harness/strategies/manifest_coverage/strategy.json

@@ -1,12 +0,0 @@
-{
-  "order": 70,
-  "id": "manifest_coverage",
-  "label": "5 · Python trace → Rust symbol manifest",
-  "description": "Require every traced Python call to map to a compiled Rust symbol and a parity test.",
-  "functions": {
-    "ocr": {"coverage": "planned", "selectors": []},
-    "messages": {"coverage": "planned", "selectors": []},
-    "responses": {"coverage": "planned", "selectors": []},
-    "count_tokens": {"coverage": "planned", "selectors": []}
-  }
-}

tests/rust_python_harness/strategies/shadow_mode/strategy.json

@@ -1,12 +0,0 @@
-{
-  "order": 90,
-  "id": "shadow_mode",
-  "label": "6b · Production shadow comparison",
-  "description": "Exercise shadow-mode mismatch metrics and redacted parity artifacts.",
-  "functions": {
-    "ocr": {"coverage": "planned", "selectors": []},
-    "messages": {"coverage": "planned", "selectors": []},
-    "responses": {"coverage": "planned", "selectors": []},
-    "count_tokens": {"coverage": "planned", "selectors": []}
-  }
-}

tests/rust_python_harness/strategies/transform_stream/strategy.json

@@ -1,12 +0,0 @@
-{
-  "order": 40,
-  "id": "transform_stream",
-  "label": "2c · transform_stream chaos split",
-  "description": "Re-cut recorded SSE bytes and compare emitted chunks and final assembly.",
-  "functions": {
-    "ocr": {"coverage": "not_applicable", "selectors": [], "note": "OCR is not streaming."},
-    "messages": {"coverage": "planned", "selectors": []},
-    "responses": {"coverage": "planned", "selectors": []},
-    "count_tokens": {"coverage": "not_applicable", "selectors": [], "note": "count_tokens is not streaming."}
-  }
-}

tests/test_rust_python_harness.py

@@ -1,26 +1,33 @@
 from __future__ import annotations
 
+import importlib
 import json
 from pathlib import Path
 
 import pytest
 
-from tests.rust_python_harness.catalog import load_catalog
-from tests.rust_python_harness.cli import _pick_values, _select
-from tests.rust_python_harness.models import (
-    CaseResult,
-    Coverage,
-    HarnessCase,
-    HarnessRun,
-    RunStatus,
-    SDK_FUNCTIONS,
-)
-from tests.rust_python_harness.runner import (
-    run_pytest,
-    runnable_selectors,
-    selector_matches_node,
-)
-from tests.rust_python_harness.ui import _format_duration, _rerun_command, _summary
+catalog = importlib.import_module("tests.rust-python-harness.catalog")
+cli = importlib.import_module("tests.rust-python-harness.cli")
+models = importlib.import_module("tests.rust-python-harness.models")
+runner = importlib.import_module("tests.rust-python-harness.runner")
+ui = importlib.import_module("tests.rust-python-harness.ui")
+
+load_catalog = catalog.load_catalog
+_pick_values = cli._pick_values
+_coverage_pytest_args = cli._coverage_pytest_args
+_select = cli._select
+CaseResult = models.CaseResult
+Coverage = models.Coverage
+HarnessCase = models.HarnessCase
+HarnessRun = models.HarnessRun
+RunStatus = models.RunStatus
+SDK_FUNCTIONS = models.SDK_FUNCTIONS
+run_pytest = runner.run_pytest
+runnable_selectors = runner.runnable_selectors
+selector_matches_node = runner.selector_matches_node
+_format_duration = ui._format_duration
+_rerun_command = ui._rerun_command
+_summary = ui._summary
 
 
 def _case(
@@ -48,19 +55,13 @@ def _manifest() -> dict[str, object]:
     }
 
 
-def test_should_load_every_tdd_strategy_in_order() -> None:
+def test_should_load_the_three_harness_strategies_in_order() -> None:
     strategies = load_catalog()
 
     assert [strategy.id for strategy in strategies] == [
-        "end_to_end",
-        "transform_request",
-        "transform_response",
-        "transform_stream",
-        "cassettes",
-        "callbacks",
-        "manifest_coverage",
-        "dual_build_suite",
-        "shadow_mode",
+        "e2e_fuzz_tests",
+        "unit_tests_rust",
+        "validate_sub_methods",
     ]
     assert all(
         tuple(case.sdk_function for case in strategy.cases) == SDK_FUNCTIONS
@@ -162,10 +163,10 @@ def test_should_replace_a_pass_with_a_teardown_error() -> None:
 def test_should_filter_the_catalog_by_strategy_and_sdk_function() -> None:
     strategies = load_catalog()
 
-    cases = _select(strategies, {"end_to_end"}, {"messages"})
+    cases = _select(strategies, {"e2e_fuzz_tests"}, {"messages"})
 
     assert len(cases) == 1
-    assert cases[0].key == "end_to_end:messages"
+    assert cases[0].key == "e2e_fuzz_tests:messages"
 
 
 def test_should_reject_an_unknown_strategy() -> None:
@@ -199,3 +200,16 @@ def test_should_format_developer_facing_run_context() -> None:
     assert _rerun_command("tests/test_parity.py::test_one[value with spaces]") == (
         "poetry run pytest 'tests/test_parity.py::test_one[value with spaces]' -q"
     )
+
+
+def test_should_build_python_coverage_reports_below_the_target_directory(
+    tmp_path: Path,
+) -> None:
+    args = _coverage_pytest_args(tmp_path)
+
+    assert tmp_path.is_dir()
+    assert "--cov=litellm" in args
+    assert "--cov-context=test" in args
+    assert f"--cov-report=json:{tmp_path / 'python.json'}" in args
+    assert f"--cov-report=xml:{tmp_path / 'python.xml'}" in args
+    assert f"--cov-report=html:{tmp_path / 'python-html'}" in args