test: show parity confidence by SDK section

tests/rust-python-harness/README.md

@@ -64,6 +64,11 @@ and prints the three slowest tests when the run ends. Each failure includes a fo
 `poetry run pytest ... -q` command. Redirected output and CI automatically use the
 line-oriented plain renderer; `--plain` lets you opt into it locally.
 
+The final screen includes a confidence score for every SDK section. It is the direct
+ratio of required strategy rows with passing evidence, such as `1/3 = 33%`; High means
+all required strategies passed, Medium means some passed, and Low means none passed.
+This behavioral score is intentionally shown separately from Python and Rust LOC.
+
 Coverage reports are written outside the three strategy folders at
 `target/rust-python-harness/`. Open `python-html/index.html` to inspect executed and
 missing Python lines; `python.json` and `python.xml` are available for automation.

tests/rust-python-harness/cli.py

@@ -158,7 +158,11 @@ def main(argv: Sequence[str] | None = None) -> int:
     visible_strategies = tuple(
         strategy for strategy in strategies if strategy.id in selected_strategy_ids
     )
-    dashboard = make_dashboard(visible_strategies, plain=args.plain)
+    dashboard = make_dashboard(
+        visible_strategies,
+        plain=args.plain,
+        confidence_strategies=strategies,
+    )
     pytest_args = [*args.pytest_arg]
     if args.coverage:
         pytest_args.extend(_coverage_pytest_args())

tests/rust-python-harness/models.py

@@ -27,6 +27,12 @@ class RunStatus(str, Enum):
     NOT_APPLICABLE = "not_applicable"
 
 
+class ConfidenceLevel(str, Enum):
+    HIGH = "HIGH"
+    MEDIUM = "MEDIUM"
+    LOW = "LOW"
+
+
 SDK_FUNCTIONS = ("ocr", "messages", "responses", "count_tokens")
 
 
@@ -140,3 +146,74 @@ class HarnessRun:
         for result in results.values():
             result.set_initial_status()
         return cls(results=results)
+
+
+@dataclass(frozen=True)
+class SectionConfidence:
+    sdk_function: str
+    verified_strategies: int
+    required_strategies: int
+    level: ConfidenceLevel
+    details: tuple[str, ...]
+
+    @property
+    def percentage(self) -> int:
+        if not self.required_strategies:
+            return 0
+        return round(100 * self.verified_strategies / self.required_strategies)
+
+
+def section_confidence(
+    run: HarnessRun, strategies: Iterable[Strategy]
+) -> tuple[SectionConfidence, ...]:
+    strategy_list = tuple(strategies)
+    scores: list[SectionConfidence] = []
+    for sdk_function in SDK_FUNCTIONS:
+        cases = tuple(
+            case
+            for strategy in strategy_list
+            for case in strategy.cases
+            if case.sdk_function == sdk_function
+            and case.coverage is not Coverage.NOT_APPLICABLE
+        )
+        verified = 0
+        details: list[str] = []
+        for case in cases:
+            result = run.results.get(case.key)
+            status = result.status if result is not None else RunStatus.NOT_RUN
+            if status is RunStatus.PASSED:
+                verified += 1
+            details.append(
+                f"{STATUS_LABELS[status]} {case.strategy_id} ({case.coverage.value})"
+            )
+        required = len(cases)
+        if required and verified == required:
+            level = ConfidenceLevel.HIGH
+        elif verified:
+            level = ConfidenceLevel.MEDIUM
+        else:
+            level = ConfidenceLevel.LOW
+        scores.append(
+            SectionConfidence(
+                sdk_function=sdk_function,
+                verified_strategies=verified,
+                required_strategies=required,
+                level=level,
+                details=tuple(details),
+            )
+        )
+    return tuple(scores)
+
+
+STATUS_LABELS = {
+    RunStatus.NOT_RUN: "·",
+    RunStatus.QUEUED: "○",
+    RunStatus.RUNNING: "◉",
+    RunStatus.PASSED: "✓",
+    RunStatus.FAILED: "✗",
+    RunStatus.SKIPPED: "↷",
+    RunStatus.ERROR: "!",
+    RunStatus.MISSING: "?",
+    RunStatus.PLANNED: "—",
+    RunStatus.NOT_APPLICABLE: "n/a",
+}

tests/rust-python-harness/ui.py

@@ -8,7 +8,14 @@ from contextlib import AbstractContextManager
 from pathlib import Path
 from typing import Any
 
-from .models import Coverage, HarnessRun, RunStatus, SDK_FUNCTIONS, Strategy
+from .models import (
+    Coverage,
+    HarnessRun,
+    RunStatus,
+    SDK_FUNCTIONS,
+    Strategy,
+    section_confidence,
+)
 
 STATUS_GLYPHS = {
     RunStatus.NOT_RUN: "·",
@@ -74,11 +81,16 @@ def _cell_text(run: HarnessRun, strategy_id: str, sdk_function: str) -> tuple[st
 
 
 class RichDashboard(AbstractContextManager["RichDashboard"]):
-    def __init__(self, strategies: Sequence[Strategy]) -> None:
+    def __init__(
+        self,
+        strategies: Sequence[Strategy],
+        confidence_strategies: Sequence[Strategy],
+    ) -> None:
         from rich.console import Console
         from rich.live import Live
 
         self.strategies = strategies
+        self.confidence_strategies = confidence_strategies
         self.console = Console()
         self.live: Any = Live(
             console=self.console, refresh_per_second=12, transient=False
@@ -176,6 +188,29 @@ class RichDashboard(AbstractContextManager["RichDashboard"]):
                     for nodeid, duration in slow
                 )
             )
+        from rich import box
+        from rich.table import Table
+
+        confidence_table = Table(
+            title="Port confidence by SDK section", box=box.ROUNDED, expand=True
+        )
+        confidence_table.add_column("SDK section")
+        confidence_table.add_column("Score", justify="right")
+        confidence_table.add_column("Confidence")
+        confidence_table.add_column("Strategy evidence", ratio=4)
+        confidence_styles = {"HIGH": "green", "MEDIUM": "yellow", "LOW": "red"}
+        for score in section_confidence(run, self.confidence_strategies):
+            confidence_table.add_row(
+                score.sdk_function,
+                f"{score.verified_strategies}/{score.required_strategies}  {score.percentage}%",
+                f"[{confidence_styles[score.level.value]}]{score.level.value}[/]",
+                "  ".join(score.details),
+            )
+        self.console.print(confidence_table)
+        self.console.print(
+            "[dim]Score = required strategies with passing evidence. "
+            "LOC coverage remains a separate report.[/dim]"
+        )
         style = "green" if exit_code == 0 else "red"
         self.console.print(
             f"[{style}]Harness finished in {_format_duration(run.duration)} "
@@ -184,8 +219,13 @@ class RichDashboard(AbstractContextManager["RichDashboard"]):
 
 
 class PlainDashboard(AbstractContextManager["PlainDashboard"]):
-    def __init__(self, strategies: Sequence[Strategy]) -> None:
+    def __init__(
+        self,
+        strategies: Sequence[Strategy],
+        confidence_strategies: Sequence[Strategy],
+    ) -> None:
         self.strategies = strategies
+        self.confidence_strategies = confidence_strategies
         self._seen: dict[str, tuple[RunStatus, int]] = {}
 
     def __enter__(self) -> "PlainDashboard":
@@ -218,12 +258,28 @@ class PlainDashboard(AbstractContextManager["PlainDashboard"]):
         )
         for nodeid, _ in run.failures[:5]:
             print(f"Rerun: {_rerun_command(nodeid)}", flush=True)
+        print("Port confidence by SDK section", flush=True)
+        for score in section_confidence(run, self.confidence_strategies):
+            print(
+                f"  {score.sdk_function:12} "
+                f"{score.verified_strategies}/{score.required_strategies} "
+                f"{score.percentage:3}% {score.level.value:6}  "
+                f"{' | '.join(score.details)}",
+                flush=True,
+            )
+        print(
+            "  Score = required strategies with passing evidence; LOC is reported separately.",
+            flush=True,
+        )
         print(f"Harness finished with exit code {exit_code}", flush=True)
 
 
 def make_dashboard(
-    strategies: Sequence[Strategy], plain: bool = False
+    strategies: Sequence[Strategy],
+    plain: bool = False,
+    confidence_strategies: Sequence[Strategy] | None = None,
 ) -> RichDashboard | PlainDashboard:
+    confidence_strategies = confidence_strategies or strategies
     interactive_terminal = (
         sys.stdout.isatty()
         and not os.environ.get("CI")
@@ -233,7 +289,7 @@ def make_dashboard(
         try:
             import rich  # noqa: F401
 
-            return RichDashboard(strategies)
+            return RichDashboard(strategies, confidence_strategies)
         except ImportError:
             pass
-    return PlainDashboard(strategies)
+    return PlainDashboard(strategies, confidence_strategies)

tests/test_rust_python_harness.py

@@ -22,6 +22,7 @@ HarnessCase = models.HarnessCase
 HarnessRun = models.HarnessRun
 RunStatus = models.RunStatus
 SDK_FUNCTIONS = models.SDK_FUNCTIONS
+section_confidence = models.section_confidence
 run_pytest = runner.run_pytest
 runnable_selectors = runner.runnable_selectors
 selector_matches_node = runner.selector_matches_node
@@ -213,3 +214,23 @@ def test_should_build_python_coverage_reports_below_the_target_directory(
     assert f"--cov-report=json:{tmp_path / 'python.json'}" in args
     assert f"--cov-report=xml:{tmp_path / 'python.xml'}" in args
     assert f"--cov-report=html:{tmp_path / 'python-html'}" in args
+
+
+def test_should_report_confidence_for_each_sdk_section() -> None:
+    strategies = load_catalog()
+    cases = tuple(case for strategy in strategies for case in strategy.cases)
+    run = HarnessRun.from_cases(cases)
+    passing = run.results["e2e_fuzz_tests:responses"]
+    passing.collected.add("tests/test_parity.py::test_one")
+    passing.record("tests/test_parity.py::test_one", RunStatus.PASSED)
+
+    scores = {
+        score.sdk_function: score for score in section_confidence(run, strategies)
+    }
+
+    assert scores["responses"].verified_strategies == 1
+    assert scores["responses"].required_strategies == 3
+    assert scores["responses"].percentage == 33
+    assert scores["responses"].level.value == "MEDIUM"
+    assert scores["count_tokens"].percentage == 0
+    assert scores["count_tokens"].level.value == "LOW"