From cc43b85b5a3a23dc0890c211420a740a6eada680 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 2 Sep 2026 15:35:59 -0700 Subject: [PATCH] test: show parity confidence by SDK section --- tests/rust-python-harness/README.md | 5 ++ tests/rust-python-harness/cli.py | 6 ++- tests/rust-python-harness/models.py | 77 +++++++++++++++++++++++++++++ tests/rust-python-harness/ui.py | 68 ++++++++++++++++++++++--- tests/test_rust_python_harness.py | 21 ++++++++ 5 files changed, 170 insertions(+), 7 deletions(-) diff --git a/tests/rust-python-harness/README.md b/tests/rust-python-harness/README.md index 310b916590b..e94ac87c3b3 100644 --- a/tests/rust-python-harness/README.md +++ b/tests/rust-python-harness/README.md @@ -64,6 +64,11 @@ and prints the three slowest tests when the run ends. Each failure includes a fo `poetry run pytest ... -q` command. Redirected output and CI automatically use the line-oriented plain renderer; `--plain` lets you opt into it locally. +The final screen includes a confidence score for every SDK section. It is the direct +ratio of required strategy rows with passing evidence, such as `1/3 = 33%`; High means +all required strategies passed, Medium means some passed, and Low means none passed. +This behavioral score is intentionally shown separately from Python and Rust LOC. + Coverage reports are written outside the three strategy folders at `target/rust-python-harness/`. Open `python-html/index.html` to inspect executed and missing Python lines; `python.json` and `python.xml` are available for automation. diff --git a/tests/rust-python-harness/cli.py b/tests/rust-python-harness/cli.py index fe492fe1a6e..41c46f9613a 100644 --- a/tests/rust-python-harness/cli.py +++ b/tests/rust-python-harness/cli.py @@ -158,7 +158,11 @@ def main(argv: Sequence[str] | None = None) -> int: visible_strategies = tuple( strategy for strategy in strategies if strategy.id in selected_strategy_ids ) - dashboard = make_dashboard(visible_strategies, plain=args.plain) + dashboard = make_dashboard( + visible_strategies, + plain=args.plain, + confidence_strategies=strategies, + ) pytest_args = [*args.pytest_arg] if args.coverage: pytest_args.extend(_coverage_pytest_args()) diff --git a/tests/rust-python-harness/models.py b/tests/rust-python-harness/models.py index c6019264a30..21097e0f7d0 100644 --- a/tests/rust-python-harness/models.py +++ b/tests/rust-python-harness/models.py @@ -27,6 +27,12 @@ class RunStatus(str, Enum): NOT_APPLICABLE = "not_applicable" +class ConfidenceLevel(str, Enum): + HIGH = "HIGH" + MEDIUM = "MEDIUM" + LOW = "LOW" + + SDK_FUNCTIONS = ("ocr", "messages", "responses", "count_tokens") @@ -140,3 +146,74 @@ class HarnessRun: for result in results.values(): result.set_initial_status() return cls(results=results) + + +@dataclass(frozen=True) +class SectionConfidence: + sdk_function: str + verified_strategies: int + required_strategies: int + level: ConfidenceLevel + details: tuple[str, ...] + + @property + def percentage(self) -> int: + if not self.required_strategies: + return 0 + return round(100 * self.verified_strategies / self.required_strategies) + + +def section_confidence( + run: HarnessRun, strategies: Iterable[Strategy] +) -> tuple[SectionConfidence, ...]: + strategy_list = tuple(strategies) + scores: list[SectionConfidence] = [] + for sdk_function in SDK_FUNCTIONS: + cases = tuple( + case + for strategy in strategy_list + for case in strategy.cases + if case.sdk_function == sdk_function + and case.coverage is not Coverage.NOT_APPLICABLE + ) + verified = 0 + details: list[str] = [] + for case in cases: + result = run.results.get(case.key) + status = result.status if result is not None else RunStatus.NOT_RUN + if status is RunStatus.PASSED: + verified += 1 + details.append( + f"{STATUS_LABELS[status]} {case.strategy_id} ({case.coverage.value})" + ) + required = len(cases) + if required and verified == required: + level = ConfidenceLevel.HIGH + elif verified: + level = ConfidenceLevel.MEDIUM + else: + level = ConfidenceLevel.LOW + scores.append( + SectionConfidence( + sdk_function=sdk_function, + verified_strategies=verified, + required_strategies=required, + level=level, + details=tuple(details), + ) + ) + return tuple(scores) + + +STATUS_LABELS = { + RunStatus.NOT_RUN: "·", + RunStatus.QUEUED: "○", + RunStatus.RUNNING: "◉", + RunStatus.PASSED: "✓", + RunStatus.FAILED: "✗", + RunStatus.SKIPPED: "↷", + RunStatus.ERROR: "!", + RunStatus.MISSING: "?", + RunStatus.PLANNED: "—", + RunStatus.NOT_APPLICABLE: "n/a", +} diff --git a/tests/rust-python-harness/ui.py b/tests/rust-python-harness/ui.py index 0c8f9ebed8b..57fedd17fa6 100644 --- a/tests/rust-python-harness/ui.py +++ b/tests/rust-python-harness/ui.py @@ -8,7 +8,14 @@ from contextlib import AbstractContextManager from pathlib import Path from typing import Any -from .models import Coverage, HarnessRun, RunStatus, SDK_FUNCTIONS, Strategy +from .models import ( + Coverage, + HarnessRun, + RunStatus, + SDK_FUNCTIONS, + Strategy, + section_confidence, +) STATUS_GLYPHS = { RunStatus.NOT_RUN: "·", @@ -74,11 +81,16 @@ def _cell_text(run: HarnessRun, strategy_id: str, sdk_function: str) -> tuple[st class RichDashboard(AbstractContextManager["RichDashboard"]): - def __init__(self, strategies: Sequence[Strategy]) -> None: + def __init__( + self, + strategies: Sequence[Strategy], + confidence_strategies: Sequence[Strategy], + ) -> None: from rich.console import Console from rich.live import Live self.strategies = strategies + self.confidence_strategies = confidence_strategies self.console = Console() self.live: Any = Live( console=self.console, refresh_per_second=12, transient=False @@ -176,6 +188,29 @@ class RichDashboard(AbstractContextManager["RichDashboard"]): for nodeid, duration in slow ) ) + from rich import box + from rich.table import Table + + confidence_table = Table( + title="Port confidence by SDK section", box=box.ROUNDED, expand=True + ) + confidence_table.add_column("SDK section") + confidence_table.add_column("Score", justify="right") + confidence_table.add_column("Confidence") + confidence_table.add_column("Strategy evidence", ratio=4) + confidence_styles = {"HIGH": "green", "MEDIUM": "yellow", "LOW": "red"} + for score in section_confidence(run, self.confidence_strategies): + confidence_table.add_row( + score.sdk_function, + f"{score.verified_strategies}/{score.required_strategies} {score.percentage}%", + f"[{confidence_styles[score.level.value]}]{score.level.value}[/]", + " ".join(score.details), + ) + self.console.print(confidence_table) + self.console.print( + "[dim]Score = required strategies with passing evidence. " + "LOC coverage remains a separate report.[/dim]" + ) style = "green" if exit_code == 0 else "red" self.console.print( f"[{style}]Harness finished in {_format_duration(run.duration)} " @@ -184,8 +219,13 @@ class RichDashboard(AbstractContextManager["RichDashboard"]): class PlainDashboard(AbstractContextManager["PlainDashboard"]): - def __init__(self, strategies: Sequence[Strategy]) -> None: + def __init__( + self, + strategies: Sequence[Strategy], + confidence_strategies: Sequence[Strategy], + ) -> None: self.strategies = strategies + self.confidence_strategies = confidence_strategies self._seen: dict[str, tuple[RunStatus, int]] = {} def __enter__(self) -> "PlainDashboard": @@ -218,12 +258,28 @@ class PlainDashboard(AbstractContextManager["PlainDashboard"]): ) for nodeid, _ in run.failures[:5]: print(f"Rerun: {_rerun_command(nodeid)}", flush=True) + print("Port confidence by SDK section", flush=True) + for score in section_confidence(run, self.confidence_strategies): + print( + f" {score.sdk_function:12} " + f"{score.verified_strategies}/{score.required_strategies} " + f"{score.percentage:3}% {score.level.value:6} " + f"{' | '.join(score.details)}", + flush=True, + ) + print( + " Score = required strategies with passing evidence; LOC is reported separately.", + flush=True, + ) print(f"Harness finished with exit code {exit_code}", flush=True) def make_dashboard( - strategies: Sequence[Strategy], plain: bool = False + strategies: Sequence[Strategy], + plain: bool = False, + confidence_strategies: Sequence[Strategy] | None = None, ) -> RichDashboard | PlainDashboard: + confidence_strategies = confidence_strategies or strategies interactive_terminal = ( sys.stdout.isatty() and not os.environ.get("CI") @@ -233,7 +289,7 @@ def make_dashboard( try: import rich # noqa: F401 - return RichDashboard(strategies) + return RichDashboard(strategies, confidence_strategies) except ImportError: pass - return PlainDashboard(strategies) + return PlainDashboard(strategies, confidence_strategies) diff --git a/tests/test_rust_python_harness.py b/tests/test_rust_python_harness.py index 9a3cc8f7bf7..90dc38663dd 100644 --- a/tests/test_rust_python_harness.py +++ b/tests/test_rust_python_harness.py @@ -22,6 +22,7 @@ HarnessCase = models.HarnessCase HarnessRun = models.HarnessRun RunStatus = models.RunStatus SDK_FUNCTIONS = models.SDK_FUNCTIONS +section_confidence = models.section_confidence run_pytest = runner.run_pytest runnable_selectors = runner.runnable_selectors selector_matches_node = runner.selector_matches_node @@ -213,3 +214,23 @@ def test_should_build_python_coverage_reports_below_the_target_directory( assert f"--cov-report=json:{tmp_path / 'python.json'}" in args assert f"--cov-report=xml:{tmp_path / 'python.xml'}" in args assert f"--cov-report=html:{tmp_path / 'python-html'}" in args + + +def test_should_report_confidence_for_each_sdk_section() -> None: + strategies = load_catalog() + cases = tuple(case for strategy in strategies for case in strategy.cases) + run = HarnessRun.from_cases(cases) + passing = run.results["e2e_fuzz_tests:responses"] + passing.collected.add("tests/test_parity.py::test_one") + passing.record("tests/test_parity.py::test_one", RunStatus.PASSED) + + scores = { + score.sdk_function: score for score in section_confidence(run, strategies) + } + + assert scores["responses"].verified_strategies == 1 + assert scores["responses"].required_strategies == 3 + assert scores["responses"].percentage == 33 + assert scores["responses"].level.value == "MEDIUM" + assert scores["count_tokens"].percentage == 0 + assert scores["count_tokens"].level.value == "LOW"