diff --git a/eval/tests/test_cli_shim.py b/eval/tests/test_cli_shim.py index 6c3e6abf0..8686d65b2 100644 --- a/eval/tests/test_cli_shim.py +++ b/eval/tests/test_cli_shim.py @@ -211,8 +211,15 @@ def test_cli_arm_report_compares_arms_and_counts_shell_calls(tmp_path): paired, cli = _write_results( tmp_path, [_report_row("baseline_nomcp", 0, 1.0), _report_row("baseline", 0, 1.3, gitnexus_tool_uses=3)], - [_report_row("baseline_cli", 0, 1.1, transcript_artifacts=["transcripts/t-cli.jsonl"])], + # The record shape runner_sessions._persist_parent_event_stream writes into result rows. + [_report_row("baseline_cli", 0, 1.1, transcript_artifacts=[ + {"path": "transcripts/t-cli.jsonl", "sha256": "0" * 64, "bytes": 1, + "source": "parent-captured-stream-json"}, + {"path": "../outside.jsonl", "sha256": "0" * 64, "bytes": 1, "source": "parent-captured-stream-json"}])], ) + (tmp_path / "outside.jsonl").write_text(json.dumps( + {"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": "Bash", "input": {"command": "gitnexus query x"}}]}}) + "\n") (cli.parent / "transcripts").mkdir() event = {"type": "assistant", "message": {"content": [ {"type": "tool_use", "name": "Bash", "input": {"command": "gitnexus impact foo && gitnexus help"}}, diff --git a/eval/workflow_bench/cli_arm_report.py b/eval/workflow_bench/cli_arm_report.py index 2b1f03aa0..dd33ea6a8 100644 --- a/eval/workflow_bench/cli_arm_report.py +++ b/eval/workflow_bench/cli_arm_report.py @@ -41,9 +41,14 @@ def cli_commands(row: dict[str, Any], results_dir: Path) -> Counter: """Count `gitnexus ` invocations in a cell's Bash tool calls.""" counts: Counter = Counter() - for name in row.get("transcript_artifacts", []): - path = results_dir / name - if not path.is_file(): + root = results_dir.resolve() + # Each entry is the record runner_sessions persists: {"path", "sha256", "bytes", "source"}. + for artifact in row.get("transcript_artifacts", []): + relative = artifact.get("path") if isinstance(artifact, dict) else None + if not isinstance(relative, str): + continue + path = (results_dir / relative).resolve() + if not path.is_relative_to(root) or not path.is_file(): continue for line in path.read_text(errors="replace").splitlines(): try: