From 39bee52c10060cbac10434ccfbcab00846c3018c Mon Sep 17 00:00:00 2001 From: abhigyantrumio Date: Thu, 29 Oct 2026 13:49:18 +0530 Subject: [PATCH] fix(eval): read CLI-arm transcripts from the persisted artifact records Result rows store transcript_artifacts as the {"path", "sha256", "bytes", "source"} records runner_sessions persists, not bare paths, so joining results_dir with an entry raised TypeError on real output. Read the record's path, and skip anything that resolves outside the results dir. The report test now uses the real record shape. Co-Authored-By: Claude Opus 5.5 --- eval/tests/test_cli_shim.py | 9 ++++++++- eval/workflow_bench/cli_arm_report.py | 11 ++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/eval/tests/test_cli_shim.py b/eval/tests/test_cli_shim.py index 6c3e6abf0..8686d65b2 100644 --- a/eval/tests/test_cli_shim.py +++ b/eval/tests/test_cli_shim.py @@ -211,8 +211,15 @@ def test_cli_arm_report_compares_arms_and_counts_shell_calls(tmp_path): paired, cli = _write_results( tmp_path, [_report_row("baseline_nomcp", 0, 1.0), _report_row("baseline", 0, 1.3, gitnexus_tool_uses=3)], - [_report_row("baseline_cli", 0, 1.1, transcript_artifacts=["transcripts/t-cli.jsonl"])], + # The record shape runner_sessions._persist_parent_event_stream writes into result rows. + [_report_row("baseline_cli", 0, 1.1, transcript_artifacts=[ + {"path": "transcripts/t-cli.jsonl", "sha256": "0" * 64, "bytes": 1, + "source": "parent-captured-stream-json"}, + {"path": "../outside.jsonl", "sha256": "0" * 64, "bytes": 1, "source": "parent-captured-stream-json"}])], ) + (tmp_path / "outside.jsonl").write_text(json.dumps( + {"type": "assistant", "message": {"content": [ + {"type": "tool_use", "name": "Bash", "input": {"command": "gitnexus query x"}}]}}) + "\n") (cli.parent / "transcripts").mkdir() event = {"type": "assistant", "message": {"content": [ {"type": "tool_use", "name": "Bash", "input": {"command": "gitnexus impact foo && gitnexus help"}}, diff --git a/eval/workflow_bench/cli_arm_report.py b/eval/workflow_bench/cli_arm_report.py index 2b1f03aa0..dd33ea6a8 100644 --- a/eval/workflow_bench/cli_arm_report.py +++ b/eval/workflow_bench/cli_arm_report.py @@ -41,9 +41,14 @@ def cli_commands(row: dict[str, Any], results_dir: Path) -> Counter: """Count `gitnexus ` invocations in a cell's Bash tool calls.""" counts: Counter = Counter() - for name in row.get("transcript_artifacts", []): - path = results_dir / name - if not path.is_file(): + root = results_dir.resolve() + # Each entry is the record runner_sessions persists: {"path", "sha256", "bytes", "source"}. + for artifact in row.get("transcript_artifacts", []): + relative = artifact.get("path") if isinstance(artifact, dict) else None + if not isinstance(relative, str): + continue + path = (results_dir / relative).resolve() + if not path.is_relative_to(root) or not path.is_file(): continue for line in path.read_text(errors="replace").splitlines(): try: