fix(eval): read CLI-arm transcripts from the persisted artifact records

Result rows store transcript_artifacts as the {"path", "sha256", "bytes",
"source"} records runner_sessions persists, not bare paths, so joining
results_dir with an entry raised TypeError on real output. Read the
record's path, and skip anything that resolves outside the results dir.
The report test now uses the real record shape.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
abhigyantrumio 2026-10-29 13:49:18 +05:30
parent f326072e4f
commit 39bee52c10
2 changed files with 16 additions and 4 deletions

View file

@ -211,8 +211,15 @@ def test_cli_arm_report_compares_arms_and_counts_shell_calls(tmp_path):
paired, cli = _write_results(
tmp_path,
[_report_row("baseline_nomcp", 0, 1.0), _report_row("baseline", 0, 1.3, gitnexus_tool_uses=3)],
[_report_row("baseline_cli", 0, 1.1, transcript_artifacts=["transcripts/t-cli.jsonl"])],
# The record shape runner_sessions._persist_parent_event_stream writes into result rows.
[_report_row("baseline_cli", 0, 1.1, transcript_artifacts=[
{"path": "transcripts/t-cli.jsonl", "sha256": "0" * 64, "bytes": 1,
"source": "parent-captured-stream-json"},
{"path": "../outside.jsonl", "sha256": "0" * 64, "bytes": 1, "source": "parent-captured-stream-json"}])],
)
(tmp_path / "outside.jsonl").write_text(json.dumps(
{"type": "assistant", "message": {"content": [
{"type": "tool_use", "name": "Bash", "input": {"command": "gitnexus query x"}}]}}) + "\n")
(cli.parent / "transcripts").mkdir()
event = {"type": "assistant", "message": {"content": [
{"type": "tool_use", "name": "Bash", "input": {"command": "gitnexus impact foo && gitnexus help"}},

View file

@ -41,9 +41,14 @@ def cli_commands(row: dict[str, Any], results_dir: Path) -> Counter:
"""Count `gitnexus <command>` invocations in a cell's Bash tool calls."""
counts: Counter = Counter()
for name in row.get("transcript_artifacts", []):
path = results_dir / name
if not path.is_file():
root = results_dir.resolve()
# Each entry is the record runner_sessions persists: {"path", "sha256", "bytes", "source"}.
for artifact in row.get("transcript_artifacts", []):
relative = artifact.get("path") if isinstance(artifact, dict) else None
if not isinstance(relative, str):
continue
path = (results_dir / relative).resolve()
if not path.is_relative_to(root) or not path.is_file():
continue
for line in path.read_text(errors="replace").splitlines():
try: