mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-11-17 09:33:02 +00:00
fix(eval): read CLI-arm transcripts from the persisted artifact records
Result rows store transcript_artifacts as the {"path", "sha256", "bytes",
"source"} records runner_sessions persists, not bare paths, so joining
results_dir with an entry raised TypeError on real output. Read the
record's path, and skip anything that resolves outside the results dir.
The report test now uses the real record shape.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f326072e4f
commit
39bee52c10
2 changed files with 16 additions and 4 deletions
|
|
@ -211,8 +211,15 @@ def test_cli_arm_report_compares_arms_and_counts_shell_calls(tmp_path):
|
|||
paired, cli = _write_results(
|
||||
tmp_path,
|
||||
[_report_row("baseline_nomcp", 0, 1.0), _report_row("baseline", 0, 1.3, gitnexus_tool_uses=3)],
|
||||
[_report_row("baseline_cli", 0, 1.1, transcript_artifacts=["transcripts/t-cli.jsonl"])],
|
||||
# The record shape runner_sessions._persist_parent_event_stream writes into result rows.
|
||||
[_report_row("baseline_cli", 0, 1.1, transcript_artifacts=[
|
||||
{"path": "transcripts/t-cli.jsonl", "sha256": "0" * 64, "bytes": 1,
|
||||
"source": "parent-captured-stream-json"},
|
||||
{"path": "../outside.jsonl", "sha256": "0" * 64, "bytes": 1, "source": "parent-captured-stream-json"}])],
|
||||
)
|
||||
(tmp_path / "outside.jsonl").write_text(json.dumps(
|
||||
{"type": "assistant", "message": {"content": [
|
||||
{"type": "tool_use", "name": "Bash", "input": {"command": "gitnexus query x"}}]}}) + "\n")
|
||||
(cli.parent / "transcripts").mkdir()
|
||||
event = {"type": "assistant", "message": {"content": [
|
||||
{"type": "tool_use", "name": "Bash", "input": {"command": "gitnexus impact foo && gitnexus help"}},
|
||||
|
|
|
|||
|
|
@ -41,9 +41,14 @@ def cli_commands(row: dict[str, Any], results_dir: Path) -> Counter:
|
|||
"""Count `gitnexus <command>` invocations in a cell's Bash tool calls."""
|
||||
|
||||
counts: Counter = Counter()
|
||||
for name in row.get("transcript_artifacts", []):
|
||||
path = results_dir / name
|
||||
if not path.is_file():
|
||||
root = results_dir.resolve()
|
||||
# Each entry is the record runner_sessions persists: {"path", "sha256", "bytes", "source"}.
|
||||
for artifact in row.get("transcript_artifacts", []):
|
||||
relative = artifact.get("path") if isinstance(artifact, dict) else None
|
||||
if not isinstance(relative, str):
|
||||
continue
|
||||
path = (results_dir / relative).resolve()
|
||||
if not path.is_relative_to(root) or not path.is_file():
|
||||
continue
|
||||
for line in path.read_text(errors="replace").splitlines():
|
||||
try:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue