mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-15 23:32:49 +00:00
d541105340
The first fully-logged skill-evolution run failed for five deterministic reasons that had nothing to do with the candidate under test. Each is fixed at the layer that actually owns the contract: - Strict provider adapters materialize omitted optional string arguments as "". The MCP alias normalizer now treats a blank optional alias as absent (a blank REQUIRED target is still rejected), and a trusted PreToolUse hook strips blank strings before Read/GitNexus tool calls. - MCP semantic errors rode home in a successful envelope and logged as result=ok. SessionProgress now inspects the payload and reports them as semantic-error. - Claude Code's nested sandbox overlays absent root dotfiles with device nodes, which the provenance snapshot read as unauthorized workspace changes. Those names are excluded at the workspace root and hidden from git via an immutable excludes file. - The proposer could not read /evidence from Bash (missing allowRead entry) and had no offline gitnexus runner, so it fell back to npx and hit the network. Both are now mounted; ripgrep is installed in CI. - selected-rows.json advertised host artifact names that do not exist in the mount. Rows now name their staged patch_file/transcript_files, the prompt describes the real layout, and oversized bundles compact artifacts before dropping evidence rows so no row is silently lost. Also replaces two benchmark scenarios that main already satisfies (trivial-version-alias, inv-bug-pdg-note) with non-vacuous ones, verified to fail against a pristine checkout. Co-authored-by: Cursor <cursoragent@cursor.com>
262 lines
9 KiB
Python
262 lines
9 KiB
Python
"""Live progress reporting for long headless sessions.
|
|
|
|
The session event stream is evidence and is redacted before it is written
|
|
anywhere, so progress may only report metadata derived from it. These tests pin
|
|
that boundary along with the signals that distinguish work from a wedged run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import time
|
|
|
|
from workflow_bench.runner_sessions import SessionProgress
|
|
|
|
|
|
def _drain_lines(stream: io.StringIO) -> list[str]:
|
|
return [line for line in stream.getvalue().splitlines() if line.strip()]
|
|
|
|
|
|
def test_progress_reports_bounded_redacted_tool_io_but_never_model_prose() -> None:
|
|
stream = io.StringIO()
|
|
progress = SessionProgress(
|
|
"gen 0 proposer",
|
|
stream=stream,
|
|
heartbeat_s=3600,
|
|
secrets=("SECRET-TOKEN-abc123",),
|
|
)
|
|
events = [
|
|
{"type": "system", "subtype": "init"},
|
|
{
|
|
"type": "assistant",
|
|
"message": {
|
|
"content": [
|
|
{"type": "text", "text": "SECRET-REASONING-abc123"},
|
|
{
|
|
"type": "tool_use",
|
|
"id": "t1",
|
|
"name": "Grep",
|
|
"input": {
|
|
"pattern": "TODO",
|
|
"path": "/workspace",
|
|
"token": "SECRET-TOKEN-abc123",
|
|
},
|
|
},
|
|
]
|
|
},
|
|
},
|
|
{
|
|
"type": "user",
|
|
"message": {
|
|
"content": [
|
|
{
|
|
"type": "tool_result",
|
|
"tool_use_id": "t1",
|
|
"is_error": False,
|
|
"content": "src/a.py:1: TODO " + "x" * 1000,
|
|
}
|
|
]
|
|
},
|
|
},
|
|
{"type": "result", "num_turns": 1, "is_error": False, "total_cost_usd": 1.5},
|
|
]
|
|
for event in events:
|
|
progress.observe((json.dumps(event) + "\n").encode())
|
|
|
|
output = stream.getvalue()
|
|
assert "SECRET-REASONING-abc123" not in output
|
|
assert "SECRET-TOKEN-abc123" not in output
|
|
assert "[REDACTED]" in output
|
|
assert "session initialized" in output
|
|
assert "turn 1 · Grep" in output
|
|
assert 'tool Grep input={"pattern":"TODO","path":"/workspace","token":"[REDACTED]"}' in output
|
|
assert "tool Grep result=ok output=" in output
|
|
assert "truncated" in output
|
|
assert "finished · 1 turns · ok · $1.50" in output
|
|
|
|
|
|
def test_progress_reports_errors_and_mcp_io_but_skips_other_tool_payloads() -> None:
|
|
stream = io.StringIO()
|
|
progress = SessionProgress("flow", stream=stream, heartbeat_s=3600)
|
|
events = [
|
|
{
|
|
"type": "assistant",
|
|
"message": {
|
|
"content": [
|
|
{
|
|
"type": "tool_use",
|
|
"id": "m1",
|
|
"name": "mcp__gitnexus__query",
|
|
"input": {"search_query": "call resolution"},
|
|
},
|
|
{
|
|
"type": "tool_use",
|
|
"id": "e1",
|
|
"name": "Edit",
|
|
"input": {"file_path": "secret.py", "new_string": "do not log"},
|
|
},
|
|
]
|
|
},
|
|
},
|
|
{
|
|
"type": "user",
|
|
"message": {
|
|
"content": [
|
|
{
|
|
"type": "tool_result",
|
|
"tool_use_id": "m1",
|
|
"is_error": True,
|
|
"content": "repository is not indexed",
|
|
},
|
|
{
|
|
"type": "tool_result",
|
|
"tool_use_id": "e1",
|
|
"content": "edited secret.py",
|
|
},
|
|
]
|
|
},
|
|
},
|
|
]
|
|
for event in events:
|
|
progress.observe((json.dumps(event) + "\n").encode())
|
|
|
|
output = stream.getvalue()
|
|
assert 'tool mcp__gitnexus__query input={"search_query":"call resolution"}' in output
|
|
assert 'tool mcp__gitnexus__query result=error output="repository is not indexed"' in output
|
|
assert "do not log" not in output
|
|
assert "edited secret.py" not in output
|
|
|
|
|
|
def test_progress_distinguishes_mcp_semantic_errors_from_transport_success() -> None:
|
|
stream = io.StringIO()
|
|
progress = SessionProgress("flow", stream=stream, heartbeat_s=3600)
|
|
events = [
|
|
{
|
|
"type": "assistant",
|
|
"message": {
|
|
"content": [
|
|
{
|
|
"type": "tool_use",
|
|
"id": "m1",
|
|
"name": "mcp__gitnexus__impact",
|
|
"input": {"target": "missing", "direction": "upstream"},
|
|
}
|
|
]
|
|
},
|
|
},
|
|
{
|
|
"type": "user",
|
|
"message": {
|
|
"content": [
|
|
{
|
|
"type": "tool_result",
|
|
"tool_use_id": "m1",
|
|
"is_error": False,
|
|
"content": [
|
|
{
|
|
"type": "text",
|
|
"text": '{"error":"Target missing not found"}\n\n---\n**Next:** retry',
|
|
}
|
|
],
|
|
}
|
|
]
|
|
},
|
|
},
|
|
]
|
|
for event in events:
|
|
progress.observe((json.dumps(event) + "\n").encode())
|
|
|
|
assert "tool mcp__gitnexus__impact result=semantic-error" in stream.getvalue()
|
|
|
|
|
|
def test_progress_calls_out_api_retries_because_that_is_the_stuck_signature() -> None:
|
|
stream = io.StringIO()
|
|
progress = SessionProgress("proposer", stream=stream, heartbeat_s=3600)
|
|
event = {
|
|
"type": "system",
|
|
"subtype": "api_retry",
|
|
"attempt": 7,
|
|
"max_retries": 10,
|
|
"retry_delay_ms": 34199.87,
|
|
"error": "unknown",
|
|
}
|
|
progress.observe((json.dumps(event) + "\n").encode())
|
|
|
|
line = _drain_lines(stream)[-1]
|
|
assert "API retry 7/10 in 34s" in line
|
|
assert "no response from the model endpoint" in line
|
|
|
|
|
|
def test_progress_speaks_up_while_a_session_is_silent() -> None:
|
|
stream = io.StringIO()
|
|
with SessionProgress("proposer", stream=stream, heartbeat_s=0.05):
|
|
time.sleep(0.35)
|
|
|
|
heartbeats = [line for line in _drain_lines(stream) if "still running" in line]
|
|
assert heartbeats, "a silent session must still report that it is alive"
|
|
assert "0 turns" in heartbeats[0]
|
|
|
|
|
|
def test_progress_survives_partial_chunks_garbage_and_unbounded_lines() -> None:
|
|
stream = io.StringIO()
|
|
progress = SessionProgress("proposer", stream=stream, heartbeat_s=3600)
|
|
payload = json.dumps(
|
|
{"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "t1", "name": "Bash"}]}}
|
|
).encode()
|
|
# An event split across reads, non-JSON noise, and a huge newline-free run.
|
|
progress.observe(payload[:10])
|
|
progress.observe(payload[10:] + b"\nnot json at all\n")
|
|
progress.observe(b"x" * (4 * 1024 * 1024))
|
|
progress.observe(b'\n{"type":"result","num_turns":2,"is_error":true}\n')
|
|
|
|
output = stream.getvalue()
|
|
assert "turn 1 · Bash" in output
|
|
assert "finished · 2 turns · error" in output
|
|
|
|
|
|
def test_progress_sanitizes_a_hostile_tool_name() -> None:
|
|
stream = io.StringIO()
|
|
progress = SessionProgress("proposer", stream=stream, heartbeat_s=3600)
|
|
event = {
|
|
"type": "assistant",
|
|
"message": {"content": [{"type": "tool_use", "id": "t1", "name": "Bash\nFAKE-LOG-LINE injected"}]},
|
|
}
|
|
progress.observe((json.dumps(event) + "\n").encode())
|
|
|
|
assert "FAKE-LOG-LINE" not in stream.getvalue()
|
|
assert len(_drain_lines(stream)) == 1
|
|
|
|
|
|
def test_cell_failure_detail_line_explains_why_a_cell_failed() -> None:
|
|
from workflow_bench.runner import cell_failure_detail_line
|
|
|
|
assert cell_failure_detail_line("t", "workflow", 0, {"error_kind": None}) is None
|
|
assert cell_failure_detail_line("t", "workflow", 0, {"error_kind": "x"}) is None
|
|
|
|
line = cell_failure_detail_line(
|
|
"trivial-status-json-alias",
|
|
"candidate_workflow",
|
|
1,
|
|
{
|
|
"error_kind": "plan-evidence-invalid",
|
|
"error_detail": "unauthorized workspace path\ntoken=sk-secret-value",
|
|
},
|
|
("sk-secret-value",),
|
|
)
|
|
assert line is not None
|
|
assert line.startswith("[trivial-status-json-alias][candidate_workflow][run 1] detail: ")
|
|
assert "unauthorized workspace path" in line
|
|
assert "sk-secret-value" not in line
|
|
assert "\n" not in line
|
|
|
|
|
|
def test_cell_failure_detail_line_bounds_a_huge_detail() -> None:
|
|
from workflow_bench.runner import MAX_CELL_DETAIL_CHARS, cell_failure_detail_line
|
|
|
|
line = cell_failure_detail_line(
|
|
"t", "workflow", 0, {"error_kind": "session-error", "error_detail": {"stdout_tail": "y" * 50_000}}
|
|
)
|
|
assert line is not None
|
|
assert "truncated" in line
|
|
assert len(line) < MAX_CELL_DETAIL_CHARS + 200
|