From f6127c5aee86bece366edd839b582d65b745fdaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Fri, 30 Oct 2026 12:58:34 +0000 Subject: [PATCH 01/26] feat(eval): measure cost drivers in evaluation summaries (#3773) * feat(eval): add cost diagnostics and trials * style(eval): format retrieval case fixtures * fix(eval): address review and CI failures Refs #3773. Validate locked uv commands, model aliases and oversized budgets. Stop the fake MCP server before releasing its listener and replace the wall-clock staleness assertion with a concurrent result check. Note: pre-existing failure in test_oracle_assets.py reference-checkout type resolution not addressed by this PR. * feat(eval): publish detailed run measurements Collect per-sweep provider ledgers and publish sanitized measurements in the Actions summary and artifacts. Validate experiment tasks with the runner and withhold incomplete token totals. Refs #3773. Note: pre-existing failure in test_oracle_assets.py reference-checkout type resolution not addressed by this PR. * fix(eval): validate inputs and quoted commands Reject non-mapping task YAML before reading fields. Attribute literal quoted CLI queries while preserving Bash attribution for shell operators and expansions. Refs #3773. * fix(eval): count empty filtered query results * fix(eval): handle alternate event content --------- Co-authored-by: Gergo Magyar --- .github/workflows/release-evaluation.yml | 55 ++ eval/tests/test_cli_shim.py | 21 +- eval/tests/test_cost_diagnostics.py | 526 ++++++++++++++++++ eval/tests/test_mock_provider.py | 12 +- eval/tests/test_pr_evaluation.py | 17 + eval/tests/test_retrieval_diagnostics.py | 65 +++ eval/tests/test_session_diagnostics.py | 302 ++++++++++ eval/workflow_bench/README.md | 201 +++++++ eval/workflow_bench/cli_arm_report.py | 7 + eval/workflow_bench/cost_diagnostics.py | 283 ++++++++++ eval/workflow_bench/diagnostic_experiments.py | 162 ++++++ eval/workflow_bench/litellm_usage_callback.py | 32 ++ eval/workflow_bench/mcp_loading_report.py | 5 + eval/workflow_bench/pr_evaluation.py | 7 +- eval/workflow_bench/release_report.py | 6 +- eval/workflow_bench/retrieval_cases.json | 37 ++ eval/workflow_bench/retrieval_diagnostics.py | 92 +++ eval/workflow_bench/runner.py | 36 ++ eval/workflow_bench/runner_sessions.py | 24 +- eval/workflow_bench/session_budget.py | 20 + eval/workflow_bench/session_diagnostics.py | 471 ++++++++++++++++ eval/workflow_bench/workload_diagnostics.py | 247 ++++++++ gitnexus/test/unit/staleness.test.ts | 20 +- 23 files changed, 2628 insertions(+), 20 deletions(-) create mode 100644 eval/tests/test_cost_diagnostics.py create mode 100644 eval/tests/test_retrieval_diagnostics.py create mode 100644 eval/tests/test_session_diagnostics.py create mode 100644 eval/workflow_bench/cost_diagnostics.py create mode 100644 eval/workflow_bench/diagnostic_experiments.py create mode 100644 eval/workflow_bench/retrieval_cases.json create mode 100644 eval/workflow_bench/retrieval_diagnostics.py create mode 100644 eval/workflow_bench/session_budget.py create mode 100644 eval/workflow_bench/session_diagnostics.py create mode 100644 eval/workflow_bench/workload_diagnostics.py diff --git a/.github/workflows/release-evaluation.yml b/.github/workflows/release-evaluation.yml index 996c18aad..f5001a377 100644 --- a/.github/workflows/release-evaluation.yml +++ b/.github/workflows/release-evaluation.yml @@ -414,6 +414,8 @@ jobs: if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 )) (( remaining > 0 )) || { echo '::error::No time remains before the stop and evidence reserve.'; exit 1; } + GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/$runtime/provider-usage.jsonl" \ + GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-$runtime" \ timeout --signal=TERM --kill-after=60 "${remaining}s" \ uv run --locked --extra dev python -m workflow_bench.runner \ --tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \ @@ -437,6 +439,8 @@ jobs: set -euo pipefail remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 )) (( remaining > 0 )) || { echo '::warning::No time remains for the CLI-only arm.'; exit 1; } + GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/candidate/cli-provider-usage.jsonl" \ + GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-cli" \ timeout --signal=TERM --kill-after=60 "${remaining}s" \ uv run --locked --extra dev python -m workflow_bench.runner \ --tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \ @@ -456,6 +460,8 @@ jobs: set -euo pipefail remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 )) (( remaining > 0 )) || { echo '::error::No time remains for deferred sessions before the evidence reserve.'; exit 1; } + GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/candidate/deferred-provider-usage.jsonl" \ + GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-deferred" \ timeout --signal=TERM --kill-after=60 "${remaining}s" \ uv run --locked --extra dev python -m workflow_bench.runner \ --tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \ @@ -539,6 +545,49 @@ jobs: --paired "$candidate/raw/results.jsonl" --cli "$candidate/cli-raw/results.jsonl" \ --out "$RUNNER_TEMP/release-evaluation/public" cat "$RUNNER_TEMP/release-evaluation/public/cli-arm-comparison.md" >> "$GITHUB_STEP_SUMMARY" + - name: Summarize time, cost, and tool measurements + if: ${{ always() && steps.resolve.outputs.sha != '' }} + working-directory: eval + env: + DEFERRED_ARM: ${{ inputs.mcp_tool_loading }} + run: | + set -euo pipefail + root="$RUNNER_TEMP/release-evaluation" + public="$root/public" + mkdir -p "$public" + failed=0 + for variant in stable candidate cli deferred; do + case "$variant" in + stable) + if [[ -n "$INPUT_PR" ]]; then continue; fi + results="$root/stable/raw/results.jsonl" + ledger="$root/stable/provider-usage.jsonl" ;; + candidate) + results="$root/candidate/raw/results.jsonl" + ledger="$root/candidate/provider-usage.jsonl" ;; + cli) + if [[ "$CLI_ARM" != true ]]; then continue; fi + results="$root/candidate/cli-raw/results.jsonl" + ledger="$root/candidate/cli-provider-usage.jsonl" ;; + deferred) + if [[ "$DEFERRED_ARM" != true || -n "$INPUT_PR" ]]; then continue; fi + results="$root/candidate/deferred/results.jsonl" + ledger="$root/candidate/deferred-provider-usage.jsonl" ;; + esac + if ! uv run --locked --extra dev python -m workflow_bench.cost_diagnostics \ + --results "$results" --provider-usage "$ledger" --allow-incomplete \ + --out "$public/$variant-diagnostics.json" --markdown-out "$public/$variant-diagnostics.md"; then + printf '\nDetailed measurements for %s could not be validated; inspect the step log.\n' "$variant" >> "$GITHUB_STEP_SUMMARY" + failed=1 + continue + fi + { + printf '\n
Detailed measurements: %s\n\n' "$variant" + cat "$public/$variant-diagnostics.md" + printf '\n
\n' + } >> "$GITHUB_STEP_SUMMARY" + done + exit "$failed" - name: Upload sanitized release evidence if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -553,6 +602,8 @@ jobs: ${{ runner.temp }}/release-evaluation/public/release-quality-gate.md ${{ runner.temp }}/release-evaluation/public/mcp-tool-loading.json ${{ runner.temp }}/release-evaluation/public/mcp-tool-loading.md + ${{ runner.temp }}/release-evaluation/public/*-diagnostics.json + ${{ runner.temp }}/release-evaluation/public/*-diagnostics.md if-no-files-found: error retention-days: 30 - name: Upload the opt-in CLI-only arm comparison @@ -563,6 +614,8 @@ jobs: path: | ${{ runner.temp }}/release-evaluation/public/cli-arm-comparison.json ${{ runner.temp }}/release-evaluation/public/cli-arm-comparison.md + ${{ runner.temp }}/release-evaluation/public/cli-diagnostics.json + ${{ runner.temp }}/release-evaluation/public/cli-diagnostics.md if-no-files-found: warn retention-days: 30 - name: Upload sanitized PR cost comparison @@ -573,6 +626,8 @@ jobs: path: | ${{ runner.temp }}/release-evaluation/public/pr-cost-comparison.json ${{ runner.temp }}/release-evaluation/public/pr-cost-comparison.md + ${{ runner.temp }}/release-evaluation/public/*-diagnostics.json + ${{ runner.temp }}/release-evaluation/public/*-diagnostics.md if-no-files-found: error retention-days: 30 diff --git a/eval/tests/test_cli_shim.py b/eval/tests/test_cli_shim.py index 8686d65b2..4b23b3a68 100644 --- a/eval/tests/test_cli_shim.py +++ b/eval/tests/test_cli_shim.py @@ -129,13 +129,17 @@ class _FakeMcpServer: self.listener = socket.socket(socket.AF_UNIX) self.listener.bind(str(path)) self.listener.listen(1) + self.listener.settimeout(0.1) + self.stopped = threading.Event() self.thread = threading.Thread(target=self._serve, daemon=True) self.thread.start() def _serve(self) -> None: - while True: + while not self.stopped.is_set(): try: connection, _ = self.listener.accept() + except TimeoutError: + continue except OSError: return with connection, connection.makefile("rwb") as stream: @@ -157,7 +161,20 @@ class _FakeMcpServer: stream.flush() def close(self) -> None: - self.listener.close() + self.stopped.set() + try: + # Wait for accept to exit before releasing its fd for another server. + self.thread.join(timeout=5) + finally: + self.listener.close() + assert not self.thread.is_alive(), "fake MCP server did not stop" + + +@pytest.mark.skipif(not hasattr(socket, "AF_UNIX"), reason="the shim speaks MCP over a Unix socket") +def test_fake_mcp_server_close_stops_idle_accept_thread(tmp_path): + server = _FakeMcpServer(tmp_path / "mcp.sock", []) + server.close() + assert not server.thread.is_alive() @pytest.mark.skipif(not hasattr(socket, "AF_UNIX"), reason="the shim speaks MCP over a Unix socket") diff --git a/eval/tests/test_cost_diagnostics.py b/eval/tests/test_cost_diagnostics.py new file mode 100644 index 000000000..38bbe582c --- /dev/null +++ b/eval/tests/test_cost_diagnostics.py @@ -0,0 +1,526 @@ +"""Offline measurements must not invent attribution, completeness or savings.""" + +import copy +import json +import os +import subprocess +import sys +from collections import Counter +from pathlib import Path + +import pytest +import yaml + +from tests.test_session_diagnostics import result, use +from workflow_bench.cost_diagnostics import build_cost_report, provider_summary +from workflow_bench.diagnostic_experiments import VARIANTS, prepare_plan +from workflow_bench.session_diagnostics import diagnose_session, public_diagnostics +from workflow_bench.workload_diagnostics import classify_bash + + +def trace(actions): + events = [] + for index, (tool, args, body) in enumerate(actions): + events += [use(tool, tool_id=str(index), args=args), result(body, tool_id=str(index))] + return events + + +def test_graph_reference_is_not_confused_with_supplied_source_and_paths_stay_private(): + events = trace([ + ("mcp__gitnexus__query", {}, {"definitions": [{"filePath": "src/private.ts"}]}), + ("mcp__gitnexus__context", {}, {"symbol": {"filePath": "src/private.ts", "content": "secret source"}}), + ("Read", {"file_path": "/workspace/src/private.ts", "offset": 10, "limit": 5}, "source"), + ("Read", {"file_path": "/workspace/src/private.ts", "offset": 10, "limit": 5}, "source"), + ("Read", {"file_path": "/workspace/other/private.ts"}, "unrelated"), + ]) + report = diagnose_session(events, None, 10) + work = report["workload"] + assert work["source_reads"] == 3 + assert work["repeated_source_reads"] == 1 + assert work["reads_after_graph_reference"] == work["reads_after_graph_content_reference"] == 2 + assert work["query_context_read_chains"] == 2 + assert "private.ts" not in json.dumps(public_diagnostics([report])) + assert "secret source" not in json.dumps(report) + + +def test_read_before_graph_result_is_not_credited_as_a_followup(): + events = [use(), use("Read", tool_id="read", args={"file_path": "src/a.ts"}), + result({"filePath": "src/a.ts"}), result("source", tool_id="read")] + work = diagnose_session(events, None, 10)["workload"] + assert work["reads_after_graph_reference"] == 0 + + +def test_server_reported_time_is_separate_from_host_wait_and_missing_stays_unknown(): + events = trace([("mcp__gitnexus__query", {}, {"timing": {"wall": 125}, "epistemic": "lower-bound"}), + ("mcp__gitnexus__impact", {}, {"risk": "LOW"})]) + report = diagnose_session(events, [0, 2, 3, 5], 6) + safe = public_diagnostics([report])[0] + assert safe["tools"]["mcp__gitnexus__query"]["server_wall_s"] == 0.125 + assert safe["tools"]["mcp__gitnexus__query"]["elapsed_s"] == 2 + assert safe["tools"]["mcp__gitnexus__query"]["lower_bound"] == 1 + assert safe["tools"]["mcp__gitnexus__impact"]["server_wall_s"] is None + + +@pytest.mark.parametrize("command", ["cd gitnexus && npm test", "uv run --locked --extra dev python -m pytest"]) +def test_verification_repetition_respects_edits_and_unknown_shell_mutations(command): + check = ("Bash", {"command": command}, "ok") + events = trace([check, check, ("Edit", {"file_path": "secret"}, "ok"), check, + ("Bash", {"command": "python custom_script.py"}, "ok"), check]) + work = diagnose_session(events, None, 10)["workload"] + assert work["verification_calls"] == 4 + assert work["verification_repeated_without_observed_change"] == 1 + assert work["verification_repeated_after_possible_change"] == 2 + assert work["edits_after_verification"] == 1 + + +@pytest.mark.parametrize(("command", "category"), [ + ("uv run python -m pytest -q", "test"), ("npm run test:unit", "test"), + ("uv run --locked --extra dev python -m pytest -q", "test"), + ("uv run --frozen --no-sync --offline --extra=dev pytest", "test"), + ("uv run --project ./eval --group dev --python 3.12 -- pytest", "test"), + ("uv run --locked --extra dev ruff check .", "lint"), + ("uv run --locked --extra dev ruff format .", "mutation"), + ("uv run --unknown pytest", "other"), ("uv run --extra", "other"), + ("uv run --extra= pytest", "other"), ("uv run --extra --locked pytest", "other"), + ("uv run --locked=true pytest", "other"), ("uv run --locked", "other"), + ("uv run -- --locked pytest", "other"), + ("npx tsc --noEmit", "typecheck"), ("npx eslint .", "lint"), + ("npx eslint . --fix", "mutation"), ("npm run build", "build"), + ("cat source.ts", "source"), ("rg 'x' src", "source"), + ("echo pytest", "other"), ("npm test && npm run build", "mixed"), + ("cd gitnexus && npm test && touch x", "mixed"), + ("cat x > y", "mixed"), ("sed -i 's/x/y/' a", "mutation"), + ("sed --in-place 's/x/y/' a", "mutation"), ("npx prettier -w .", "mutation"), + ("uv run ruff format .", "mutation"), ("uv run ruff format --check .", "lint"), + ("echo $(npm test)", "mixed"), ("unterminated '", "other"), +]) +def test_shell_classification_is_conservative(command, category): + assert classify_bash(command) == category + + +def test_usage_missing_fields_are_unknown_and_cache_counts_are_not_double_counted(): + first = {"type": "assistant", "message": {"id": "a", "usage": { + "input_tokens": 10, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20, "output_tokens": 4, + }}} + second = {"type": "assistant", "message": {"id": "b", "usage": {"input_tokens": 30}}} + report = diagnose_session([first, second, {"type": "system", "subtype": "compact_boundary"}], None, 5) + work = report["workload"] + assert work["usage_snapshots"] == 2 and work["complete_input_snapshots"] == 1 + assert work["max_input_tokens"] == work["first_input_tokens"] == 130 + assert work["sum_input_tokens"] is work["last_input_tokens"] is None + assert work["sum_cache_read_tokens"] is work["sum_output_tokens"] is None + assert work["compactions"] == 1 + work["prompt"] = "secret" + assert "secret" not in json.dumps(public_diagnostics([report])) + + +def provider_event(): + return {"status": "success", "response_id": "one", "sweep_id": "sweep", "provider": "litellm-normalized", + "native_usage": {"prompt_tokens": 100, "completion_tokens": 20, + "prompt_tokens_details": {"cached_tokens": 60, "cache_write_tokens": 10}}, + "request_metrics": {"tools_bytes": 300, "duration_s": 2}} + + +def test_provider_deduplicates_identical_hooks_and_refuses_conflicting_totals(): + event = provider_event() + report = provider_summary([event, copy.deepcopy(event)]) + assert report["observed_requests"] == 1 and report["duplicate_callbacks"] == 1 + assert report["token_totals"]["total_input_tokens"] == 100 + assert report["token_totals"]["ordinary_input_tokens"] == 30 + assert report["token_totals"]["reasoning_output_tokens"] is None + assert report["attribution"] == "ledger_only_no_cell_join" + conflicting = copy.deepcopy(event) + conflicting["native_usage"]["prompt_tokens"] = 120 + report = provider_summary([event, conflicting]) + assert report["conflicting_response_ids"] == 1 + assert all(value is None for value in report["token_totals"].values()) + + +def test_missing_identity_failed_usage_and_unknown_providers_remain_visible(): + event = provider_event() + del event["response_id"] + report = provider_summary([event, event, {"status": "failure", "provider": "unknown"}]) + assert report["requests_without_response_id"] == 3 and not report["identity_complete"] + assert report["failures"] == report["invalid_usage_records"] == 1 + assert report["token_totals"]["total_input_tokens"] is None + assert report["request_metrics"]["tools_bytes"]["missing"] == 1 + + +def test_cost_report_keeps_failures_in_spend_and_missing_telemetry_is_not_zero(): + rows = [{"arm": "baseline", "task": "test", "resolved": True, "ok": True, "cost_usd": 2}, + {"arm": "baseline", "task": "test", "resolved": False, "ok": False, "cost_usd": 3}] + report = build_cost_report(rows) + group = report["groups"][0] + assert group["cost_per_solved"] == 5 + assert group["failed_cells"] == 1 and group["diagnostic_cells"] == 0 + assert group["metrics"]["duration_s"]["mean"] is None + rows[1]["cost_usd"] = float("nan") + assert build_cost_report(rows)["groups"][0]["cost_per_solved"] is None + + +def test_different_model_or_prompt_cohorts_and_duplicate_exports_cannot_blend(): + row = {"arm": "baseline", "task": "test", "run": 0, "model": "model-a", "cost_usd": 2} + other = row | {"model": "model-b", "cost_usd": 20} + assert len(build_cost_report([row, other])["groups"]) == 2 + with pytest.raises(ValueError, match="duplicate cell"): + build_cost_report([row, row]) + + +def test_tool_summaries_preserve_cohort_boundaries_and_budget_treatment(): + diagnostic = diagnose_session(trace([("Read", {"file_path": "source.ts"}, "source")]), None, 5) + row = {"arm": "baseline", "task": "test", "run": 0, "max_budget_usd": None, + "session_diagnostics": [diagnostic]} + report = build_cost_report([row, row | {"max_budget_usd": 5}]) + assert len(report["groups"]) == 2 + assert report["tool_markdown"].count("Diagnostics available for 1/1 cells") == 2 + assert "Diagnostics available for 2/2 cells" not in report["tool_markdown"] + + +def test_proxy_envelope_metrics_preserve_accounting_without_recording_content(tmp_path, monkeypatch): + from types import SimpleNamespace + + from workflow_bench.litellm_usage_callback import USAGE_LOG_ENV_VAR, ProviderUsageLogger + + log = tmp_path / "provider.jsonl" + monkeypatch.setenv(USAGE_LOG_ENV_VAR, str(log)) + kwargs = {"messages": [{"role": "user", "content": "SECRET PROMPT"}], + "optional_params": {"tools": [{"name": "PRIVATE TOOL"}]}, "response_cost": 0.25} + ProviderUsageLogger()._append("success", kwargs, SimpleNamespace(usage={"prompt_tokens": 20}), 10, 12) + payload = log.read_text() + row = json.loads(payload) + assert "SECRET" not in payload and "PRIVATE" not in payload + assert row["request_metrics"]["tools_count"] == 1 + assert row["request_metrics"]["messages_count"] == 1 + assert row["request_metrics"]["duration_s"] == 2 + assert row["request_metrics"]["proxy_estimated_cost_usd"] == 0.25 + ProviderUsageLogger()._append("failure", {"optional_params": [], "response_cost": float("nan")}, + SimpleNamespace(usage={"prompt_tokens": 20}), None, None) + row = json.loads(log.read_text().splitlines()[-1]) + assert row["native_usage"]["prompt_tokens"] == 20 + assert row["request_metrics"]["tools_bytes"] is row["request_metrics"]["duration_s"] is None + + +@pytest.mark.parametrize("fail", [False, True]) +def test_cell_phases_include_cleanup_and_partial_failure_without_changing_credit(monkeypatch, tmp_path, fail): + from tests.test_runner_hardening import _cell_context, _stub_cell_dependencies + from workflow_bench import runner + from workflow_bench.session_diagnostics import CELL_PHASES + + _stub_cell_dependencies(monkeypatch, tmp_path) + if fail: + def explode(*args, **kwargs): + raise RuntimeError("synthetic failure") + monkeypatch.setattr(runner, "run_arm", explode) + row = runner.run_cell(_cell_context(tmp_path), 2, "workflow") + phases = row["cell_phase_seconds"] + assert set(phases) <= set(CELL_PHASES) + assert "cleanup" in phases and "agent_and_verification" in phases + assert ("artifacts" not in phases) == fail + assert sum(phases.values()) == pytest.approx(row["cell_duration_s"]) + assert row["cell_finished_at"] >= row["cell_started_at"] + assert row["resolved"] is (not fail) + + +def test_release_and_cli_artifacts_publish_safe_diagnostics_and_keep_phase_validation(tmp_path): + from tests.test_cli_shim import _report_row, _write_results + from tests.test_release_report import DIGEST, NOW, SHA, metadata, rows + from workflow_bench import cli_arm_report, release_report + + diagnostic = diagnose_session(trace([("Bash", {"command": "gitnexus query secret"}, "ok")]), None, 10) + extra = {"session_diagnostics": [diagnostic], "cell_duration_s": 12, + "cell_phase_seconds": {"clone": 1.0, "secret content": 99, "mcp_startup": float("nan")}} + measurements = rows() + measurements[1].update(extra) + report = release_report.build_report(measurements, metadata(), now=NOW) + release_report.validate_report(report, runtime_sha=SHA, task_set_digest=DIGEST, now=NOW) + assert report["per_run"][1]["cell_phase_seconds"] == {"clone": 1.0, "mcp_startup": None} + assert "secret" not in json.dumps(report) + paired, cli = _write_results(tmp_path, [_report_row("baseline", 0, 1), _report_row("baseline_nomcp", 0, 1)], + [_report_row("baseline_cli", 0, 1, **extra)]) + report = cli_arm_report.build_report(paired, cli) + assert "secret" not in json.dumps(report) + assert "Tool diagnostics: baseline_cli" in cli_arm_report.render_markdown(report) + + +def plan_args(): + return dict(repository="/repo", runtime="/runtime", runtime_sha="a" * 40, harness_sha="b" * 40, + model="gpt-5.6-sol", effort="xhigh", repeats=5, seed=3762, variants=list(VARIANTS)) + + +def task_document(): + return {"tasks": [ + {"id": name, "ref": "c" * 40, "prompt": "Fix this", "class": task_class, "verify": "true", + "oracle": {"command": "true", "files": [{"source": "fixture.txt", "target": "fixture.txt"}]}} + for name, task_class in (("one", "trivial"), ("two", "cross-module")) + ]} + + +def test_plan_is_deterministic_balanced_isolated_and_never_runs_a_model(): + original = task_document() + plan = prepare_plan(original, **plan_args()) + assert plan == prepare_plan(original, **plan_args()) + assert len(plan["cells"]) == 50 and plan["position_balanced"] + assert plan["execution"] == "not_started" and plan["release_gate_eligible"] is False + assert original == task_document() + for task in ("one", "two"): + for variant in VARIANTS: + positions = Counter(c["position"] for c in plan["cells"] if c["task"] == task and c["variant"] == variant) + assert set(positions.values()) == {1} and len(positions) == 5 + ledgers = {c["env"]["GITNEXUS_BENCH_PROVIDER_USAGE"] for c in plan["cells"]} + assert len(ledgers) == 50 + for cell in plan["cells"]: + assert (cell["prompt_sha256"] != cell["base_prompt_sha256"]) == (cell["variant"] == "mcp_optional_prompt") + assert "--unsafe-no-bwrap" not in cell["argv"] + + +@pytest.mark.parametrize("override", [{"runtime_sha": "main"}, {"model": "latest"}, {"repeats": 0}, + {"variants": ["cli", "cli"]}, {"task_ids": ["missing"]}]) +def test_plan_rejects_unpinned_or_ambiguous_inputs(override): + with pytest.raises(ValueError): + prepare_plan(task_document(), **(plan_args() | override)) + + +@pytest.mark.parametrize("document", [None, [], "tasks", 42]) +def test_plan_rejects_non_mapping_yaml_documents(document): + with pytest.raises(ValueError, match="task document must contain tasks"): + prepare_plan(document, **plan_args()) + + +@pytest.mark.parametrize("model", [None, 1, "", " ", " auto ", "claude-sonnet-LATEST", + "provider/latest", "provider:auto", "provider@AUTO"]) +def test_plan_uses_runner_model_validation(model): + with pytest.raises(ValueError): + prepare_plan(task_document(), **(plan_args() | {"model": model})) + + +def test_plan_normalizes_model_in_metadata_and_commands(): + plan = prepare_plan(task_document(), **(plan_args() | {"model": " gpt-5.6-sol "})) + assert plan["model"] == "gpt-5.6-sol" + for cell in plan["cells"]: + assert cell["argv"][cell["argv"].index("--model") + 1] == "gpt-5.6-sol" + + +@pytest.mark.parametrize("task_id", ["_task", "-task", "a" * 129]) +def test_plan_rejects_task_ids_the_runner_cannot_use(task_id): + document = task_document() + document["tasks"][0]["id"] = task_id + with pytest.raises(ValueError, match="artifact-safe slug"): + prepare_plan(document, **plan_args()) + + +def test_every_prepared_task_satisfies_the_runner_metadata_contract(): + from workflow_bench.runner_tasks import select_tasks + + document = task_document() + document["tasks"][0]["id"] = "a.task-1" + plan = prepare_plan(document, **plan_args()) + for cell in plan["cells"]: + selected, skipped = select_tasks(cell["task_document"]["tasks"], include_expensive=True) + assert selected == cell["task_document"]["tasks"] and skipped == [] + document["tasks"][0]["verify"] = "" + with pytest.raises(ValueError, match="verify"): + prepare_plan(document, **plan_args()) + + +@pytest.mark.parametrize("intervening_read", [False, True]) +def test_post_graph_input_totals_require_every_relevant_snapshot(intervening_read): + complete = {"type": "assistant", "message": {"id": "complete", "usage": { + "input_tokens": 10, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20, + }}} + incomplete = {"type": "assistant", "message": {"id": "incomplete", "usage": {"input_tokens": 30}}} + events = [use(), result({}), complete] + if intervening_read: + events += [use("Read", tool_id="read", args={"file_path": "src/a.ts"}), result("source", tool_id="read")] + events.append(incomplete) + work = diagnose_session(events, None, 5)["workload"] + assert work["sum_input_tokens"] is None + assert work["usage_after_graph_snapshots"] == (1 if intervening_read else 2) + assert work["input_tokens_after_graph"] == (130 if intervening_read else None) + # Preserve the actual first/last positions, even when a snapshot is incomplete. + reversed_usage = diagnose_session([incomplete, complete], None, 5)["workload"] + assert reversed_usage["first_input_tokens"] is None and reversed_usage["last_input_tokens"] == 130 + complete_usage = diagnose_session([complete], None, 5)["workload"] + assert complete_usage["sum_input_tokens"] == 130 + + +def test_trial_plan_passes_each_cell_budget_without_starting_a_model(): + args = plan_args() | {"repeats": 1, "variants": ["no_graph", "mcp_eager"], "task_ids": ["one"], "max_budget_usd": 5.0} + plan = prepare_plan(task_document(), **args) + assert len(plan["cells"]) == 2 + assert plan["planned_cli_budget_usd"] == 10.0 + assert plan["execution"] == "not_started" + for cell in plan["cells"]: + assert cell["argv"][cell["argv"].index("--max-budget-usd") + 1] == "5.0" + + +@pytest.mark.parametrize("budget", [0, -1, float("nan"), float("inf"), True, 10 ** 1000]) +def test_invalid_budget_fails_before_any_process_starts(monkeypatch, tmp_path, budget): + from workflow_bench import runner_sessions + + def must_not_start(*args, **kwargs): + pytest.fail("invalid budget launched a process") + + monkeypatch.setattr(runner_sessions, "run_managed", must_not_start) + with pytest.raises(ValueError, match="max-budget-usd"): + runner_sessions.run_claude("task", tmp_path, claude_bin="claude", timeout=1, max_budget_usd=budget) + + +def test_cli_budget_stop_remains_a_failed_billed_session(monkeypatch, tmp_path): + from tests.test_workflow_bench_sessions import fake_cli_result, report_variant + from workflow_bench import runner_sessions + + captured = [] + + def run(command, **kwargs): + captured.extend(command) + return fake_cli_result(report_variant(subtype="error_max_budget_usd", total_cost_usd=5.1)) + + monkeypatch.setattr(runner_sessions, "run_managed", run) + record = runner_sessions.run_claude("task", tmp_path, claude_bin="claude", timeout=1, max_budget_usd=5) + assert captured[captured.index("--max-budget-usd") + 1] == "5.0" + assert record["ok"] is False and record["cost_usd"] == 5.1 + assert record["max_budget_usd"] == 5 + + +def test_budget_rejects_multisession_arms_before_reading_task_inputs(monkeypatch, capsys): + from workflow_bench import runner + + monkeypatch.setattr("sys.argv", ["runner", "--tasks", "nonexistent.yaml", "--model", "gpt-5.6-sol", + "--arms", "workflow", "--max-budget-usd", "5"]) + with pytest.raises(SystemExit) as failure: + runner.main() + assert failure.value.code == 2 + assert "--max-budget-usd is limited to single-session baseline arms" in capsys.readouterr().err + + +def test_summary_cli_gathers_safe_measurements_and_provider_coverage(tmp_path, monkeypatch): + from workflow_bench import cost_diagnostics + + diagnostic = diagnose_session(trace([ + ("mcp__gitnexus__query", {"query": "SECRET QUERY"}, {"staleness": {"status": "behind"}}), + ("Bash", {"command": "uv run --locked --extra dev pytest"}, "SECRET RESULT"), + ]), [0, 2, 3, 5], 8) + rows = [{"task": "one", "arm": "baseline", "run": 0, "cost_usd": 2, "duration_s": 8, + "ok": False, "resolved": False, "session_diagnostics": [diagnostic], "private": "SECRET PATH", + "cell_phase_seconds": {"clone": 1, "agent_and_verification": 9, "PRIVATE PHASE": 123}}] + results, ledger = tmp_path / "results.jsonl", tmp_path / "provider.jsonl" + results.write_text(json.dumps(rows[0]) + "\n") + event = provider_event() | {"messages": "SECRET PROMPT", "response_id": "SECRET ID"} + ledger.write_text(json.dumps(event) + "\n") + out, markdown = tmp_path / "diagnostics.json", tmp_path / "diagnostics.md" + monkeypatch.setattr("sys.argv", ["cost_diagnostics", "--results", str(results), "--provider-usage", str(ledger), + "--out", str(out), "--markdown-out", str(markdown)]) + cost_diagnostics.main() + body = markdown.read_text() + assert "SECRET" not in out.read_text() + body and "PRIVATE PHASE" not in body + assert "Solved: 0/1; failed sessions: 1" in body + assert "| clone | 1 / 0 | 1 |" in body + assert "| verification_calls | 1 | 1/1 |" in body + assert "| tools_bytes | 1 / 0 | 300 |" in body + assert "| total_input_tokens | 100 |" in body + assert "Largest, slowest, and problematic tool calls" in body + assert "requests without identity: 0" in body + assert json.loads(out.read_text())["provider"]["ledger_complete"] is True + + +def test_summary_cli_reports_missing_legacy_telemetry_and_torn_provider_rows(tmp_path, monkeypatch): + from workflow_bench import cost_diagnostics + + results, ledger = tmp_path / "results.jsonl", tmp_path / "provider.jsonl" + results.write_text(json.dumps({"task": "old", "arm": "baseline", "cost_usd": 1}) + "\n{torn") + ledger.write_text(json.dumps(provider_event()) + "\n{torn") + out, markdown = tmp_path / "diagnostics.json", tmp_path / "diagnostics.md" + monkeypatch.setattr("sys.argv", ["cost_diagnostics", "--results", str(results), "--provider-usage", str(ledger), + "--allow-incomplete", "--out", str(out), "--markdown-out", str(markdown)]) + cost_diagnostics.main() + report = json.loads(out.read_text()) + assert len(report["input_problems"]) == 2 + assert report["provider"]["observed_requests"] == 1 + assert report["provider"]["ledger_complete"] is False + assert all(value is None for value in report["provider"]["token_totals"].values()) + body = markdown.read_text() + assert "Partial input" in body and "diagnostics: 0/1" in body + assert "| clone | 0 / 1 | n/a |" in body + assert "| total_input_tokens | n/a |" in body + + +def test_summary_cli_reports_unavailable_inputs_after_early_failure(tmp_path, monkeypatch): + from workflow_bench import cost_diagnostics + + out, markdown = tmp_path / "diagnostics.json", tmp_path / "diagnostics.md" + monkeypatch.setattr("sys.argv", ["cost_diagnostics", "--results", str(tmp_path / "missing"), + "--provider-usage", str(tmp_path / "missing-ledger"), "--allow-incomplete", + "--out", str(out), "--markdown-out", str(markdown)]) + cost_diagnostics.main() + assert "Observed cells: 0" in markdown.read_text() + assert "Provider ledger unavailable" in markdown.read_text() + assert json.loads(out.read_text())["provider"] is None + + +def test_summary_escapes_markdown_and_html_labels(): + from workflow_bench.cost_diagnostics import render_markdown + + report = build_cost_report([{"arm": "baseline", "task": "|bad![x](https://example.test)"}]) + markdown = render_markdown(report) + assert " 0 and metrics["tools_bytes"] > 0 + assert metrics["duration_s"] >= 0 + assert "PRIVATE" not in usage_log.read_text(), "numeric envelope evidence must not retain request content" usage = normalize_usage(event["provider"], native) assert usage.total_input_tokens == 10_000 diff --git a/eval/tests/test_pr_evaluation.py b/eval/tests/test_pr_evaluation.py index c480b934a..2fbe74400 100644 --- a/eval/tests/test_pr_evaluation.py +++ b/eval/tests/test_pr_evaluation.py @@ -73,6 +73,23 @@ def test_pr_comparison_rejects_readable_runtime_cells(evidence): assert not report["complete"] and not report["improved"] +def test_pr_summary_preserves_safe_diagnostics_in_complete_and_partial_reports(evidence): + from tests.test_session_diagnostics import result, use + from workflow_bench.session_diagnostics import diagnose_session + + diagnostic = diagnose_session([use(args={"query": "SECRET"}), result("SECRET")], [1, 2], 3) + evidence[1][0]["session_diagnostics"] = [diagnostic] + evidence[1][0]["cell_phase_seconds"] = {"clone": 2, "SECRET": 9} + report = compare(evidence) + assert report["per_run"][0]["cell_phase_seconds"] == {"clone": 2} + assert "SECRET" not in json.dumps(report) + assert "Tool diagnostics: baseline" in pr_evaluation.render_markdown(report) + evidence[1].pop() + partial = compare(evidence) + assert not partial["complete"] + assert "Tool diagnostics: baseline" in pr_evaluation.render_markdown(partial) + + def test_comparison_measures_only_new_mcp_cells_and_preserves_historical_cost(evidence): before = deepcopy(evidence) report = compare(evidence) diff --git a/eval/tests/test_retrieval_diagnostics.py b/eval/tests/test_retrieval_diagnostics.py new file mode 100644 index 000000000..c80139161 --- /dev/null +++ b/eval/tests/test_retrieval_diagnostics.py @@ -0,0 +1,65 @@ +"""Partial retrieval judgments must not turn unjudged code into false positives.""" + +import copy +import json +from pathlib import Path + +import pytest + +from workflow_bench.retrieval_diagnostics import score_retrieval + +CASES = {"source_sha": "a" * 40, "query_options": {"limit": 5, "max_symbols": 10, "include_content": False}, "cases": [ + {"id": "case", "query": "find handler", "anchors": [{"file": "src/a.ts", "name": "handle"}]}, + {"id": "missing", "query": "another query", "anchors": [{"file": "src/b.ts"}]}, +]} + + +def recordings(): + return {"source_sha": "a" * 40, "runtime_sha": "b" * 40, "index_complete": True, + "index_staleness": "current", "sanitized_graph": True, + "query_options": CASES["query_options"], + "cases": [{"id": "case", "query": "find handler", "response": { + "process_symbols": [{"filePath": "src/unjudged.ts", "name": "other"}], + "definitions": [{"filePath": "src/a.ts", "name": "handle"}], + }}]} + + +def test_partial_relevance_and_missing_cases_do_not_become_precision_or_zero(): + report = score_retrieval(CASES, recordings()) + assert report["cases"][0]["anchor_recall"] == 1 + assert report["cases"][0]["first_anchor_position"] == 2 + assert report["cases"][1]["anchor_recall"] is None + assert not report["complete"] + assert "precision" not in report["cases"][0] + + +@pytest.mark.parametrize("change", [{"source_sha": "c" * 40}, {"index_staleness": "behind"}, + {"index_complete": False}, {"sanitized_graph": False}, + {"sanitized_graph": "false"}, {"runtime_sha": "main"}, {"query_options": {"limit": 1}}]) +def test_unmatched_or_incomplete_index_is_not_scored(change): + with pytest.raises(ValueError): + score_retrieval(CASES, recordings() | change) + + +def test_changed_query_duplicate_case_and_missing_symbol_are_distinct(): + recorded = recordings() + recorded["cases"][0]["query"] = "different question" + with pytest.raises(ValueError, match="query differs"): + score_retrieval(CASES, recorded) + recorded = recordings() + recorded["cases"].append(copy.deepcopy(recorded["cases"][0])) + with pytest.raises(ValueError, match="duplicate"): + score_retrieval(CASES, recorded) + recorded = recordings() + recorded["cases"][0]["response"] = {"process_symbols": [], "definitions": []} + assert score_retrieval(CASES, recorded)["cases"][0]["anchor_recall"] == 0 + + +def test_committed_cases_cover_each_current_scenario_with_pinned_source(): + import yaml + + root = Path(__file__).resolve().parents[1] / "workflow_bench" + cases = json.loads((root / "retrieval_cases.json").read_text()) + tasks = yaml.safe_load((root / "tasks.scenarios.yaml").read_text())["tasks"] + assert {c["id"] for c in cases["cases"]} == {t["id"] for t in tasks} + assert {t["ref"] for t in tasks} == {cases["source_sha"]} diff --git a/eval/tests/test_session_diagnostics.py b/eval/tests/test_session_diagnostics.py new file mode 100644 index 000000000..89b71883e --- /dev/null +++ b/eval/tests/test_session_diagnostics.py @@ -0,0 +1,302 @@ +"""Diagnostics must explain overhead without changing credit or exposing content.""" + +import json +from collections import Counter + +import pytest + +from tests.test_release_report import DIGEST, NOW, SHA, metadata, rows +from tests.test_workflow_bench_sessions import VALID_REPORT, fake_cli_result, session_record +from workflow_bench import runner_sessions, session_diagnostics as diagnostics +from workflow_bench.release_report import build_report, render_markdown, validate_report + + +def use(tool="mcp__gitnexus__query", *, tool_id="one", args=None): + return {"type": "assistant", "message": {"content": [ + {"type": "tool_use", "id": tool_id, "name": tool, "input": args or {"query": "private source"}}, + ]}} + + +def result(body, *, tool_id="one", error=False): + content = body if isinstance(body, str) else [{"type": "text", "text": json.dumps(body) + "\n\n---\nNext"}] + return {"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": tool_id, "is_error": error, "content": content}, + ]}} + + +@pytest.mark.parametrize("message", [None, {}, {"id": "message"}, {"content": None}, [], "invalid"]) +def test_top_level_content_preserves_tool_results_and_source_followups(message): + events = [use(), result({"definitions": []}), use("Read", tool_id="read"), result("source", tool_id="read")] + for event in events: + event["content"] = event["message"]["content"] + event["message"] = message + report = diagnostics.diagnose_session(events, [1, 2, 3, 4], 5) + assert report["timing_complete"] is True + assert report["tool_wait_s"] == 2 + query = report["tools"]["mcp__gitnexus__query"] + assert query["calls"] == query["empty"] == query["source_followups"] == 1 + assert query["unfinished"] == 0 + assert report["tools"]["Read"]["calls"] == 1 + + +@pytest.mark.parametrize("nested", [[], "invalid"]) +def test_present_non_null_nested_content_prevents_top_level_fallback(nested): + events = [use(), result("done")] + for event in events: + event["content"] = event["message"]["content"] + event["message"]["content"] = nested + report = diagnostics.diagnose_session(events, [1, 2], 3) + assert report["calls"] == [] + assert report["tools"] == {} + + +def test_nested_tool_content_wins_over_conflicting_top_level_blocks(): + events = [use(), result("done")] + events[0]["content"] = use("Read", tool_id="shadow")["message"]["content"] + events[1]["content"] = result("shadow", tool_id="shadow")["message"]["content"] + report = diagnostics.diagnose_session(events, [1, 2], 3) + assert list(report["tools"]) == ["mcp__gitnexus__query"] + assert report["tools"]["mcp__gitnexus__query"]["calls"] == 1 + assert report["timing_complete"] is True + assert report["tool_wait_s"] == 1 + + +@pytest.mark.parametrize(("wall", "expected"), [ + (0, 0), (500, 0.5), (123.5, 0.1235), + (None, None), (True, None), ("1", None), (-1, None), + (float("nan"), None), (float("inf"), None), (10**400, None), +]) +def test_server_timing_values_do_not_fail_or_reclassify_completed_tool_calls(wall, expected): + events = [use(), result({"timing": {"wall": wall}})] + report = diagnostics.diagnose_session(events, [1, 2], 3) + tool = report["tools"]["mcp__gitnexus__query"] + assert tool["calls"] == 1 + assert tool["errors"] == tool["unfinished"] == 0 + assert tool["server_wall_s"] == expected + assert tool["server_timed_calls"] == int(expected is not None) + assert report["timing_complete"] is True + assert report["tool_wait_s"] == 1 + + +def test_event_clock_handles_chunk_boundaries_blank_lines_and_unterminated_event(monkeypatch): + ticks = iter([10, 11, 13, 15]) + monkeypatch.setattr(diagnostics.time, "monotonic", lambda: next(ticks)) + clock = diagnostics.EventClock() + clock.observe(b'\n {"message":"') + clock.observe('é"}\r\n\n'.encode()) + clock.observe(b'{"last":true}') + assert clock.finish(2) == [3, 5] + assert clock.finish(3) is None + + +def test_event_clock_bounds_memory_and_never_claims_complete_overflow(monkeypatch): + monkeypatch.setattr(diagnostics, "MAX_EVENTS", 2) + clock = diagnostics.EventClock() + clock.observe(b'{}\n{}\n{}\n') + assert len(clock.offsets) == 2 + assert clock.finish(3) is None + + +def test_overlapping_tools_are_not_double_counted_as_wall_time(): + events = [use(), use("Bash", tool_id="two"), result({"status": "found"}), result("done", tool_id="two")] + report = diagnostics.diagnose_session(events, [1, 2, 5, 7], 10) + assert report["timing_complete"] is True + assert report["tool_wait_s"] == 6 + assert report["mcp_wait_s"] == 4 + assert report["other_wall_s"] == 4 + assert [call["duration_s"] for call in report["calls"]] == [4, 5] + assert report["tools"]["Bash"]["elapsed_s"] == 5 + + +@pytest.mark.parametrize("offsets", [None, [], [1], [3, 2], [-1, 2], [1, 20], [True, 2], [1, float("nan")]]) +def test_unavailable_or_invalid_timing_is_unknown_not_zero(offsets): + report = diagnostics.diagnose_session([use(), result("done")], offsets, 10) + assert report["timing_complete"] is False + assert report["tool_wait_s"] is report["mcp_wait_s"] is report["other_wall_s"] is None + assert report["calls"][0]["duration_s"] is None + + +@pytest.mark.parametrize(("body", "signal"), [ + ({"error": "Symbol 'privateName' not found"}, "not_found"), + ({"status": "ambiguous", "candidates": [{"name": "secret"}]}, "ambiguous"), + ({"risk": "UNKNOWN"}, "unknown_risk"), + ({"processes": [], "process_symbols": [], "definitions": []}, "empty"), + ("Error: result (108,197 characters) exceeds maximum allowed tokens", "output_limit_exceeded"), +]) +def test_resolution_problems_and_followup_source_reads_are_counted(body, signal): + events = [use(), result(body), use("Read", tool_id="read"), result("source", tool_id="read")] + report = diagnostics.diagnose_session(events, [1, 2, 3, 4], 5) + tally = report["tools"]["mcp__gitnexus__query"] + assert tally[signal] == 1 + assert tally["source_followups"] == 1 + assert report["calls"][0]["next_tools"] == ["Read"] + assert "privateName" not in json.dumps(report) + + +def test_transport_errors_are_distinct_from_successful_resolution(): + bad = diagnostics.diagnose_session([use(), result("failure", error=True)], [1, 2], 3) + good = diagnostics.diagnose_session([use(), result({"status": "found"})], [1, 2], 3) + assert bad["tools"]["mcp__gitnexus__query"]["errors"] == 1 + assert good["tools"]["mcp__gitnexus__query"]["errors"] == 0 + + +def test_plain_text_mcp_errors_are_not_counted_as_success(): + report = diagnostics.diagnose_session([use(), result("Error: database unavailable")], [1, 2], 3) + assert report["tools"]["mcp__gitnexus__query"]["errors"] == 1 + + +def test_non_ascii_payload_byte_counts_handle_escaped_lone_surrogates(): + report = diagnostics.diagnose_session([use(args={"query": "é\ud800"}), result("é\ud800")], [1, 2], 3) + assert report["calls"][0]["result_bytes"] == len('"é\\ud800"'.encode()) + assert report["timing_complete"] is True + + +def test_identical_arguments_are_counted_without_publishing_arguments_or_hashes(): + events = [use(args={"token": "super-secret"}), result("done"), + use(tool_id="two", args={"token": "super-secret"}), result("done", tool_id="two")] + report = diagnostics.diagnose_session(events, [1, 2, 3, 4], 5) + assert report["calls"][1]["repeat_of"] == 0 + assert report["tools"]["mcp__gitnexus__query"]["repeated"] == 1 + assert "super-secret" not in json.dumps(report) + assert "signature" not in json.dumps(report) + + +def test_unmatched_results_reused_ids_and_call_limit_make_timing_incomplete(monkeypatch): + for events in ([use()], [result("orphan")], [use(), result("ok"), use(), result("ok")]): + assert diagnostics.diagnose_session(events, list(range(len(events))), 10)["timing_complete"] is False + monkeypatch.setattr(diagnostics, "MAX_CALLS", 1) + report = diagnostics.diagnose_session([use(), use(tool_id="two")], [1, 2], 5) + assert report["truncated"] is True and len(report["calls"]) == 1 + assert report["tool_wait_s"] is None + + +@pytest.mark.parametrize(("command", "tool"), [ + ("gitnexus impact parseFile --direction upstream", "cli__gitnexus__impact"), + ("/usr/local/bin/gitnexus query 'some concept'", "cli__gitnexus__query"), + ("gitnexus query 'x;y'", "cli__gitnexus__query"), + ('gitnexus query "x;y"', "cli__gitnexus__query"), + (r"gitnexus query x\;y", "cli__gitnexus__query"), + ("gitnexus query '$(literal) | & > < `literal`'", "cli__gitnexus__query"), + ('gitnexus query "$(cat secret)"', "Bash"), + ('gitnexus query "`cat secret`"', "Bash"), + ("gitnexus query x > result.txt", "Bash"), + ("gitnexus query x < input.txt", "Bash"), + ("gitnexus query x; touch changed", "Bash"), + ("gitnexus query x | cat", "Bash"), + ("gitnexus query x\nnpm test", "Bash"), + ("gitnexus impact x && npm test", "Bash"), + ("echo 'gitnexus query x'", "Bash"), + ("gitnexus query $(cat secret)", "Bash"), + ("gitnexus query 'unterminated", "Bash"), +]) +def test_cli_attribution_only_covers_standalone_shim_commands(command, tool): + report = diagnostics.diagnose_session([use("Bash", args={"command": command}), result("ok")], [1, 2], 3) + assert report["calls"][0]["tool"] == tool + assert command not in json.dumps(report) + + +@pytest.mark.parametrize("mask", range(1, 8)) +@pytest.mark.parametrize("has_hit", [False, True]) +def test_empty_query_signals_cover_every_selected_response_section(mask, has_hit): + sections = [name for bit, name in enumerate(("processes", "process_symbols", "definitions")) if mask & (1 << bit)] + body = dict.fromkeys(sections, []) + if has_hit: + body[sections[0]] = [{"name": "found"}] + events = [use(), result(body), use("Read", tool_id="read", args={"file_path": "src/a.ts"}), + result("source", tool_id="read")] + report = diagnostics.diagnose_session(events, None, 5) + tool = report["tools"]["mcp__gitnexus__query"] + assert tool["empty"] == int(not has_hit) + assert tool["source_followups"] == int(not has_hit) + + +@pytest.mark.parametrize("body", [{}, {"definitions": None}, {"timing": {"wall": 1}}]) +def test_absent_or_invalid_query_sections_are_not_evidence_of_an_empty_result(body): + report = diagnostics.diagnose_session([use(), result(body)], None, 5) + assert report["tools"]["mcp__gitnexus__query"]["empty"] == 0 + + +def test_usage_snapshots_are_deduplicated_and_no_cost_is_fabricated(): + event = use() + event["message"].update(id="message", usage={"input_tokens": 10, "output_tokens": 2, "secret": "hidden"}) + later = {"type": "assistant", "message": {"id": "message", "content": [], "usage": {"output_tokens": 5}}} + report = diagnostics.diagnose_session([event, result("ok"), later], [1, 2, 3], 4) + assert report["assistant_usage"] == [{"event_index": 2, "output_tokens": 5}] + assert "cost_usd" not in json.dumps(report) + assert "hidden" not in json.dumps(report) + + +def test_public_diagnostics_is_idempotent_and_cannot_publish_arbitrary_content(): + raw = diagnostics.diagnose_session([use(), result({"status": "found", "content": "secret source"})], [1, 2], 3) + raw.update(prompt="secret prompt", api_key="secret key") + raw["tools"]["private-tool-name"] = {"calls": 5} + raw["calls"][0]["password"] = "secret password" + safe = diagnostics.public_diagnostics([raw]) + assert diagnostics.public_diagnostics(safe) == safe + assert safe[0]["largest_results"][0]["index"] == 0 + assert safe[0]["slowest_calls"][0]["duration_s"] == 1 + assert "secret" not in json.dumps(safe) + assert "private-tool-name" not in json.dumps(safe) + assert "calls" not in safe[0] + + +def test_missing_status_and_stale_or_truncated_results_are_visible_in_public_diagnostics(): + events = [use(), result({"status": "not_found", "risk": "UNKNOWN", "truncated": True, + "staleness": {"status": "behind"}}), use("Read", tool_id="read")] + report = diagnostics.diagnose_session(events, None, 3) + tally = report["tools"]["mcp__gitnexus__query"] + assert tally["stale"] == tally["truncated_results"] == tally["source_followups"] == 1 + safe = diagnostics.public_diagnostics([report]) + assert safe[0]["problem_calls"][0]["not_found"] is True + assert diagnostics.public_diagnostics(safe) == safe + + +def test_release_report_exports_diagnostics_without_changing_gate_or_accepting_injected_fields(): + measurements = rows() + raw = diagnostics.diagnose_session([use(), result({"error": "private symbol not found"})], [1, 2], 3) + measurements[1]["session_diagnostics"] = [raw] + report = build_report(measurements, metadata(), now=NOW) + assert report["complete"] is True + assert report["paired"]["mean_cost_change_pct"] == 25 + validate_report(report, runtime_sha=SHA, task_set_digest=DIGEST, now=NOW) + text = render_markdown(report) + assert "Diagnostics available for 1/3 cells" in text + assert "not proof of a workaround" in text + assert "private symbol" not in json.dumps(report) + report["per_run"][1]["session_diagnostics"][0]["secret"] = "forged" + with pytest.raises(ValueError, match="outside the published report schema"): + validate_report(report, runtime_sha=SHA, task_set_digest=DIGEST, now=NOW) + + +def test_runner_collects_timings_without_progress_or_changing_session_cost(monkeypatch, tmp_path): + events = [use(), result("ok"), json.loads(VALID_REPORT)] + raw = b"".join((json.dumps(event) + "\n").encode() for event in events) + + def run(*args, **kwargs): + for line in raw.splitlines(keepends=True): + kwargs["stdout_observer"](line) + return fake_cli_result(raw.decode()) + + monkeypatch.setattr(runner_sessions, "run_managed", run) + record = runner_sessions.run_claude("task", tmp_path, claude_bin="claude", timeout=5) + assert record["ok"] is True and record["cost_usd"] == 0.1 + assert record["session_diagnostics"][0]["timing_complete"] is True + assert len(record["session_diagnostics"][0]["calls"]) == 1 + combined = runner_sessions.sum_sessions([record, record]) + assert len(combined["session_diagnostics"]) == 2 + assert combined["cost_usd"] == 0.2 + assert runner_sessions.sum_sessions([session_record()])["session_diagnostics"] == [] + + +def test_diagnostics_tool_whitelist_matches_the_harness_surface(): + assert {f"mcp__gitnexus__{name}" for name in diagnostics.COMMANDS} == set(runner_sessions.GITNEXUS_READ_ONLY_TOOLS) + + +def test_observational_followups_can_count_one_action_for_multiple_problem_results(): + first = use() + second_block = use(tool_id="two")["message"]["content"][0] + first["message"]["content"].append(second_block) + events = [first, result({"risk": "UNKNOWN"}), result({"risk": "UNKNOWN"}, tool_id="two"), use("Read", tool_id="read")] + report = diagnostics.diagnose_session(events, None, 5) + assert report["tools"]["mcp__gitnexus__query"]["source_followups"] == 2 + assert Counter(call["tool"] for call in report["calls"])["Read"] == 1 diff --git a/eval/workflow_bench/README.md b/eval/workflow_bench/README.md index 7b64b6729..3ca916555 100644 --- a/eval/workflow_bench/README.md +++ b/eval/workflow_bench/README.md @@ -930,6 +930,207 @@ measurements unchanged and compare only runs with matching task contracts. ## MCP response cost +### Session diagnostics + +Every new headless session records `session_diagnostics` in `results.jsonl`. +The parent timestamps complete stdout event lines without changing the agent +prompt, tool responses, sandbox, or release criteria. No additional model calls +are made. Existing results without diagnostics remain valid and have unknown +timing, rather than measured zero time. + +The release evaluation's GitHub Actions summary includes a **Detailed +measurements** section for each measured runtime or optional arm. It shows cost +and wall-time distributions, setup/cleanup phases, CLI token counters, tool +outcomes and outliers, context/cache signals, repeated verification, and provider +request sizes and usage. Matching `*-diagnostics.json` and `*-diagnostics.md` +files are included in the sanitized evaluation artifacts. PR comparison +summaries retain the candidate's safe diagnostics too. + +Each sweep writes its own provider ledger. Only aggregate numeric provider +measurements reach the summary and artifacts; raw request identities and usage +ledgers remain on the runner. Missing or torn inputs are marked as partial, +and provider token totals are withheld when ledger records are unreadable. +Concurrent requests are not assigned to individual cells. Historical runs +without this instrumentation show missing telemetry; regenerating a report +cannot recover tool timings that were never recorded. + +Each session contains: + +- `calls`: tool name, event indexes, input/result JSON bytes, receipt offsets, + elapsed time, repeated-argument references, result status, staleness, + truncation, and the next assistant action's tool names. MCP errors, missing + symbols, ambiguity, empty query results, UNKNOWN risk, and output-limit + failures are counted separately. Standalone `gitnexus` shim commands are + identified; compound shell commands remain `Bash` because their elapsed time + includes other work. +- `assistant_usage`: the last available usage snapshot for each assistant + message, keyed by event index. Missing fields stay missing. These are + main-session CLI counters, not a complete provider or subagent ledger. +- `workload`: repeated reads, reads of files referenced by graph results, + query/context/read sequences, graph calls before the first observed edit, + verification commands and repeats, edits after verification, compactions, + and input-context growth. Shell commands are classified conservatively; + compound commands remain mixed. Unknown shell actions invalidate the + assumption that the checkout stayed unchanged between verification calls. + These are action counts, including failed edit attempts, not proven waste. +- `tools`: per-tool counts, elapsed sums, response bytes, and source-read + follow-ups after problematic results. One source action can follow multiple + problem results, so follow-ups are associations, not unique extra reads. +- `tool_wait_s`: the union of observed call intervals; parallel tools count + once. `mcp_wait_s` is a subset. `other_wall_s` includes model requests, + startup, transport, retries and scheduling; it is not a measurement of model + reasoning time. Missing results, unmatched IDs, clock gaps or diagnostics + limits make complete wait totals unavailable. +- `server_wall_s`: the sum of tool-reported `timing.wall` milliseconds, converted + to seconds, with a separate coverage count. Missing server timing stays + unknown. This does not equal the host's tool wait or model request time. + +The normal release JSON and Markdown include numeric per-tool summaries. The +JSON also keeps each session's five largest results, five slowest calls, and +five problematic calls with the largest results, +with event indexes for locating the detail in local `results.jsonl`. Raw tool +arguments, result text, source, prompts, IDs and argument hashes are excluded +from these diagnostics. Call and assistant-usage details are bounded at 2,048 +entries per session; `truncated` exposes that limit. Receipt timing is bounded +at 100,000 event lines. Diagnostics do not make incomplete task evidence valid. + +Use the counters to investigate explanations separately: large successful +responses suggest payload overhead; missing/ambiguous targets followed by +source reads suggest a possible workaround; repeated calls suggest a possible +navigation loop. These are signals, not proof of poor resolution or causation. +An empty result is not proof that a symbol is absent. Repeated arguments can be +justified after edits, and smaller output can require more follow-up reads. +Graph precision/recall requires expected-symbol labels or source verification. +The reported session cost is the money measurement; diagnostics never invent +per-tool dollar costs from bytes or elapsed time. + +### Cost investigations and controlled experiments + +No paid rerun is needed to prepare experiments or summarize existing evidence. +The commands below only read local files and write diagnostic artifacts. +Run them from `eval/` with the project's locked environment. + +For the first trial, prepare two cells for one task, with a $5 CLI-reported +budget stop per cell. This checks telemetry delivery before a larger comparison: + +```bash +uv run --locked python -m workflow_bench.diagnostic_experiments \ + --tasks workflow_bench/tasks.scenarios.yaml \ + --repository /path/to/GitNexus --runtime /path/to/built/GitNexus \ + --harness-sha FULL_HARNESS_SHA --runtime-sha FULL_RUNTIME_SHA \ + --model gpt-5.6-sol --effort xhigh --seed 3762 --repeats 1 \ + --variants no_graph mcp_eager --task-ids trivial-status-json-alias \ + --max-budget-usd 5 --out /tmp/gitnexus-diagnostics-trial +``` + +The runner forwards `--max-budget-usd` to Claude Code and accepts it only for +single-session baseline arms. The default remains uncapped. Budget-stopped +sessions stay failed and retain reported spend; they cannot establish savings +or successful task completion. The $10 planned total uses CLI accounting, not +a provider invoice cap: an in-flight request or different provider accounting +can exceed it. Preparing the plan does not authorize or execute the trial. + +| Possible cause | Measurement | What still needs an experiment | +| --- | --- | --- | +| Missing, ambiguous or incomplete resolution | Tool outcomes, UNKNOWN risk, lower-bound/stale/truncated responses, next source actions | Verify the intended target exists at the indexed source revision | +| Successful but irrelevant retrieval | Fixed source anchors in `retrieval_cases.json`, anchor recall and first anchor position | Capture the same queries against the sanitized task graph; unjudged results are not false positives | +| Too little detail or repeated investigation | Query/context/read chains, repeated arguments/reads, reads after graph references with and without source | A follow-up read may be necessary; compare matched tasks before calling it waste | +| Large responses and context growth | Result bytes, complete input usage snapshots, first/last/peak context, cache reads/writes, compactions | Compare response-size treatments at equal quality; bytes are not tokens | +| Mandatory graph workflow | Graph calls before edits; eager MCP versus optional-use prompt arm | Prompt ablation measures behavior; it does not remove repository instruction bytes | +| Tool catalog and transport | Eager/deferred/CLI arms; proxy tools/messages bytes and counts | Request sizes are available only when the proxy callback receives those fields | +| Verification and scope growth | Test/typecheck/lint/build counts, repeated checks, edits after verification, diff insertions/deletions/files | Larger diffs and repeated tests are not inherently unnecessary | +| Task fit and quality ceiling | Per-task cost distribution, solved/failed/missing counts, cost per solved task | Add harder independently validated tasks before claiming a quality improvement at 100% success | +| Order, concurrency, warm caches | Seeded rotated order, one worker and separate ledger per cell | Run matched worker-count/cold-start studies separately; fresh processes do not guarantee cold provider/filesystem caches | +| Retries, provider translation and accounting | CLI retry events; deduplicated success/failure callbacks, native normalized usage, reasoning tokens, proxy durations and estimates | Reconcile with provider billing; do not join concurrent requests to cells by timestamp | +| Setup, indexing and infrastructure | Cell start/finish timestamps, total duration, clone/index-materialization/dependency/MCP-startup/sandbox/agent-and-verification/artifact/cleanup spans | Shared index-build preparation, CPU/RSS, isolated oracle spans and infrastructure prices need separate evidence; do not fold them into model dollars | + +Summarize a runner result file (including failed cells) and optionally its proxy +ledger. Historical release reports with `per_run` rows are also accepted: + +```bash +uv run --locked python -m workflow_bench.cost_diagnostics \ + --results /path/to/results.jsonl \ + --provider-usage /path/to/provider_usage.jsonl \ + --out /tmp/cost-diagnostics.json --markdown-out /tmp/cost-diagnostics.md +``` + +Unknown fields remain null. Distinct model/effort/source/prompt/runtime/loading +cohorts are kept separate. Duplicate exported cell identities are rejected. +Provider response-ID conflicts withhold token totals; missing IDs prevent a +deduplicated total. Failed requests remain in the accounting. Proxy cost is an +estimate, not a provider invoice. The report is descriptive and does not change +release gates. It cannot infer cells omitted from its input: compare every +expected cell in the experiment plan before computing a treatment effect. +`cell_duration_s` includes setup and cleanup; `duration_s` retains its existing +session meaning. Phase spans partition the cell, including a failed active +phase, and unvisited phases stay missing. Shared snapshot construction happens +before cells and is not included. Agent and harness verification share one +phase; it must not be presented as isolated LLM reasoning time. Raw rows retain +UTC start/finish timestamps for order analysis; public release JSON retains +only numeric durations and whitelisted phase names. + +Prepare a pilot focused on the overhead floor and the expensive retry task: + +```bash +uv run --locked python -m workflow_bench.diagnostic_experiments \ + --tasks workflow_bench/tasks.scenarios.yaml \ + --repository /path/to/GitNexus --runtime /path/to/built/GitNexus \ + --harness-sha FULL_HARNESS_SHA --runtime-sha FULL_RUNTIME_SHA \ + --model gpt-5.6-sol --effort xhigh --seed 3762 --repeats 5 \ + --task-ids trivial-status-json-alias cross-module-parse-retry \ + --out /tmp/gitnexus-cost-experiment +``` + +This creates `plan.json` and one task manifest per cell. It has **no execution +mode** and never starts an agent. Five arms × two tasks × five repeats prepares +50 cells; this is a design, not spend authorization. Use `--variants` to prepare +a narrower comparison. Position balance requires a repeat count divisible by +the number of arms. The plan records argv arrays, environment paths, task class, +block/position, source pins and prompt/manifest digests. It refuses to overwrite +an existing directory. Pin the final instrumentation commit before execution; +an uncommitted implementation is not reproduced by its parent SHA alone. + +After separately authorizing spend, run cells serially in the listed sequence, +from the recorded `cwd`, with the recorded environment and a fresh exclusive +proxy ledger. Each subprocess runs one task/arm/repetition. Its raw `run` is +zero; retain the plan's cell ID and block when assembling comparisons. Do not +mix the optional-prompt arm with ordinary release evidence or reuse its results +for a required-guidance arm. Retain failures and missing cells in the denominator. +Record machine/container, dependency bindings, cache policy, start/finish times +and setup/oracle timing alongside the existing runner output. Reconcile the +ledger sum with CLI-reported session spend, rather than estimating dollars per +tool from duration or payload size. + +The four source-verified retrieval cases cover the same pinned scenario source. +Capture each fixed query from the **sanitized** task index used by the runner, +with the same query limits and content setting. Save parsed response objects as: + +```json +{ + "source_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "runtime_sha": "FULL_RUNTIME_SHA", + "sanitized_graph": true, + "index_complete": true, + "index_staleness": "current", + "query_options": {"limit": 5, "max_symbols": 10, "include_content": false}, + "cases": [{"id": "CASE_ID", "query": "EXACT_FIXTURE_QUERY", "response": {}}] +} +``` + +Verify those provenance fields against the index receipt; the offline scorer +checks declarations, not the database itself. Keep capture metadata and raw +responses private, since they contain repository source. Score without an LLM: + +```bash +uv run --locked python -m workflow_bench.retrieval_diagnostics \ + --recordings /path/to/retrieval-recordings.json --out /tmp/retrieval-scores.json +``` + +Missing cases remain missing, errors are not successful empty queries, and stale +or mismatched source recordings are rejected. These are partial anchor labels; +there is no precision score or claim that every other result is irrelevant. +Anchor position follows flattened response order, not a global search rank. + With `include_content: true`, queries share a 16,000-character preview allowance equally across unique symbols on the returned page. The database returns only those prefixes, after ranking and pagination. `content_truncated: true` marks diff --git a/eval/workflow_bench/cli_arm_report.py b/eval/workflow_bench/cli_arm_report.py index dd33ea6a8..6a5850c52 100644 --- a/eval/workflow_bench/cli_arm_report.py +++ b/eval/workflow_bench/cli_arm_report.py @@ -15,6 +15,8 @@ from collections import Counter from pathlib import Path from typing import Any +from .session_diagnostics import public_cell_diagnostics, render_diagnostics + ARMS = ("baseline_nomcp", "baseline", "baseline_cli") EXPOSURE = {"baseline_nomcp": "none", "baseline": "stdio-only", "baseline_cli": "stdio-only"} METRICS = ("cost_usd", "duration_s", "num_turns", "cache_read_input_tokens", "output_tokens") @@ -141,6 +143,10 @@ def build_report(paired: Path, cli: Path) -> dict[str, Any]: "pairs": len(matched), "expected_pairs": len(pairs), } + report["diagnostic_rows"] = [ + {**{field: row.get(field) for field in ("task", "run", "arm")}, **public_cell_diagnostics(row)} + for row, _ in rows + ] return report @@ -180,6 +186,7 @@ def render_markdown(report: dict[str, Any]) -> str: if report["problems"]: lines += ["", "**Problems:**", *[f"- {problem}" for problem in report["problems"]]] lines += ["", "Small pinned task set; descriptive evidence only, not statistical significance.", ""] + lines += render_diagnostics(report.get("diagnostic_rows", [])) return "\n".join(lines) diff --git a/eval/workflow_bench/cost_diagnostics.py b/eval/workflow_bench/cost_diagnostics.py new file mode 100644 index 000000000..546685fb3 --- /dev/null +++ b/eval/workflow_bench/cost_diagnostics.py @@ -0,0 +1,283 @@ +"""Offline cost evidence report; no API calls, pricing guesses, or release gates.""" + +from __future__ import annotations + +import argparse +import html +import json +import math +import statistics +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from .provider_usage import NormalizedUsage, UsageSemanticsError, normalize_usage +from .session_diagnostics import CELL_PHASES, public_diagnostics, render_diagnostics + +NUMERIC = ("cost_usd", "duration_s", "cell_duration_s", "num_turns", "diff_insertions", "diff_deletions", "diff_files", + "input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "output_tokens") +COHORT = ("model", "effort", "task_base_sha", "task_prompt_digest", "runtime_digest", "mcp_tool_loading", + "sandbox_dependency_content_digest", "oracle_digest", "max_budget_usd") +REQUEST_METRICS = ("messages_bytes", "tools_bytes", "system_bytes", "messages_count", "tools_count", + "duration_s", "proxy_estimated_cost_usd") + + +def _number(value: Any) -> bool: + if type(value) not in (int, float) or value < 0: + return False + try: + return math.isfinite(value) + except OverflowError: + return False + + +def distribution(values: list[Any]) -> dict[str, Any]: + measured = [v for v in values if _number(v)] + return { + "observed": len(measured), "missing": len(values) - len(measured), + "sum_observed": sum(measured) if measured else None, + "mean": statistics.mean(measured) if measured else None, + "median": statistics.median(measured) if measured else None, + "min": min(measured) if measured else None, "max": max(measured) if measured else None, + "stdev": statistics.stdev(measured) if len(measured) > 1 else None, + } + + +def provider_summary(events: list[dict[str, Any]]) -> dict[str, Any]: + """Deduplicate identical callbacks, withhold totals on identity conflicts. + + Never join to cells by wall-clock proximity. Concurrent sweeps require a + session ID mapping; serial experiments can use one exclusive ledger/cell. + """ + seen: dict[tuple, dict] = {} + unique, duplicates, conflicts, unidentified = [], 0, set(), 0 + for event in events: + response_id = event.get("response_id") + if not isinstance(response_id, str) or not response_id: + unique.append(event) + unidentified += 1 + continue + key = (event.get("sweep_id"), event.get("provider"), response_id) + if key in seen: + if seen[key] == event: + duplicates += 1 + else: + conflicts.add(key) + continue + seen[key] = event + unique.append(event) + normalized = [] + invalid = 0 + for event in unique: + try: + normalized.append(asdict(normalize_usage(event.get("provider"), event.get("native_usage")))) + except (UsageSemanticsError, TypeError, ValueError): + invalid += 1 + normalized.append({}) + token_fields = NormalizedUsage.__dataclass_fields__ + token_totals = { + field: sum(row[field] for row in normalized) + if normalized and not conflicts and not unidentified and all(_number(row.get(field)) for row in normalized) else None + for field in token_fields + } + metrics = [e.get("request_metrics") if not conflicts and isinstance(e.get("request_metrics"), dict) else {} for e in unique] + return { + "records": len(events), "observed_requests": len(unique), "duplicate_callbacks": duplicates, + "conflicting_response_ids": len(conflicts), "requests_without_response_id": unidentified, + "invalid_usage_records": invalid, + "successes": sum(e.get("status") == "success" for e in unique), + "failures": sum(e.get("status") == "failure" for e in unique), + "requests_with_session_id": sum(isinstance(e.get("session_id"), str) and bool(e["session_id"]) for e in unique), + "unique_actual_models": len({e["actual_model"] for e in unique if isinstance(e.get("actual_model"), str)}), + "token_totals": token_totals, + "request_metrics": {field: distribution([m.get(field) for m in metrics]) for field in REQUEST_METRICS}, + "identity_complete": bool(unique) and not conflicts and unidentified == 0, + "attribution": "ledger_only_no_cell_join", + "limitations": [ + "No response ID means callbacks cannot be deduplicated; observed totals may overcount.", + "Proxy cost is an estimate, not an invoice. Failures without usage may still incur cost.", + "Callback duration includes proxy/provider work; it is not isolated model thinking time.", + "Request payload bytes are serialized sizes, not token counts.", + ], + } + + +def build_cost_report(rows: list[dict[str, Any]], provider_events: list[dict[str, Any]] | None = None) -> dict[str, Any]: + groups: dict[tuple, list[dict]] = {} + seen_cells = set() + for row in rows: + arm, task = row.get("arm"), row.get("task") + if not isinstance(arm, str) or not isinstance(task, str): + raise ValueError("each cell needs arm and task labels") + identity_fields = {field: row.get(field) if isinstance(row.get(field), str) else "" for field in COHORT} + if arm == "baseline_cli" and "mcp_tool_loading" in row and row["mcp_tool_loading"] is None: + identity_fields["mcp_tool_loading"] = "not_applicable" + if "max_budget_usd" in row: + budget = row["max_budget_usd"] + identity_fields["max_budget_usd"] = "uncapped" if budget is None else str(float(budget)) if _number(budget) and budget > 0 else "" + cohort = tuple(identity_fields[field] for field in COHORT) + identity = (arm, task, row.get("run"), cohort) + if type(row.get("run")) is int: + if identity in seen_cells: + raise ValueError("duplicate cell identity; do not concatenate repeated exports") + seen_cells.add(identity) + groups.setdefault((arm, task, cohort), []).append(row) + summaries = [] + tool_sections = [] + for (arm, task, cohort), cells in sorted(groups.items()): + costs = [row.get("cost_usd") for row in cells] + solved = sum(row.get("resolved") is True for row in cells) + summaries.append({ + "diagnostic_group": len(summaries) + 1, + "arm": arm, "task": task, "cells": len(cells), "solved": solved, + "cohort": dict(zip(COHORT, cohort, strict=True)), + "provenance_complete": all(cohort), + "quality_missing": sum(type(row.get("resolved")) is not bool for row in cells), + "failed_cells": sum(row.get("ok") is False for row in cells), + "diagnostic_cells": sum(bool(public_diagnostics(row.get("session_diagnostics"))) for row in cells), + "metrics": {field: distribution([row.get(field) for row in cells]) for field in NUMERIC}, + "cell_phases": {phase: distribution([ + row["cell_phase_seconds"].get(phase) if isinstance(row.get("cell_phase_seconds"), dict) else None + for row in cells + ]) for phase in CELL_PHASES}, + "cost_per_solved": sum(costs) / solved if solved and all(_number(c) for c in costs) else None, + }) + details = render_diagnostics(cells) + if details: + tool_sections += [f"### Diagnostic group {len(summaries)}", *details] + return { + "schema": "gitnexus.cost-diagnostics/v1", "cells": len(rows), "groups": summaries, + "provider": provider_summary(provider_events) if provider_events is not None else None, + "tool_markdown": "\n".join(tool_sections), + "limitations": [ + "Descriptive observations only; no causal attribution, significance claim or release verdict.", + "Groups include failed cells and cost per solved includes their reported spend.", + "Missing cells require comparison to the experiment manifest; this report cannot infer omitted rows.", + "Legacy rows without diagnostics cannot reveal retrieval relevance, instruction overhead or tool timings.", + "Main-loop session telemetry excludes subagent internals; session cost may include subagents.", + "Task labels, model/effort, runtime/source pins and prompt digests must match the intended treatment.", + ], + } + + +def render_markdown(report: dict[str, Any]) -> str: + """Render measured distributions and coverage, without guessing missing spend.""" + def number(value: Any) -> str: + return f"{value:,.4f}".rstrip("0").rstrip(".") if _number(value) else "n/a" + + def label(value: Any) -> str: + return html.escape(str(value)).replace("\n", " ").translate( + {ord(char): f"&#{ord(char)};" for char in "\\`*_[]()|!"} + ) + + def table(metrics: dict[str, dict]) -> list[str]: + lines = ["| Measurement | Observed / missing | Observed total | Mean | Median | Min | Max | Std. dev. |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"] + for name, values in metrics.items(): + lines.append(f"| {name} | {values['observed']} / {values['missing']} | " + " | ".join( + number(values[field]) for field in ("sum_observed", "mean", "median", "min", "max", "stdev") + ) + " |") + return lines + + lines = ["## Time, cost, and tool measurements", "", + f"Observed cells: {report['cells']}. Missing measurements are shown as `n/a`, not zero.", + "Cell token counters come from the CLI. Provider tokens below use provider-specific accounting; do not add the two together.", ""] + if report.get("input_problems"): + lines += ["**Partial input: totals cover only the readable records.**", "", + *[f"- {problem}" for problem in report["input_problems"]], ""] + for group in report["groups"]: + lines += [f"### Group {group['diagnostic_group']}: {label(group['arm'])} / {label(group['task'])}", "", + (f"Solved: {group['solved']}/{group['cells']}; failed sessions: {group['failed_cells']}; " + f"quality unknown: {group['quality_missing']}; diagnostics: {group['diagnostic_cells']}/{group['cells']}."), + f"Cost per solved cell (USD, including failed-session spend): {number(group['cost_per_solved'])}.", + f"Complete cohort provenance: {'yes' if group['provenance_complete'] else 'no'}.", "", + "
Measurements and cohort", "", + "| Cohort field | Value |", "| --- | --- |", + *[f"| {field} | {label(value) if value else 'unknown'} |" + for field, value in group["cohort"].items()], "", + *table(group["metrics"]), "", "Cell phases (seconds; agent_and_verification includes grading):", "", + *table(group["cell_phases"]), "", "
", ""] + provider = report["provider"] + lines += ["### Provider requests", ""] + if provider is None: + lines += ["Provider ledger unavailable. Request counts, provider tokens, and proxy cost are unknown.", ""] + else: + lines += [(f"Observed requests: {provider['observed_requests']}; successes: {provider['successes']}; " + f"failures: {provider['failures']}; duplicate callbacks removed: {provider['duplicate_callbacks']}."), + (f"Conflicting response identities: {provider['conflicting_response_ids']}; " + f"requests without identity: {provider['requests_without_response_id']}; " + f"invalid usage records: {provider['invalid_usage_records']}."), + "Attribution is to this sweep's ledger. Requests are not assigned to individual concurrent cells.", "", + "| Provider token field | Observed total |", "| --- | ---: |", + *[f"| {field} | {number(value)} |" for field, value in provider["token_totals"].items()], "", + *table(provider["request_metrics"]), "", + *provider["limitations"], ""] + lines += [report["tool_markdown"], "", *report["limitations"], ""] + return "\n".join(lines) + + +def _read_rows(path: Path, *, allow_incomplete: bool, name: str, problems: list[str]) -> list[dict]: + if not path.is_file() and allow_incomplete: + problems.append(f"{name} file is missing.") + return [] + raw = path.read_text() + try: + doc = json.loads(raw) + except json.JSONDecodeError: + rows = [] + for line in raw.splitlines(): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + if not allow_incomplete: + raise + rows.append(None) + else: + rows = doc.get("per_run", [doc]) if isinstance(doc, dict) else doc + if not isinstance(rows, list): + raise ValueError(f"expected {name} rows") + invalid = sum(not isinstance(row, dict) for row in rows) + if invalid: + if not allow_incomplete: + raise ValueError(f"expected {name} rows") + problems.append(f"{name}: {invalid} unreadable record(s).") + return [row for row in rows if isinstance(row, dict)] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results", type=Path, required=True, help="runner results.jsonl or release report JSON") + parser.add_argument("--provider-usage", type=Path) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--markdown-out", type=Path) + parser.add_argument("--allow-incomplete", action="store_true", help="report missing/torn inputs after an interrupted run") + args = parser.parse_args() + problems: list[str] = [] + rows = _read_rows(args.results, allow_incomplete=args.allow_incomplete, name="Results", problems=problems) + provider = None + provider_problems: list[str] = [] + if args.provider_usage: + provider = _read_rows(args.provider_usage, allow_incomplete=args.allow_incomplete, + name="Provider ledger", problems=provider_problems) + if provider_problems and not provider: + provider = None + problems += provider_problems + report = build_cost_report(rows, provider) + if report["provider"] is not None: + report["provider"]["ledger_complete"] = not provider_problems + if provider_problems: + report["provider"]["identity_complete"] = False + report["provider"]["token_totals"] = dict.fromkeys(report["provider"]["token_totals"]) + report["input_problems"] = problems + with args.out.open("x") as handle: + json.dump(report, handle, indent=2, allow_nan=False) + handle.write("\n") + if args.markdown_out: + with args.markdown_out.open("x") as handle: + handle.write(render_markdown(report)) + + +if __name__ == "__main__": + main() diff --git a/eval/workflow_bench/diagnostic_experiments.py b/eval/workflow_bench/diagnostic_experiments.py new file mode 100644 index 000000000..80fdd6f75 --- /dev/null +++ b/eval/workflow_bench/diagnostic_experiments.py @@ -0,0 +1,162 @@ +"""Prepare counterbalanced experiment inputs. This module NEVER runs an agent.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import random +import re +from pathlib import Path +from typing import Any + +import yaml + +from .runner_tasks import normalized_model_identifier, select_tasks +from .session_budget import session_budget + +VARIANTS = { + "no_graph": ("baseline_nomcp", "eager"), + "mcp_eager": ("baseline", "eager"), + "mcp_deferred": ("baseline", "deferred"), + "cli": ("baseline_cli", "eager"), + "mcp_optional_prompt": ("baseline", "eager"), +} +OPTIONAL_PROMPT = """ + +Experiment-specific instruction: GitNexus is available, but using it is optional +for this task. This overrides repository instructions that require graph-first +navigation, impact analysis before every edit, or detect_changes before commit. +Use the tools that help solve the task. Keep all other repository requirements, +tests, and safety constraints. Do not install or bootstrap additional tools. +""" + + +def prepare_plan( + document: dict[str, Any], *, repository: str, runtime: str, runtime_sha: str, + harness_sha: str, model: str, effort: str, repeats: int, seed: int, + variants: list[str], task_ids: list[str] | None = None, max_budget_usd: float | None = None, +) -> dict[str, Any]: + """A seeded Latin-square order within each task; no execution side effects. + + One runner process per cell makes its provider ledger attributable without + guessing session IDs. Run indices in runner output are always zero; the + experiment's block/sequence fields carry repetition identity instead. + """ + max_budget_usd = session_budget(max_budget_usd) + if not variants or len(set(variants)) != len(variants) or any(v not in VARIANTS for v in variants): + raise ValueError("variants must be unique known experiment variants") + if not 1 <= repeats <= 100: + raise ValueError("repeats must be between 1 and 100") + if not all(re.fullmatch(r"[0-9a-f]{40}", sha) for sha in (runtime_sha, harness_sha)): + raise ValueError("runtime and harness require full commit SHAs") + if not isinstance(model, str): + raise ValueError("use a named model, not auto/latest") + model = normalized_model_identifier(model) + if effort not in ("low", "medium", "high", "xhigh", "max"): + raise ValueError("invalid effort") + if not isinstance(document, dict): + raise ValueError("task document must contain tasks") + tasks = document.get("tasks") + if not isinstance(tasks, list) or not tasks or not all(isinstance(t, dict) for t in tasks): + raise ValueError("task document must contain tasks") + tasks, _ = select_tasks([{**task, "repo": repository} for task in tasks], include_expensive=True) + ids = [t["id"] for t in tasks] + if task_ids is not None and (not task_ids or not set(task_ids) <= set(ids)): + raise ValueError("unknown or empty task selection") + tasks = [t for t in tasks if task_ids is None or t["id"] in task_ids] + if any(not re.fullmatch(r"[0-9a-f]{40}", str(t.get("ref", ""))) or not isinstance(t.get("prompt"), str) for t in tasks): + raise ValueError("each task needs a pinned ref and prompt") + rng = random.Random(seed) + order = variants.copy() + rng.shuffle(order) + rng.shuffle(tasks) + cells = [] + for block in range(repeats): + for task_index, task in enumerate(tasks): + rotation = (block + task_index) % len(order) + for position, variant in enumerate(order[rotation:] + order[:rotation]): + arm, loading = VARIANTS[variant] + prepared = copy.deepcopy(task) + prepared["repo"] = repository + if variant == "mcp_optional_prompt": + prepared["prompt"] += OPTIONAL_PROMPT + sequence = len(cells) + cell_id = f"cell-{sequence:04d}" + cells.append({ + "cell_id": cell_id, "sequence": sequence, "block": block, + "position": position, "task": task["id"], "task_class": task.get("class"), + "variant": variant, "arm": arm, "tool_loading": loading, + "base_prompt_sha256": hashlib.sha256(task["prompt"].encode()).hexdigest(), + "prompt_sha256": hashlib.sha256(prepared["prompt"].encode()).hexdigest(), + "task_document": {"tasks": [prepared]}, + "argv": ["uv", "run", "--locked", "python", "-m", "workflow_bench.runner", + "--tasks", f"{cell_id}.tasks.json", "--runs", "1", "--workers", "1", + "--arms", arm, "--mcp-tool-loading", loading, + "--gitnexus-root", runtime, "--model", model, "--effort", effort, + "--include-expensive", *(["--max-budget-usd", str(max_budget_usd)] if max_budget_usd is not None else []), + "--out", f"results/{cell_id}"], + "env": {"GITNEXUS_BENCH_SWEEP_ID": cell_id, + "GITNEXUS_BENCH_PROVIDER_USAGE": f"results/{cell_id}.provider.jsonl"}, + }) + return { + "schema": "gitnexus.diagnostic-experiments/v1", "execution": "not_started", + "harness_sha": harness_sha, "runtime_sha": runtime_sha, "seed": seed, + "model": model, "effort": effort, "repeats": repeats, "workers": 1, + "max_budget_usd_per_cell": max_budget_usd, + "planned_cli_budget_usd": len(cells) * max_budget_usd if max_budget_usd is not None else None, + "position_balanced": repeats % len(variants) == 0, + "release_gate_eligible": False, "cells": cells, + "limitations": [ + "No model-spend authorization; this plan does not execute commands.", + "Verify checkout SHAs, instrumentation diff, dependencies and oracle bindings before execution.", + "Run serially in listed sequence, with an exclusive proxy/ledger per cell; never reuse ledgers.", + "Optional-prompt treatment changes tool-use policy, not repository guidance bytes or tool schemas.", + "This plan controls concurrency at one; a separate matched worker-count study is needed for contention.", + "Fresh processes do not guarantee cold filesystem/provider caches; record machine and cache policy.", + "CLI budget stops use reported model cost; in-flight requests and provider billing can exceed them.", + ], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tasks", type=Path, required=True) + parser.add_argument("--repository", type=Path, required=True) + parser.add_argument("--runtime", type=Path, required=True) + parser.add_argument("--runtime-sha", required=True) + parser.add_argument("--harness-sha", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--effort", default="xhigh") + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--seed", type=int, default=3762) + parser.add_argument("--max-budget-usd", type=float, help="CLI-reported budget stop per cell, not an invoice cap") + parser.add_argument("--variants", nargs="+", choices=VARIANTS, default=list(VARIANTS)) + parser.add_argument("--task-ids", nargs="+") + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + plan = prepare_plan( + yaml.safe_load(args.tasks.read_text()), repository=str(args.repository.resolve()), + runtime=str(args.runtime.resolve()), runtime_sha=args.runtime_sha, harness_sha=args.harness_sha, + model=args.model, effort=args.effort, repeats=args.repeats, seed=args.seed, + variants=args.variants, task_ids=args.task_ids, max_budget_usd=args.max_budget_usd, + ) + # Refuse overwrite: a previous plan/ledger may already be evidence. + args.out.mkdir(parents=True, exist_ok=False) + root = args.out.resolve() + (root / "results").mkdir() + for cell in plan["cells"]: + manifest = root / f"{cell['cell_id']}.tasks.json" + manifest.write_text(json.dumps(cell.pop("task_document"), indent=2) + "\n") + cell["task_manifest_sha256"] = hashlib.sha256(manifest.read_bytes()).hexdigest() + cell["argv"][cell["argv"].index("--tasks") + 1] = str(manifest) + cell["argv"][-1] = str(root / "results" / cell["cell_id"]) + cell["env"]["GITNEXUS_BENCH_PROVIDER_USAGE"] = str(root / "results" / f"{cell['cell_id']}.provider.jsonl") + cell["cwd"] = str(Path(__file__).resolve().parents[1]) + (root / "plan.json").write_text(json.dumps(plan, indent=2) + "\n") + print(f"Prepared {len(plan['cells'])} cells at {root}; no agents started.") + + +if __name__ == "__main__": + main() diff --git a/eval/workflow_bench/litellm_usage_callback.py b/eval/workflow_bench/litellm_usage_callback.py index 8277d0102..3e3585c81 100644 --- a/eval/workflow_bench/litellm_usage_callback.py +++ b/eval/workflow_bench/litellm_usage_callback.py @@ -19,6 +19,7 @@ accounting because the log write failed would be the worse outcome. from __future__ import annotations import json +import math import os import threading from typing import Any @@ -67,6 +68,36 @@ def _plain(value: Any) -> Any: return None +def _request_metrics(kwargs: dict, start_time: Any, end_time: Any) -> dict: + """Numeric envelope evidence only; never log request content or headers.""" + optional = kwargs.get("optional_params") + optional = optional if isinstance(optional, dict) else {} + metrics = {} + for field in ("messages", "tools", "system"): + value = kwargs.get(field, optional.get(field)) + metrics[f"{field}_bytes"] = None + if isinstance(value, (str, dict, list)): + try: + metrics[f"{field}_bytes"] = len(json.dumps( + value, ensure_ascii=False, separators=(",", ":"), + ).encode(errors="backslashreplace")) + except (TypeError, ValueError, RecursionError): + # Keep size unknown when serialization fails; usage accounting + # must still be recorded even without this optional metric. + pass + if field != "system": + metrics[f"{field}_count"] = len(value) if isinstance(value, list) else None + try: + elapsed = end_time - start_time + elapsed = elapsed.total_seconds() if hasattr(elapsed, "total_seconds") else elapsed + except (TypeError, ValueError): + elapsed = None + metrics["duration_s"] = elapsed if type(elapsed) in (int, float) and math.isfinite(elapsed) and elapsed >= 0 else None + cost = kwargs.get("response_cost") + metrics["proxy_estimated_cost_usd"] = cost if type(cost) in (int, float) and math.isfinite(cost) and cost >= 0 else None + return metrics + + class ProviderUsageLogger(CustomLogger): async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: # noqa: ANN001 self._append("success", kwargs, response_obj, start_time, end_time) @@ -123,6 +154,7 @@ class ProviderUsageLogger(CustomLogger): "completed_at": str(end_time), # Verbatim. Not flattened, not renamed, not summed. "native_usage": _plain(getattr(response_obj, "usage", None)), + "request_metrics": _request_metrics(kwargs, start_time, end_time), } line = json.dumps(event, default=str) + "\n" with _LOCK, open(path, "a", encoding="utf-8") as handle: diff --git a/eval/workflow_bench/mcp_loading_report.py b/eval/workflow_bench/mcp_loading_report.py index a66a4d153..029ead208 100644 --- a/eval/workflow_bench/mcp_loading_report.py +++ b/eval/workflow_bench/mcp_loading_report.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Any from .release_report import META_FIELDS, build_report +from .session_diagnostics import render_diagnostics COUNTERS = ("num_turns", "input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "output_tokens") BINDINGS = ("task_prompt_digest", "sandbox_dependency_content_digest") @@ -105,6 +106,10 @@ def render_markdown(report: dict[str, Any]) -> str: "Hooks disabled; prompts, MCP backend and outputs unchanged. Descriptive evidence only; no release approval.", "", ] + for mode in ("none", "eager", "deferred"): + details = render_diagnostics([row for row in report.get("per_run", []) if row.get("mcp_tool_loading") == mode]) + if details: + lines += [f"### Diagnostics: {mode}", *details, ""] if not report["complete"]: return "\n".join([*lines, "**Incomplete or mismatched evidence; no savings estimate.**", "", *[f"- {p}" for p in report["problems"]], ""]) diff --git a/eval/workflow_bench/pr_evaluation.py b/eval/workflow_bench/pr_evaluation.py index b106aa236..9df44c806 100644 --- a/eval/workflow_bench/pr_evaluation.py +++ b/eval/workflow_bench/pr_evaluation.py @@ -18,6 +18,8 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any +from .session_diagnostics import public_cell_diagnostics, render_diagnostics + SHA = re.compile(r"[0-9a-f]{40}") REPO = re.compile(r"[\w.-]+/[\w.-]+") COST_LABEL = "run-cost-evaluation" @@ -349,6 +351,7 @@ def compare_candidate( and _number(row.get("duration_s")) ) safe = {field: row[field] for field in ROW_FIELDS if field in row} + safe.update(public_cell_diagnostics(row)) try: json.dumps(safe, allow_nan=False) except (ValueError, TypeError, OverflowError): @@ -414,7 +417,8 @@ def render_markdown(report: dict[str, Any]) -> str: "", ] if not report["complete"]: - return "\n".join([*lines, "**Incomplete evidence; no savings conclusion.**", "", *report["problems"], ""]) + return "\n".join([*lines, "**Incomplete evidence; no savings conclusion.**", "", *report["problems"], + *render_diagnostics(report.get("per_run", [])), ""]) lines += [ "| MCP sessions | Solved | Total model cost | Mean model cost |", "| --- | --- | --- | --- |", @@ -446,6 +450,7 @@ def render_markdown(report: dict[str, Any]) -> str: "", "Only the PR MCP cells are newly measured; no-MCP and stable sessions are not rerun.", "Historical comparisons are descriptive and do not establish statistical significance or isolate this PR from other runtime changes.", + *render_diagnostics(report.get("per_run", [])), "", ] ) diff --git a/eval/workflow_bench/release_report.py b/eval/workflow_bench/release_report.py index 3ceeb089a..668282727 100644 --- a/eval/workflow_bench/release_report.py +++ b/eval/workflow_bench/release_report.py @@ -17,6 +17,7 @@ import yaml from .oracle_assets import capture_task_oracles from .runner_tasks import normalized_model_identifier, select_tasks +from .session_diagnostics import public_cell_diagnostics, render_diagnostics from .task_assets import DEPENDENCY_CONTENT_BINDING_FIELD SCHEMA = "gitnexus.release-evaluation/v1" @@ -110,7 +111,9 @@ def build_report(rows: list[Any], metadata: dict[str, Any], *, now: datetime | N problems.append(f"row {index}: invalid measurement") continue # Invalid rows still need a public, JSON-serializable failure receipt. - safe_rows.append({key: _json_safe(row[key]) for key in ROW_FIELDS if key in row}) + safe_row = {key: _json_safe(row[key]) for key in ROW_FIELDS if key in row} + safe_row.update(public_cell_diagnostics(row)) + safe_rows.append(safe_row) task, run, arm = row.get("task"), row.get("run"), row.get("arm") if not isinstance(task, str) or type(run) is not int or not isinstance(arm, str): problems.append(f"row {index}: invalid cell identity") @@ -205,6 +208,7 @@ def render_markdown(report: dict[str, Any]) -> str: "This small pinned task set is release evidence; it does not establish general gains or statistical significance.", "", ] + lines += render_diagnostics(report.get("per_run", [])) return "\n".join(lines) diff --git a/eval/workflow_bench/retrieval_cases.json b/eval/workflow_bench/retrieval_cases.json new file mode 100644 index 000000000..256f9b323 --- /dev/null +++ b/eval/workflow_bench/retrieval_cases.json @@ -0,0 +1,37 @@ +{ + "source_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce", + "judgment": "partial source-verified anchors; no precision labels", + "query_options": { "limit": 5, "max_symbols": 10, "include_content": false }, + "cases": [ + { + "id": "trivial-status-json-alias", + "query": "status CLI command JSON output option registration", + "anchors": [{ "file": "gitnexus/src/cli/index.ts" }] + }, + { + "id": "inv-feature-c-include-provenance", + "query": "C include capture quoted and system header provenance", + "anchors": [ + { + "file": "gitnexus/src/core/ingestion/languages/c/import-decomposer.ts", + "name": "splitCInclude" + } + ] + }, + { + "id": "inv-feature-list-repos-filter", + "query": "list_repos MCP repository listings filter by name", + "anchors": [{ "file": "gitnexus/src/mcp/local/local-backend.ts", "name": "listRepos" }] + }, + { + "id": "cross-module-parse-retry", + "query": "parse worker transient failure error exit retry quarantine dispatch", + "anchors": [ + { + "file": "gitnexus/src/core/ingestion/workers/worker-pool.ts", + "name": "handleWorkerDeath" + } + ] + } + ] +} diff --git a/eval/workflow_bench/retrieval_diagnostics.py b/eval/workflow_bench/retrieval_diagnostics.py new file mode 100644 index 000000000..ee13b2c5d --- /dev/null +++ b/eval/workflow_bench/retrieval_diagnostics.py @@ -0,0 +1,92 @@ +"""Score fixed source anchors against recorded retrievals, without an LLM judge. + +These are partial relevance judgments: unjudged results are not false positives. +The score is anchor coverage, not end-to-end task success or search precision. +""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any + + +def score_retrieval(cases: dict[str, Any], recordings: dict[str, Any]) -> dict[str, Any]: + if recordings.get("source_sha") != cases.get("source_sha") or not cases.get("source_sha"): + raise ValueError("retrieval recordings must match the fixture source SHA") + if recordings.get("index_complete") is not True or recordings.get("index_staleness") != "current": + raise ValueError("retrieval recordings require a complete current index receipt") + if recordings.get("sanitized_graph") is not True or not re.fullmatch(r"[0-9a-f]{40}", str(recordings.get("runtime_sha", ""))): + raise ValueError("record the sanitized graph and runtime identity") + if recordings.get("query_options") != cases.get("query_options") or not cases.get("query_options"): + raise ValueError("record the fixed query options from the fixture") + rows = recordings.get("cases", []) + if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows): + raise ValueError("recorded cases must be a list") + by_id = {row.get("id"): row for row in rows} + if len(by_id) != len(rows): + raise ValueError("duplicate retrieval case ID") + expected_ids = {case["id"] for case in cases["cases"]} + if not set(by_id) <= expected_ids: + raise ValueError("unknown retrieval case ID") + scores = [] + for case in cases["cases"]: + row = by_id.get(case["id"]) + if row is None: + scores.append({"id": case["id"], "status": "missing", "anchor_recall": None}) + continue + if row.get("query") != case["query"]: + raise ValueError(f"query differs for case {case['id']}") + body = row.get("response") + if not isinstance(body, dict): + raise ValueError("record parsed query response objects") + symbols = [*body.get("process_symbols", []), *body.get("definitions", [])] + anchors = [] + for symbol in symbols: + if isinstance(symbol, dict) and isinstance(symbol.get("filePath"), str): + anchor = (symbol["filePath"], symbol.get("name")) + if anchor not in anchors: + anchors.append(anchor) + matched = [] + for expected in case["anchors"]: + positions = [i + 1 for i, (path, name) in enumerate(anchors) + if path == expected["file"] and ("name" not in expected or name == expected["name"])] + matched.append(min(positions) if positions else None) + failed = bool(body.get("error")) + scores.append({ + "id": case["id"], "status": "error" if failed else "observed", + "expected_anchors": len(matched), "found_anchors": sum(p is not None for p in matched), + "anchor_recall": sum(p is not None for p in matched) / len(matched) if matched and not failed else None, + "first_anchor_position": min((p for p in matched if p is not None), default=None), + "returned_anchors": len(anchors), "response_bytes": len(json.dumps(body, ensure_ascii=True).encode()), + "truncated": body.get("truncated") is True or body.get("partial") is True, + }) + return { + "schema": "gitnexus.retrieval-diagnostics/v1", "source_sha": cases["source_sha"], + "runtime_sha": recordings["runtime_sha"], "cases": scores, + "complete": all(row["status"] == "observed" and not row.get("truncated") for row in scores), + "limitations": [ + "Source anchors are partial judgments; no precision or false-positive rate is inferred.", + "Position is flattened process_symbols then definitions order, not a calibrated relevance rank.", + "Coverage is conditional on the recorded query limits and index layers; preserve the capture receipt.", + "The scorer checks declared provenance; the capture must verify it against the sanitized index.", + ], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cases", type=Path, default=Path(__file__).with_name("retrieval_cases.json")) + parser.add_argument("--recordings", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + report = score_retrieval(json.loads(args.cases.read_text()), json.loads(args.recordings.read_text())) + with args.out.open("x") as handle: + json.dump(report, handle, indent=2, allow_nan=False) + handle.write("\n") + + +if __name__ == "__main__": + main() diff --git a/eval/workflow_bench/runner.py b/eval/workflow_bench/runner.py index cf8748271..6d37446b1 100644 --- a/eval/workflow_bench/runner.py +++ b/eval/workflow_bench/runner.py @@ -56,6 +56,8 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any +from .session_budget import session_budget + import yaml from .mcp_transport import connector_config, isolated_mcp @@ -492,6 +494,7 @@ def run_arm( "progress_label": progress_label, "claude_bin": sandbox.claude_bin, "timeout": args.timeout, + "max_budget_usd": getattr(args, "max_budget_usd", None), "model": args.model, "effort": args.effort, "env": env, @@ -1212,6 +1215,18 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: expected kinds is deliberately left to propagate — a harness bug must not be recorded as an ordinary infra-error and averaged into the evidence. """ + cell_started_at = datetime.now(UTC).isoformat() + cell_started = time.monotonic() + phase_started = cell_started + active_phase = "validation" + phase_seconds: dict[str, float] = {} + + def enter_phase(name: str) -> None: + nonlocal active_phase, phase_started + now = time.monotonic() + phase_seconds[active_phase] = phase_seconds.get(active_phase, 0.0) + now - phase_started + active_phase, phase_started = name, now + args = ctx.args task = ctx.task worktree: Path | None = None @@ -1235,6 +1250,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: raise RuntimeError("sanitized graph snapshot is unavailable") if ctx.asset_snapshot is None: raise RuntimeError("task asset snapshot is unavailable") + enter_phase("clone") if ctx.clone_template is not None: if not ctx.sanitized_head: raise RuntimeError("clone template is missing its sanitized HEAD") @@ -1243,9 +1259,11 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: else: worktree = make_worktree(ctx.repo, ctx.task_sha, ctx.trees_dir) sanitized_head = sanitize_clone_for_hidden_oracles(worktree) + enter_phase("graph_materialize") if gitnexus_available: assert ctx.graph_snapshot is not None ctx.graph_snapshot.materialize(worktree, sanitized_head=sanitized_head) + enter_phase("dependencies") dependency_mounts = stage_task_assets( task, repo=ctx.repo, @@ -1258,12 +1276,14 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: else [] ) ce_mounts = ce_plugin_mounts_for_arm(execution_arm, ctx.ce_plugin_snapshot) + enter_phase("mcp_startup") if isolated_baseline: endpoint = resources.enter_context(isolated_mcp( clone=worktree, mounts=[*dependency_mounts, *gitnexus_mounts], bwrap_bin=ctx.bwrap_bin, )) gitnexus_mounts = [endpoint] runtime_exposure = "stdio-only" + enter_phase("sandbox_setup") with prepare_sandbox( clone=worktree, claude_bin=args.claude_bin, @@ -1346,6 +1366,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: before_work_digest = ( implementation_diff_digest(sandbox, orig_sha) if execution_arm in IMPLEMENTATION_ARMS else "" ) + enter_phase("agent_and_verification") record = run_arm( execution_arm, task, @@ -1360,6 +1381,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: oracle_snapshot=ctx.oracle_snapshot, mcp_config_json=connector_config() if isolated_baseline else None, ) + enter_phase("artifacts") if execution_arm in ("review", "ce_review"): review_source = review_output_path(sandbox, REVIEW_OUTPUT) if review_source.is_file() and not review_source.is_symlink(): @@ -1433,6 +1455,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: detail = redact_text(str(exc), credential_secrets(args)) print(f"[{task['id']}][{arm}][run {run_idx}] infra-error: {detail}") finally: + enter_phase("cleanup") try: resources.close() except (SandboxError, OSError, RuntimeError) as exc: @@ -1453,8 +1476,14 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]: record["error_detail"] = ( f"primary={primary_kind}: {primary_detail}; cleanup: {type(cleanup_error).__name__}: {cleanup_error}" )[:2000] + cell_finished = time.monotonic() + phase_seconds["cleanup"] = cell_finished - phase_started record.update( { + "cell_started_at": cell_started_at, + "cell_finished_at": datetime.now(UTC).isoformat(), + "cell_duration_s": cell_finished - cell_started, + "cell_phase_seconds": phase_seconds, "task": task["id"], "class": task.get("class", ""), "run": run_idx, @@ -2105,6 +2134,10 @@ def build_parser() -> argparse.ArgumentParser: "subagent spend — token metrics count only the main loop", ) parser.add_argument("--promotion-min-runs", type=int, default=3) + parser.add_argument( + "--max-budget-usd", type=float, + help="opt-in CLI-reported cost stop per baseline session; provider billing may exceed it", + ) parser.add_argument("--promotion-min-improvement", type=float, default=5.0) parser.add_argument("--promotion-max-task-regression", type=float, default=20.0) parser.add_argument("--task-bindings-json", default=None, help=argparse.SUPPRESS) @@ -2126,6 +2159,9 @@ def main() -> None: parser = build_parser() args = parser.parse_args() try: + args.max_budget_usd = session_budget(args.max_budget_usd) + if args.max_budget_usd is not None and any(arm not in ("baseline", "baseline_nomcp", "baseline_cli") for arm in args.arms): + raise ValueError("--max-budget-usd is limited to single-session baseline arms") args.model = normalized_model_identifier(args.model) args.proposer_model = ( normalized_model_identifier(args.proposer_model, flag="--proposer-model") diff --git a/eval/workflow_bench/runner_sessions.py b/eval/workflow_bench/runner_sessions.py index 808629ccf..309178ef8 100644 --- a/eval/workflow_bench/runner_sessions.py +++ b/eval/workflow_bench/runner_sessions.py @@ -18,6 +18,8 @@ from pathlib import Path, PurePosixPath from typing import Any from .process_control import run_managed +from .session_budget import session_budget +from .session_diagnostics import EventClock, diagnose_session from .proposer_sandbox import ( SANDBOX_GITNEXUS, SANDBOX_GITNEXUS_REGISTRY, @@ -728,6 +730,7 @@ def run_claude( transcript_secrets: tuple[str, ...] = (), plugin_dirs: Sequence[str] = (), progress_label: str | None = None, + max_budget_usd: float | None = None, ) -> dict[str, Any]: """Run one headless session and return its usage record.""" @@ -735,6 +738,7 @@ def run_claude( # every file under the sandbox HOME is writable by agent tools and cannot # serve as trusted evidence. del transcript_projects, transcript_cwd, transcript_wait_seconds + max_budget_usd = session_budget(max_budget_usd) cmd = [ claude_bin, @@ -766,10 +770,13 @@ def run_claude( cmd += ["--model", model] if effort: cmd += ["--effort", effort] + if max_budget_usd is not None: + cmd += ["--max-budget-usd", str(max_budget_usd)] for tool in disallowed_tools or []: cmd += ["--disallowedTools", tool] managed_cmd = [*(command_prefix or []), *cmd] started = time.monotonic() + event_clock = EventClock() with contextlib.ExitStack() as progress_stack: progress = ( progress_stack.enter_context( @@ -781,6 +788,11 @@ def run_claude( if progress_label else None ) + def observe(chunk: bytes) -> None: + event_clock.observe(chunk) + if progress is not None: + progress.observe(chunk) + proc = run_managed( managed_cmd, cwd=None if command_prefix else cwd, @@ -789,17 +801,19 @@ def run_claude( require_pid_namespace=require_pid_namespace, stdin_data=prompt.encode(), capture_stdout_bytes=MAX_TRANSCRIPT_BYTES, - stdout_observer=progress.observe if progress is not None else None, + stdout_observer=observe, ) wall_s = time.monotonic() - started event_stream_error: str | None = None events: list[dict[str, Any]] = [] + event_offsets: list[float] | None = None try: if proc.stdout_capture is None: raise ValueError("parent process did not capture Claude stdout") if proc.stdout_capture_overflow: raise ValueError(f"parent-captured event stream exceeds {MAX_TRANSCRIPT_BYTES} bytes") events = _parse_parent_event_stream(proc.stdout_capture) + event_offsets = event_clock.finish(len(events)) data, result_index = _session_result(events, proc.duration_s) # A session that used a background task drains its bookkeeping after # the final result event (`background_tasks_changed`, `task_updated`, @@ -811,9 +825,12 @@ def run_claude( # a property of what the evidence readers below can see, rather than an # assumption that a `system` event never carries a tool_use block. events = events[: result_index + 1] + if event_offsets is not None: + event_offsets = event_offsets[: result_index + 1] except (UnicodeError, ValueError) as exc: event_stream_error = str(exc) data = {} + event_offsets = None usage = data.get("usage") or {} subtype = data.get("subtype") well_formed = all(field in usage for field in USAGE_FIELDS) @@ -860,6 +877,8 @@ def run_claude( "transcript_missing": False, "tool_use_counts": tool_use_counts, "gitnexus_tool_uses": sum(count for name, count in tool_use_counts.items() if name.startswith("mcp__gitnexus")), + "session_diagnostics": [diagnose_session(events, event_offsets, wall_s)], + "max_budget_usd": max_budget_usd, **{field: usage.get(field, 0) for field in USAGE_FIELDS}, } needs_evidence = expected_skill is not None or transcript_output_dir is not None @@ -955,4 +974,7 @@ def sum_sessions(sessions: list[dict[str, Any]]) -> dict[str, Any]: for name, count in session.get("tool_use_counts", {}).items(): tool_use_counts[name] = tool_use_counts.get(name, 0) + int(count) total["tool_use_counts"] = tool_use_counts + total["session_diagnostics"] = [ + diagnostic for session in sessions for diagnostic in session.get("session_diagnostics", []) + ] return total diff --git a/eval/workflow_bench/session_budget.py b/eval/workflow_bench/session_budget.py new file mode 100644 index 000000000..aa1783fb4 --- /dev/null +++ b/eval/workflow_bench/session_budget.py @@ -0,0 +1,20 @@ +"""Validate opt-in CLI-reported session budgets, not provider billing caps.""" + +from __future__ import annotations + +import math + + +def session_budget(value: float | None) -> float | None: + if value is None: + return None + message = "--max-budget-usd must be finite and greater than zero" + if type(value) not in (int, float): + raise ValueError(message) + try: + budget = float(value) + except OverflowError as exc: + raise ValueError(message) from exc + if not math.isfinite(budget) or budget <= 0: + raise ValueError(message) + return budget diff --git a/eval/workflow_bench/session_diagnostics.py b/eval/workflow_bench/session_diagnostics.py new file mode 100644 index 000000000..387390e3b --- /dev/null +++ b/eval/workflow_bench/session_diagnostics.py @@ -0,0 +1,471 @@ +"""Content-free diagnostics from parent-captured agent events. + +Tool intervals are observed at the stdout boundary, not inside the server. +The next tool action is a correlation, never proof of a workaround or its cost. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import shlex +import time +from collections import Counter +from pathlib import PurePosixPath +from typing import Any + +from .workload_diagnostics import COUNT_FIELDS, TOKEN_FIELDS, public_workload, workload_metrics + +SCHEMA = "gitnexus.session-diagnostics/v1" +MAX_EVENTS = 100_000 +MAX_CALLS = 2048 +CELL_PHASES = ("validation", "clone", "graph_materialize", "dependencies", "mcp_startup", + "sandbox_setup", "agent_and_verification", "artifacts", "cleanup") +COMMANDS = frozenset({ + "list_repos", "query", "context", "check", "impact", "explain", "pdg_query", + "route_map", "tool_map", "shape_check", "api_impact", "trace", "detect_changes", +}) +TOOLS = frozenset({ + "Read", "Grep", "Glob", "Edit", "Write", "Bash", "Skill", "ToolSearch", "other", + *(f"mcp__gitnexus__{name}" for name in COMMANDS), + *(f"cli__gitnexus__{name}" for name in COMMANDS), +}) +USAGE = ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "output_tokens") +COUNTERS = ( + "calls", "errors", "ambiguous", "not_found", "empty", "unknown_risk", "repeated", + "result_bytes", "max_result_bytes", "source_followups", "unfinished", "output_limit_exceeded", "timed_calls", + "stale", "truncated_results", + "lower_bound", "server_timed_calls", +) + + +class EventClock: + """Record receipt offsets without retaining or parsing event content. + + Called on the stdout reader thread; no I/O, unbounded buffers, or locks. + Several events in one chunk share a receipt time. Missing timing stays unknown. + """ + + def __init__(self) -> None: + self.started = time.monotonic() + self.offsets: list[float] = [] + self.partial = False + self.last = 0.0 + self.overflow = False + + def observe(self, chunk: bytes) -> None: + self.last = time.monotonic() - self.started + parts = chunk.split(b"\n") + for part in parts[:-1]: + if self.partial or part.strip(): + if len(self.offsets) < MAX_EVENTS: + self.offsets.append(self.last) + else: + self.overflow = True + self.partial = False + self.partial = self.partial or bool(parts[-1].strip()) + + def finish(self, expected: int) -> list[float] | None: + offsets = [*self.offsets, *([self.last] if self.partial else [])] + return offsets if not self.overflow and len(offsets) <= MAX_EVENTS and len(offsets) == expected else None + + +def _content(event: dict[str, Any]) -> list[Any]: + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None + if content is None: + content = event.get("content") + return content if isinstance(content, list) else [] + + +def _has_shell_control(command: str) -> bool: + """Reject operators/expansions while allowing quoted or escaped literals.""" + quote = None + escaped = False + for char in command: + if escaped: + escaped = False + continue + if quote == "'": + if char == "'": + quote = None + continue + if char == "\\": + escaped = True + elif char == '"': + quote = None if quote == '"' else '"' + elif char == "'" and quote is None: + quote = "'" + elif char in "`$" or (quote is None and char in ";|&\n<>()"): + return True + return False + + +def _tool_name(block: dict[str, Any]) -> str: + name = block.get("name") + if name == "Bash" and isinstance(block.get("input"), dict): + # Only a standalone shim command. Compound shell commands retain Bash + # attribution: their elapsed time cannot be called GitNexus server time. + command = block["input"].get("command") + try: + words = shlex.split(command) if isinstance(command, str) else [] + except ValueError: + words = [] + if (len(words) >= 2 and PurePosixPath(words[0]).name == "gitnexus" + and words[1] in COMMANDS and not _has_shell_control(command)): + return f"cli__gitnexus__{words[1]}" + return name if isinstance(name, str) and name in TOOLS else "other" + + +def _result_signals(content: Any) -> dict[str, Any]: + text = content if isinstance(content, str) else "\n".join( + block["text"] for block in content + if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str) + ) if isinstance(content, list) else "" + signals = { + "output_limit_exceeded": "exceeds maximum allowed tokens" in text, + "semantic_error": text.lstrip().lower().startswith("error:"), + } + try: + body = json.loads(text.split("\n\n---", 1)[0]) + except (ValueError, TypeError, RecursionError): + return signals + if not isinstance(body, dict): + return signals + error = body.get("error") + signals["semantic_error"] = isinstance(error, str) + signals["not_found"] = body.get("status") == "not_found" or isinstance(error, str) and "not found" in error.lower() + signals["ambiguous"] = body.get("status") == "ambiguous" + signals["unknown_risk"] = body.get("risk") == "UNKNOWN" + sections = [body[key] for key in ("processes", "process_symbols", "definitions") if key in body] + signals["empty"] = bool(sections) and all(section == [] for section in sections) + signals["truncated"] = body.get("truncated") is True or body.get("partial") is True + signals["lower_bound"] = body.get("epistemic") == "lower-bound" + timing = body.get("timing") + wall = timing.get("wall") if isinstance(timing, dict) else None + if type(wall) in (int, float) and 0 <= wall <= 10**12: + signals["server_wall_ms"] = wall + staleness = body.get("staleness") + if isinstance(staleness, dict) and staleness.get("status") in ("current", "behind", "diverged", "unknown"): + signals["staleness"] = staleness["status"] + return signals + + +def _union_seconds(intervals: list[tuple[float, float]]) -> float: + total, end = 0.0, 0.0 + for start, stop in sorted(intervals): + total += max(0.0, stop - max(end, start)) + end = max(end, stop) + return round(total, 6) + + +def diagnose_session( + events: list[dict[str, Any]], offsets: list[float] | None, duration_s: float, +) -> dict[str, Any]: + """Explain observed tool work; never estimate tool dollars or causal savings.""" + timing = offsets is not None and len(offsets) == len(events) and all( + type(value) in (int, float) and math.isfinite(value) and 0 <= value <= duration_s + and (index == 0 or value >= offsets[index - 1]) for index, value in enumerate(offsets) + ) + calls: list[dict[str, Any]] = [] + inputs: dict[int, Any] = {} + results: dict[int, Any] = {} + pending: dict[str, dict[str, Any]] = {} + signatures: dict[str, int] = {} + turns: dict[str, dict[str, Any]] = {} + retries = 0 + truncated = False + duplicate_ids = False + seen_ids: set[str] = set() + for index, event in enumerate(events): + if event.get("type") == "system" and event.get("subtype") == "api_retry": + retries += 1 + if event.get("type") == "assistant": + message = event.get("message") or {} + # Streaming can repeat a message id. Retain its last usage snapshot + # rather than adding cumulative snapshots as separate model requests. + if isinstance(message, dict) and isinstance(message.get("usage"), dict): + key = str(message.get("id", index)) + if key in turns or len(turns) < MAX_CALLS: + turns[key] = {"event_index": index, **{ + field: value for field in USAGE + if type(value := message["usage"].get(field)) is int and value >= 0 + }} + else: + truncated = True + for block in _content(event): + if not isinstance(block, dict) or block.get("type") != "tool_use": + continue + if len(calls) >= MAX_CALLS: + truncated = True + continue + tool_id = block.get("id") + if not isinstance(tool_id, str) or not 0 < len(tool_id) <= 256: + duplicate_ids = True + continue + if tool_id in seen_ids: + duplicate_ids = True + continue + seen_ids.add(tool_id) + name = _tool_name(block) + encoded = json.dumps(block.get("input"), sort_keys=True, ensure_ascii=False, separators=(",", ":")) + signature = hashlib.sha256((name + encoded).encode(errors="backslashreplace")).hexdigest() + call = { + "index": len(calls), "tool": name, "request_event": index, + "started_s": offsets[index] if timing else None, + "completed_s": None, "duration_s": None, "result_event": None, + "input_bytes": len(encoded.encode(errors="backslashreplace")), "result_bytes": None, + "repeat_of": signatures.get(signature), "outcome": "unfinished", + "source_followups": 0, + } + signatures[signature] = call["index"] + pending[tool_id] = call + calls.append(call) + inputs[call["index"]] = block.get("input") + elif event.get("type") == "user": + for block in _content(event): + if not isinstance(block, dict) or block.get("type") != "tool_result": + continue + tool_id = block.get("tool_use_id") + call = pending.pop(tool_id, None) if isinstance(tool_id, str) else None + if call is None: + duplicate_ids = True + continue + content = block.get("content") + results[call["index"]] = content + call["result_bytes"] = len(json.dumps( + content, ensure_ascii=False, separators=(",", ":"), + ).encode(errors="backslashreplace")) + call["result_event"] = index + if timing: + call["completed_s"] = offsets[index] + call["duration_s"] = round(offsets[index] - call["started_s"], 6) + call["outcome"] = "error" if block.get("is_error") is True else "ok" + if "__gitnexus__" in call["tool"]: + call.update(_result_signals(content)) + if call.get("semantic_error") or call.get("output_limit_exceeded"): + call["outcome"] = "error" + # Only inspect the immediately following assistant action; this is a + # temporal signal, not evidence that GitNexus caused the source read. + next_actions: dict[int, list[str]] = {} + next_tools: list[str] = [] + for index in range(len(events) - 1, -1, -1): + next_actions[index] = next_tools + if events[index].get("type") == "assistant": + next_tools = [_tool_name(b) for b in _content(events[index]) + if isinstance(b, dict) and b.get("type") == "tool_use"] + tools: dict[str, Counter] = {} + intervals, mcp_intervals = [], [] + for call in calls: + following = next_actions.get(call["result_event"], []) + call["next_tools"] = following + call["source_followups"] = sum(name in ("Read", "Grep", "Glob") for name in following) + totals = tools.setdefault(call["tool"], Counter({**dict.fromkeys(COUNTERS, 0), "elapsed_s": 0.0, "server_wall_s": 0.0})) + totals["calls"] += 1 + totals["errors"] += call["outcome"] == "error" + totals["unfinished"] += call["outcome"] == "unfinished" + totals["repeated"] += call["repeat_of"] is not None + totals["result_bytes"] += call["result_bytes"] or 0 + totals["max_result_bytes"] = max(totals["max_result_bytes"], call["result_bytes"] or 0) + for field in ("ambiguous", "not_found", "empty", "unknown_risk", "output_limit_exceeded", "lower_bound"): + totals[field] += call.get(field) is True + if "server_wall_ms" in call: + totals["server_timed_calls"] += 1 + totals["server_wall_s"] += call["server_wall_ms"] / 1000 + totals["stale"] += call.get("staleness") in ("behind", "diverged") + totals["truncated_results"] += call.get("truncated") is True + call["problematic"] = call["outcome"] == "error" or any( + call.get(field) for field in ("not_found", "ambiguous", "empty", "unknown_risk") + ) + totals["source_followups"] += call["source_followups"] if call["problematic"] else 0 + if call["duration_s"] is not None: + totals["timed_calls"] += 1 + totals["elapsed_s"] += call["duration_s"] + interval = (call["started_s"], call["completed_s"]) + intervals.append(interval) + if call["tool"].startswith("mcp__gitnexus__"): + mcp_intervals.append(interval) + complete = bool(events) and timing and not (pending or truncated or duplicate_ids) + wait = _union_seconds(intervals) if complete else None + return { + "schema": SCHEMA, "timing_complete": complete, "truncated": truncated, + "tool_wait_s": wait, "mcp_wait_s": _union_seconds(mcp_intervals) if complete else None, + "other_wall_s": round(max(0.0, duration_s - wait), 6) if wait is not None else None, + "api_retries": retries, "tools": { + name: {**values, "elapsed_s": round(values["elapsed_s"], 6) if values["timed_calls"] else None, + "server_wall_s": round(values["server_wall_s"], 6) if values["server_timed_calls"] else None} + for name, values in tools.items() + }, "calls": calls, "assistant_usage": list(turns.values()), + "workload": workload_metrics( + [(call, inputs.get(call["index"]), results.get(call["index"])) for call in calls], + sorted(turns.values(), key=lambda turn: turn["event_index"]), events, + ), + } + + +def public_diagnostics(value: Any) -> list[dict[str, Any]]: + """Idempotent metadata whitelist for public summaries and bounded outliers.""" + output = [] + for session in value[:16] if isinstance(value, list) else []: + if not isinstance(session, dict) or session.get("schema") != SCHEMA: + continue + safe: dict[str, Any] = {"schema": SCHEMA} + for field in ("timing_complete", "truncated"): + safe[field] = session.get(field) is True + for field in ("tool_wait_s", "mcp_wait_s", "other_wall_s", "api_retries"): + number = session.get(field) + safe[field] = number if type(number) in (int, float) and 0 <= number <= 10**12 else None + safe["tools"] = {} + if "workload" in session: + safe["workload"] = public_workload(session["workload"]) + tools = session.get("tools") + if isinstance(tools, dict): + for name in sorted(TOOLS): + row = tools.get(name) + if not isinstance(row, dict): + continue + safe["tools"][name] = { + field: number if type(number := row.get(field)) is int and 0 <= number <= 10**12 else None + for field in COUNTERS + } + for field in ("elapsed_s", "server_wall_s"): + elapsed = row.get(field) + safe["tools"][name][field] = elapsed if type(elapsed) in (int, float) and 0 <= elapsed <= 10**12 else None + # Keep a few inspectable outliers in the already-uploaded release JSON. + # Full calls and assistant usage remain in local results.jsonl. + for field, sort_key in ( + ("largest_results", "result_bytes"), ("slowest_calls", "duration_s"), ("problem_calls", "result_bytes"), + ): + candidates = session.get(field, session.get("calls", [])) + safe[field] = [] + if not isinstance(candidates, list): + continue + for call in sorted( + [c for c in candidates[:MAX_CALLS] if isinstance(c, dict) + and (field != "problem_calls" or c.get("problematic") is True) + and type(c.get(sort_key)) in (int, float) and 0 <= c[sort_key] <= 10**12], + key=lambda c: c[sort_key], reverse=True, + )[:5]: + tool = call.get("tool") + item = {"tool": tool if isinstance(tool, str) and tool in TOOLS else "other"} + for name in ("index", "request_event", "result_event", "input_bytes", "result_bytes", "duration_s", "source_followups", "server_wall_ms"): + number = call.get(name) + item[name] = number if type(number) in (int, float) and 0 <= number <= 10**12 else None + outcome = call.get("outcome") + item["outcome"] = outcome if outcome in ("ok", "error", "unfinished") else "unknown" + for name in ("problematic", "not_found", "ambiguous", "empty", "unknown_risk", "output_limit_exceeded"): + item[name] = call.get(name) is True + safe[field].append(item) + output.append(safe) + return output + + +def public_cell_diagnostics(row: dict[str, Any]) -> dict[str, Any]: + """Shared publication boundary for release and optional-arm artifacts.""" + safe = {} + if "session_diagnostics" in row: + safe["session_diagnostics"] = public_diagnostics(row["session_diagnostics"]) + if "cell_duration_s" in row: + value = row["cell_duration_s"] + safe["cell_duration_s"] = value if type(value) in (int, float) and 0 <= value <= 10**12 else None + phases = row.get("cell_phase_seconds") + if isinstance(phases, dict): + safe["cell_phase_seconds"] = { + field: value if type(value := phases[field]) in (int, float) and 0 <= value <= 10**12 else None + for field in CELL_PHASES if field in phases + } + return safe + + +def render_diagnostics(rows: list[dict[str, Any]]) -> list[str]: + """Add tool diagnostics to the existing public report without new CI wiring.""" + lines = [] + for arm in ("baseline_nomcp", "baseline", "baseline_cli"): + cells = [row for row in rows if row.get("arm") == arm] + sessions = [s for row in cells for s in public_diagnostics(row.get("session_diagnostics"))] + if not sessions: + continue + measured = sum(bool(public_diagnostics(row.get("session_diagnostics"))) for row in cells) + lines += ["", f"### Tool diagnostics: {arm}", "", + f"Diagnostics available for {measured}/{len(cells)} cells.", "", + "| Tool | Calls | Errors | Ambiguous | Not found | Empty | UNKNOWN risk | Repeated args | Result bytes | Largest result | Source follow-ups after problems | Timed calls | Elapsed seconds |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"] + totals: dict[str, Counter] = {} + coverage: dict[str, Counter] = {} + tool_sessions: Counter = Counter() + for session in sessions: + for name, values in session["tools"].items(): + tally = totals.setdefault(name, Counter()) + known = coverage.setdefault(name, Counter()) + tool_sessions[name] += 1 + for field, number in values.items(): + if number is not None: + known[field] += 1 + tally[field] = max(tally[field], number) if field == "max_result_bytes" else tally[field] + number + for name, tally in sorted(totals.items()): + lines.append("| " + " | ".join(str(v) for v in ( + name, *(tally[f] if coverage[name][f] == tool_sessions[name] else "n/a" for f in + ("calls", "errors", "ambiguous", "not_found", "empty", "unknown_risk", + "repeated", "result_bytes", "max_result_bytes", "source_followups", "timed_calls")), + f"{tally['elapsed_s']:.1f}" if tally["timed_calls"] else "n/a", + )) + " |") + lines += ["", "| Tool | Stale | Truncated results | Lower bound | Output limit | Unfinished | Server-timed calls | Server seconds |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"] + for name, tally in sorted(totals.items()): + lines.append("| " + " | ".join(str(v) for v in ( + name, *(tally[f] if coverage[name][f] == tool_sessions[name] else "n/a" for f in + ("stale", "truncated_results", "lower_bound", "output_limit_exceeded", "unfinished", "server_timed_calls")), + f"{tally['server_wall_s']:.3f}" if tally["server_timed_calls"] else "n/a", + )) + " |") + complete = [s for s in sessions if s["timing_complete"] and s["tool_wait_s"] is not None] + lines += ["", f"Complete host timing: {len(complete)}/{len(sessions)} sessions."] + limited = sum(s["truncated"] for s in sessions) + if limited: + lines.append(f"Diagnostics truncated in {limited} session(s); tool counts are lower bounds.") + if complete: + lines.append(f"Observed tool wait: {sum(s['tool_wait_s'] for s in complete):.1f}s; " + f"MCP wait: {sum(s['mcp_wait_s'] or 0 for s in complete):.1f}s " + "(a subset of tool wait). Parallel intervals are counted once per session.") + for field, label in (("other_wall_s", "Other session wall time (seconds)"), ("api_retries", "Observed API retries")): + values = [s[field] for s in sessions if s.get(field) is not None] + lines.append(f"{label}: {sum(values):.1f} across {len(values)}/{len(sessions)} sessions." + if values else f"{label}: n/a.") + workloads = [s["workload"] for s in sessions if s.get("workload")] + if workloads: + lines += ["", f"Workload signals available for {len(workloads)}/{len(sessions)} sessions.", "", + "| Workload signal | Observed count | Sessions reporting |", "| --- | ---: | ---: |"] + for field in COUNT_FIELDS: + values = [w[field] for w in workloads if w.get(field) is not None] + lines.append(f"| {field} | {sum(values) if values else 'n/a'} | {len(values)}/{len(sessions)} |") + peaks = [w["max_input_tokens"] for w in workloads if w.get("max_input_tokens") is not None] + if peaks: + lines.append(f"\nLargest observed CLI input context: {max(peaks):,} tokens. " + "Uses complete Anthropic-shaped usage snapshots; provider-native usage may differ.") + lines += ["", "| Context/cache signal | Session minimum | Session maximum | Sessions reporting |", + "| --- | ---: | ---: | ---: |"] + for field in TOKEN_FIELDS: + values = [w[field] for w in workloads if w.get(field) is not None] + lines.append(f"| {field} | {min(values) if values else 'n/a'} | {max(values) if values else 'n/a'} | " + f"{len(values)}/{len(sessions)} |") + lines += ["", "
Largest, slowest, and problematic tool calls", "", + "Session and call indexes identify observations; arguments and source content are omitted.", "", + "| View | Session | Call | Tool | Result bytes | Host seconds | Outcome |", + "| --- | ---: | ---: | --- | ---: | ---: | --- |"] + for view, sort_key in (("largest_results", "result_bytes"), ("slowest_calls", "duration_s"), + ("problem_calls", "result_bytes")): + calls = [(index, call) for index, session in enumerate(sessions) for call in session[view]] + for index, call in sorted(calls, key=lambda item: item[1][sort_key], reverse=True)[:5]: + duration = call["duration_s"] + lines.append(f"| {view} | {index} | {call['index']} | {call['tool']} | {call['result_bytes']} | " + f"{f'{duration:.3f}' if duration is not None else 'n/a'} | {call['outcome']} |") + lines += ["", "
"] + if lines: + lines += ["", ("Receipt timing includes transport and scheduling, not just server execution. Other wall time is not model time. " + "Bytes are serialized payload bytes, not tokens. Repeated arguments may be justified after edits. " + "Source follow-ups are the next assistant action after a problematic result, not proof of a workaround. " + "Per-tool elapsed sums may overlap. Session cost includes all work; no per-tool dollar attribution is available.")] + lines += [("A file read after a graph reference is an association, not duplicated content or low relevance. " + "Repeated verification can be necessary. Bash classification is conservative; mixed commands stay unclassified. " + "Missing usage fields remain unknown. Context totals describe observed snapshots only.")] + return lines diff --git a/eval/workflow_bench/workload_diagnostics.py b/eval/workflow_bench/workload_diagnostics.py new file mode 100644 index 000000000..7d2b56fc9 --- /dev/null +++ b/eval/workflow_bench/workload_diagnostics.py @@ -0,0 +1,247 @@ +"""Content-free workload signals. Associations are not causal cost attribution.""" + +from __future__ import annotations + +import json +import shlex +from collections import Counter +from pathlib import PurePosixPath +from typing import Any + +# Also the publication allowlist. No arguments, paths, source, or hashes leave +# this module. Unknown/missing observations must not become measured zeros. +COUNT_FIELDS = ( + "source_reads", "repeated_source_reads", "reads_after_graph_reference", + "reads_after_graph_content_reference", "query_context_read_chains", + "graph_calls", "graph_calls_before_first_edit", "edits", "edits_after_verification", + "verification_calls", "verification_repeated_without_observed_change", + "verification_repeated_after_possible_change", "verification_errors", + "bash_test", "bash_typecheck", "bash_lint", "bash_build", "bash_source", + "bash_mutation", "bash_mixed", "bash_other", "compactions", + "usage_snapshots", "complete_input_snapshots", "usage_after_graph_snapshots", + "reference_scan_limited", +) +TOKEN_FIELDS = ( + "first_input_tokens", "last_input_tokens", "max_input_tokens", "sum_input_tokens", + "sum_cache_read_tokens", "sum_cache_write_tokens", "sum_output_tokens", + "input_tokens_after_graph", +) +FIELDS = (*COUNT_FIELDS, *TOKEN_FIELDS) +VERIFY = frozenset({"test", "typecheck", "lint", "build"}) +_UV_RUN_FLAGS = frozenset({"--locked", "--frozen", "--no-sync", "--offline", "--all-extras", + "--no-dev", "--only-dev", "--isolated", "--active", "--no-project"}) +_UV_RUN_VALUES = frozenset({"--extra", "--group", "--python", "--project", "--directory", + "--package", "--with"}) + + +def classify_bash(command: Any) -> str: + """Conservative lexical classification, never shell evaluation. + + Compound commands are explicitly mixed; their time cannot be assigned to + one test/build. A common `cd dir && command` prefix is the sole exception. + """ + if not isinstance(command, str): + return "other" + try: + words = shlex.split(command) + except ValueError: + return "other" + if len(words) >= 4 and words[0] == "cd" and words[2] == "&&": + words = words[3:] + # Inspect the remainder too: shell substitutions and pipes stay mixed. + command = shlex.join(words) + if any(op in command for op in (";", "|", "&", "\n", "`", "$(", ">", "<")): + return "mixed" + if not words: + return "other" + executable = PurePosixPath(words[0]).name + args = words[1:] + if executable == "uv" and args[:1] == ["run"]: + args = args[1:] + while args and args[0].startswith("-"): + option = args.pop(0) + if option == "--": + break + name, separator, value = option.partition("=") + if name in _UV_RUN_FLAGS and not separator: + continue + if name in _UV_RUN_VALUES: + if separator and value: + continue + if not separator and args and args[0] and not args[0].startswith("-"): + args.pop(0) + continue + # Unknown or malformed options may change what uv executes. + return "other" + return classify_bash(shlex.join(args)) + if executable in ("python", "python3") and args[:2] == ["-m", "pytest"]: + return "test" + if executable in ("npm", "pnpm", "yarn", "bun", "npx") and args: + if args[0] in ("run", "exec", "x"): + args = args[1:] + executable = args[0] if args else "" + if executable in ("test", "pytest", "vitest", "jest") or executable.startswith("test:"): + return "test" + if executable in ("tsc", "mypy", "pyright", "typecheck"): + return "typecheck" + if executable in ("eslint", "ruff", "prettier", "lint"): + mutates = any(arg in ("--fix", "--write", "-w") for arg in args) + if executable == "ruff" and "format" in args and not any(arg in ("--check", "--diff") for arg in args): + mutates = True + return "mutation" if mutates else "lint" + if executable in ("build", "vite", "tsup"): + return "build" + if executable in ("rg", "grep", "cat", "head", "tail", "sed"): + return "mutation" if executable == "sed" and any( + arg.startswith(("-i", "--in-place")) for arg in args + ) else "source" + if executable in ("apply_patch", "patch", "touch", "rm", "mv", "cp", "tee"): + return "mutation" + return "other" + + +def _references(content: Any) -> tuple[dict[str, bool], bool]: + """Structured result references only; cap traversal of large tool results.""" + if isinstance(content, list): + text = "\n".join(b["text"] for b in content + if isinstance(b, dict) and isinstance(b.get("text"), str)) + else: + text = content if isinstance(content, str) else "" + if len(text) > 2_000_000: + return {}, True + try: + body = json.loads(text.split("\n\n---", 1)[0]) + except (ValueError, RecursionError): + return {}, False + stack, paths, visited = [body], {}, 0 + while stack and visited < 10_000 and len(paths) < 1000: + item = stack.pop() + visited += 1 + if isinstance(item, dict): + path = item.get("filePath", item.get("file_path")) + if isinstance(path, str) and path and len(path) <= 4096: + # Same-object source only. Descriptions and arbitrary text are + # not evidence that the response supplied the file's contents. + supplied = any(isinstance(item.get(key), str) and bool(item[key].strip()) + for key in ("content", "source", "code")) + paths[path] = paths.get(path, False) or supplied + stack.extend(value for value in item.values() if isinstance(value, (dict, list))) + elif isinstance(item, list): + stack.extend(item) + return paths, bool(stack) + + +def _same_path(read: str, reference: str) -> bool: + read_parts, ref_parts = PurePosixPath(read).parts, PurePosixPath(reference).parts + if ".." in read_parts or ".." in ref_parts: + return False + if read_parts == ref_parts: + return True + # Absolute sandbox paths can match repo-relative graph paths. Never match + # unrelated absolute roots or a basename-only relative reference. + return (len(ref_parts) > 1 and not reference.startswith("/") + and len(read_parts) >= len(ref_parts) and read_parts[-len(ref_parts):] == ref_parts) + + +def workload_metrics( + observations: list[tuple[dict[str, Any], Any, Any]], + usage: list[dict[str, Any]], events: list[dict[str, Any]], +) -> dict[str, int | None]: + """Compute bounded numeric signals from accepted, deduplicated calls. + + `observations` holds (call metadata, input, result content) only in memory. + No assumption that an unknown Bash action left the checkout unchanged. + """ + counts = Counter(dict.fromkeys(COUNT_FIELDS, 0)) + counts["compactions"] = sum(e.get("type") == "system" and e.get("subtype") == "compact_boundary" for e in events) + reads: set[str] = set() + checks: dict[str, int] = {} + mutation_epoch = 0 + verified = False + graph_results: list[tuple[int, str, dict[str, bool]]] = [] + reference_budget = 5000 + for call, args, content in observations: + args = args if isinstance(args, dict) else {} + tool = call["tool"] + if "__gitnexus__" in tool: + counts["graph_calls"] += 1 + counts["graph_calls_before_first_edit"] += counts["edits"] == 0 + if call.get("result_event") is not None: + paths, limited = _references(content) + if len(paths) > reference_budget: + paths = dict(list(paths.items())[:reference_budget]) + limited = True + reference_budget -= len(paths) + counts["reference_scan_limited"] += limited + graph_results.append((call["result_event"], tool.rsplit("__", 1)[-1], paths)) + elif tool == "Read": + counts["source_reads"] += 1 + signature = json.dumps(args, sort_keys=True, ensure_ascii=True) + counts["repeated_source_reads"] += signature in reads + reads.add(signature) + path = args.get("file_path") + prior = sorted((g for g in graph_results if g[0] < call["request_event"]), key=lambda g: g[0]) + if isinstance(path, str): + matching = [supplied for _, _, paths in prior for ref, supplied in paths.items() if _same_path(path, ref)] + counts["reads_after_graph_reference"] += bool(matching) + counts["reads_after_graph_content_reference"] += any(matching) + counts["query_context_read_chains"] += ( + len(prior) >= 2 and [g[1] for g in prior[-2:]] == ["query", "context"] + and any(_same_path(path, ref) for ref in prior[-1][2]) + ) + elif tool in ("Edit", "Write"): + counts["edits"] += 1 + counts["edits_after_verification"] += verified + mutation_epoch += 1 + elif tool == "Bash": + category = classify_bash(args.get("command")) + counts[f"bash_{category}"] += 1 + if category in VERIFY: + counts["verification_calls"] += 1 + counts["verification_errors"] += call["outcome"] == "error" + signature = json.dumps(args, sort_keys=True, ensure_ascii=True) + if signature in checks: + field = "verification_repeated_without_observed_change" if checks[signature] == mutation_epoch else "verification_repeated_after_possible_change" + counts[field] += 1 + checks[signature] = mutation_epoch + verified = True + elif category in ("mutation", "mixed", "other"): + mutation_epoch += 1 + if category == "mutation": + counts["edits"] += 1 + counts["edits_after_verification"] += verified + counts["usage_snapshots"] = len(usage) + input_fields = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens") + totals = [sum(u[f] for f in input_fields) + if all(type(u.get(f)) is int and u[f] >= 0 for f in input_fields) else None for u in usage] + complete = [total for total in totals if total is not None] + counts["complete_input_snapshots"] = len(complete) + after_graph = [] + for u, total in zip(usage, totals, strict=True): + preceding = [call for call, _, _ in observations + if call.get("result_event") is not None and call["result_event"] < u["event_index"]] + latest = max((call["result_event"] for call in preceding), default=-1) + if any("__gitnexus__" in c["tool"] and c["result_event"] == latest for c in preceding): + after_graph.append(total) + counts["usage_after_graph_snapshots"] = len(after_graph) + tokens = { + "first_input_tokens": totals[0] if totals else None, + "last_input_tokens": totals[-1] if totals else None, + "max_input_tokens": max(complete) if complete else None, + "sum_input_tokens": sum(complete) if totals and len(complete) == len(totals) else None, + "input_tokens_after_graph": sum(after_graph) if after_graph and all(t is not None for t in after_graph) else None, + } + for dest, source in (("sum_cache_read_tokens", "cache_read_input_tokens"), + ("sum_cache_write_tokens", "cache_creation_input_tokens"), ("sum_output_tokens", "output_tokens")): + # A partial sum labelled as total would understate spend. These fields + # stay unknown unless every observed usage snapshot supplies the field. + tokens[dest] = sum(u[source] for u in usage) if usage and all( + type(u.get(source)) is int and u[source] >= 0 for u in usage + ) else None + return {**counts, **tokens} + + +def public_workload(value: Any) -> dict[str, int | None]: + if not isinstance(value, dict): + return {} + return {field: n if type(n := value.get(field)) is int and 0 <= n <= 10**15 else None for field in FIELDS} diff --git a/gitnexus/test/unit/staleness.test.ts b/gitnexus/test/unit/staleness.test.ts index c54d53d02..8e856d582 100644 --- a/gitnexus/test/unit/staleness.test.ts +++ b/gitnexus/test/unit/staleness.test.ts @@ -123,7 +123,7 @@ describe('checkStalenessAsync', () => { expect(result.commitsBehind).toBe(0); }); - it('parallel calls complete faster than sequential', async () => { + it('concurrent calls agree with the synchronous lookup', async () => { let headCommit: string; try { headCommit = execFileSync('git', ['rev-parse', 'HEAD'], { @@ -137,19 +137,11 @@ describe('checkStalenessAsync', () => { const cwd = process.cwd(); const N = 10; - // Parallel - const t0 = performance.now(); - await Promise.all(Array.from({ length: N }, () => checkStalenessAsync(cwd, headCommit))); - const parallelMs = performance.now() - t0; - - // Sequential sync - const t1 = performance.now(); - for (let i = 0; i < N; i++) checkStaleness(cwd, headCommit); - const sequentialMs = performance.now() - t1; - - // Parallel should be meaningfully faster than sequential. - // Use a generous ratio to avoid flakiness on slow CI machines. - expect(parallelMs).toBeLessThan(sequentialMs * 1.5); + const expected = checkStaleness(cwd, headCommit); + const results = await Promise.all( + Array.from({ length: N }, () => checkStalenessAsync(cwd, headCommit)), + ); + expect(results).toEqual(Array.from({ length: N }, () => expected)); }); }); From 1736a1bfd806fbd113507c516e7156b0b2c001f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Fri, 30 Oct 2026 15:19:51 +0000 Subject: [PATCH 02/26] feat(eval): add capped two-cell trial (#3778) Co-authored-by: Gergo Magyar --- .github/workflows/release-evaluation.yml | 131 +++++++++++-- eval/tests/test_diagnostic_trial.py | 237 +++++++++++++++++++++++ eval/tests/test_ec2_workflows.py | 6 + eval/workflow_bench/README.md | 22 +++ eval/workflow_bench/diagnostic_trial.py | 122 ++++++++++++ 5 files changed, 502 insertions(+), 16 deletions(-) create mode 100644 eval/tests/test_diagnostic_trial.py create mode 100644 eval/workflow_bench/diagnostic_trial.py diff --git a/.github/workflows/release-evaluation.yml b/.github/workflows/release-evaluation.yml index f5001a377..9e8194106 100644 --- a/.github/workflows/release-evaluation.yml +++ b/.github/workflows/release-evaluation.yml @@ -17,6 +17,10 @@ on: description: Start, probe, and stop EC2 without paid model calls type: boolean default: false + diagnostic_trial: + description: 'Two serial cells, $5 CLI stop each; requires a full candidate SHA, gpt-5.6-sol/xhigh, and no other experiment flags' + type: boolean + default: false mcp_tool_loading: description: Add candidate deferred-MCP experiment (12 extra paid cells) type: boolean @@ -93,6 +97,25 @@ jobs: cleanup_required: ${{ steps.boot.outputs.cleanup_required }} stop_deadline_epoch: ${{ steps.boot.outputs.stop_deadline_epoch }} steps: + - name: Validate diagnostic trial authorization scope + env: + TRIAL: ${{ inputs.diagnostic_trial }} + RUNNER_ONLY: ${{ inputs.runner_only }} + CLI_ARM: ${{ inputs.cli_arm }} + DEFERRED_ARM: ${{ inputs.mcp_tool_loading }} + CANDIDATE_REF: ${{ inputs.candidate_ref }} + MODEL: ${{ inputs.model }} + EFFORT: ${{ inputs.effort }} + run: | + set -euo pipefail + if [[ "$TRIAL" == true ]]; then + if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_RUN_ATTEMPT" != 1 || + "$RUNNER_ONLY" == true || "$CLI_ARM" == true || "$DEFERRED_ARM" == true || + "$MODEL" != gpt-5.6-sol || "$EFFORT" != xhigh || ! "$CANDIDATE_REF" =~ ^[0-9a-f]{40}$ ]]; then + echo '::error::Trial requires a fresh manual dispatch, a full candidate SHA, gpt-5.6-sol/xhigh, and no other experiment flags. Paid trials are never retried automatically.' + exit 1 + fi + fi - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -234,6 +257,7 @@ jobs: working-directory: eval env: GH_TOKEN: ${{ github.token }} + DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }} run: | set -euo pipefail if [[ -n "$INPUT_PR" ]]; then @@ -244,10 +268,12 @@ jobs: fi [[ -z "$INPUT_PR_SHA" && -z "$COMPARISON_RUN_ID" ]] || { echo '::error::PR preview inputs require candidate_pr.'; exit 1; } # Resolve last stable once; both runtimes use the same trusted tasks. - stable="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq '.tag_name')" - [[ "$stable" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo '::error::No stable release found.'; exit 1; } - stable_sha="$(git rev-parse --verify "$stable^{commit}")" - echo "stable_sha=$stable_sha" >> "$GITHUB_OUTPUT" + if [[ "${DIAGNOSTIC_TRIAL:-false}" != true ]]; then + stable="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq '.tag_name')" + [[ "$stable" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo '::error::No stable release found.'; exit 1; } + stable_sha="$(git rev-parse --verify "$stable^{commit}")" + echo "stable_sha=$stable_sha" >> "$GITHUB_OUTPUT" + fi ref="$INPUT_REF" if [[ "$ref" == 'latest-rc' ]]; then ref="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --jq \ @@ -275,7 +301,7 @@ jobs: persist-credentials: false fetch-depth: 0 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - if: inputs.candidate_pr == '' + if: inputs.candidate_pr == '' && inputs.diagnostic_trial != true with: ref: ${{ steps.resolve.outputs.stable_sha }} path: stable @@ -338,24 +364,41 @@ jobs: test "$("$cli/node_modules/@anthropic-ai/claude-code-linux-x64/claude" --version)" = '2.1.214 (Claude Code)' - name: Build pinned runtime and task dependencies working-directory: eval + env: + DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }} run: | set -euo pipefail for checkout in tasks-base stable candidate; do - if [[ -n "$INPUT_PR" && "$checkout" == stable ]]; then continue; fi + if [[ ( -n "$INPUT_PR" || "${DIAGNOSTIC_TRIAL:-false}" == true ) && "$checkout" == stable ]]; then continue; fi uv run --locked --extra dev python -m workflow_bench.release_build \ --repo "$GITHUB_WORKSPACE/$checkout" done - name: Prepare pinned task set working-directory: eval + env: + DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }} run: | set -euo pipefail for runtime in stable candidate; do - if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi + if [[ ( -n "$INPUT_PR" || "${DIAGNOSTIC_TRIAL:-false}" == true ) && "$runtime" == stable ]]; then continue; fi uv run --locked --extra dev python -m workflow_bench.release_report prepare \ --repo "$GITHUB_WORKSPACE/$runtime" --task-repo "$GITHUB_WORKSPACE/tasks-base" \ --out "$RUNNER_TEMP/release-evaluation/$runtime" \ --model "$MODEL" --effort "$EFFORT" --runs 3 done + - name: Prepare the capped diagnostic trial + if: inputs.diagnostic_trial == true + working-directory: eval + env: + RUNTIME_SHA: ${{ steps.resolve.outputs.sha }} + run: | + uv run --locked --extra dev python -m workflow_bench.diagnostic_experiments \ + --tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \ + --repository "$GITHUB_WORKSPACE/candidate" --runtime "$GITHUB_WORKSPACE/candidate" \ + --harness-sha "$GITHUB_SHA" --runtime-sha "$RUNTIME_SHA" \ + --model "$MODEL" --effort "$EFFORT" --seed 3762 --repeats 1 \ + --variants no_graph mcp_eager --task-ids trivial-status-json-alias \ + --max-budget-usd 5 --out "$RUNNER_TEMP/release-evaluation/trial" - name: Require historical task dependencies before paid PR sessions if: inputs.candidate_pr != '' working-directory: eval @@ -375,8 +418,17 @@ jobs: - name: Materialize both release graphs before paid sessions timeout-minutes: 120 working-directory: eval + env: + DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }} run: | set -euo pipefail + if [[ "${DIAGNOSTIC_TRIAL:-false}" == true ]]; then + uv run --locked --extra dev python -m workflow_bench.release_preflight \ + --tasks "$RUNNER_TEMP/release-evaluation/trial/cell-0000.tasks.json" \ + --gitnexus-root "$GITHUB_WORKSPACE/candidate" \ + --claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" + exit 0 + fi for runtime in stable candidate; do if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi uv run --locked --extra dev python -m workflow_bench.release_preflight \ @@ -406,8 +458,21 @@ jobs: env: GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }} GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }} + DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }} run: | set -euo pipefail + if [[ "${DIAGNOSTIC_TRIAL:-false}" == true ]]; then + remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 )) + [[ "$GITHUB_RUN_ATTEMPT" == 1 ]] || { echo '::error::Paid trials cannot be rerun, including failed-job retries.'; exit 1; } + (( remaining > 0 )) || { echo '::error::No time remains before the stop and evidence reserve.'; exit 1; } + (( remaining <= 3600 )) || remaining=3600 + timeout --signal=TERM --kill-after=60 "${remaining}s" \ + uv run --locked --extra dev python -m workflow_bench.diagnostic_trial --execute \ + --plan "$RUNNER_TEMP/release-evaluation/trial/plan.json" \ + --runtime "$GITHUB_WORKSPACE/candidate" \ + --claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" + exit 0 + fi arms=(baseline_nomcp baseline) if [[ -n "$INPUT_PR" ]]; then arms=(baseline); fi for runtime in stable candidate; do @@ -427,6 +492,8 @@ jobs: done # Descriptive only: its own output directory leaves the paired evidence # that release_report and release_gate read exactly as without the opt-in. + # Trials reject these opt-ins before startup. Preserve the exact gates + # recognized by the historical measurement-protocol validator. - name: Run the opt-in CLI-only arm on the candidate if: env.CLI_ARM == 'true' continue-on-error: true @@ -471,7 +538,7 @@ jobs: --claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \ --out "$RUNNER_TEMP/release-evaluation/candidate/deferred" - name: Summarize complete or incomplete measurements - if: ${{ always() && steps.resolve.outputs.sha != '' }} + if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }} working-directory: eval run: | if [[ -n "$INPUT_PR" ]]; then @@ -497,7 +564,7 @@ jobs: done exit "$failed" - name: Require complete paired evidence and no task regression - if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' }} + if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }} working-directory: eval run: | uv run --locked --extra dev python -m workflow_bench.release_gate \ @@ -507,7 +574,7 @@ jobs: --stable-sha '${{ steps.resolve.outputs.stable_sha }}' \ --out "$RUNNER_TEMP/release-evaluation/public" - name: Summarize matched MCP loading experiment - if: ${{ always() && inputs.mcp_tool_loading == true && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' }} + if: ${{ always() && inputs.mcp_tool_loading == true && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }} working-directory: eval run: | uv run --locked --extra dev python -m workflow_bench.mcp_loading_report \ @@ -516,7 +583,7 @@ jobs: --metadata "$RUNNER_TEMP/release-evaluation/candidate/metadata.json" \ --out "$RUNNER_TEMP/release-evaluation/public" - name: Collect sanitized evidence and publish summary - if: ${{ always() && steps.resolve.outputs.sha != '' }} + if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }} run: | set -euo pipefail public="$RUNNER_TEMP/release-evaluation/public" @@ -535,7 +602,7 @@ jobs: if [[ -f "$public/$report.md" ]]; then cat "$public/$report.md" >> "$GITHUB_STEP_SUMMARY"; fi done - name: Summarize the opt-in CLI-only arm - if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' }} + if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }} continue-on-error: true working-directory: eval run: | @@ -546,7 +613,7 @@ jobs: --out "$RUNNER_TEMP/release-evaluation/public" cat "$RUNNER_TEMP/release-evaluation/public/cli-arm-comparison.md" >> "$GITHUB_STEP_SUMMARY" - name: Summarize time, cost, and tool measurements - if: ${{ always() && steps.resolve.outputs.sha != '' }} + if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }} working-directory: eval env: DEFERRED_ARM: ${{ inputs.mcp_tool_loading }} @@ -588,8 +655,40 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" done exit "$failed" + - name: Summarize the diagnostic trial + if: ${{ always() && inputs.diagnostic_trial == true && steps.resolve.outputs.sha != '' }} + working-directory: eval + run: | + set -euo pipefail + root="$RUNNER_TEMP/release-evaluation" + mkdir -p "$root/public" + printf "\n## Two-cell diagnostic trial\n\nOne task, one repetition per arm. \$5 CLI budget stop per cell; provider billing can exceed the planned \$10. This trial checks telemetry and is not release-quality evidence.\n" >> "$GITHUB_STEP_SUMMARY" + failed=0 + for cell in cell-0000 cell-0001; do + if ! uv run --locked --extra dev python -m workflow_bench.cost_diagnostics \ + --results "$root/trial/results/$cell/results.jsonl" \ + --provider-usage "$root/trial/results/$cell.provider.jsonl" --allow-incomplete \ + --out "$root/public/trial-$cell-diagnostics.json" \ + --markdown-out "$root/public/trial-$cell-diagnostics.md"; then + printf '\nMeasurements for %s could not be validated.\n' "$cell" >> "$GITHUB_STEP_SUMMARY" + failed=1 + continue + fi + cat "$root/public/trial-$cell-diagnostics.md" >> "$GITHUB_STEP_SUMMARY" + done + exit "$failed" + - name: Upload sanitized diagnostic trial measurements + if: ${{ always() && inputs.diagnostic_trial == true && steps.resolve.outputs.sha != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: diagnostic-trial-${{ steps.resolve.outputs.sha }}-${{ github.run_id }} + path: | + ${{ runner.temp }}/release-evaluation/public/trial-*-diagnostics.json + ${{ runner.temp }}/release-evaluation/public/trial-*-diagnostics.md + if-no-files-found: error + retention-days: 30 - name: Upload sanitized release evidence - if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' }} + if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-agent-evaluation-${{ steps.resolve.outputs.sha }} @@ -607,7 +706,7 @@ jobs: if-no-files-found: error retention-days: 30 - name: Upload the opt-in CLI-only arm comparison - if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' }} + if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: release-cli-arm-${{ steps.resolve.outputs.sha }} @@ -619,7 +718,7 @@ jobs: if-no-files-found: warn retention-days: 30 - name: Upload sanitized PR cost comparison - if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr != '' }} + if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr != '' && inputs.diagnostic_trial != true }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: pr-cost-comparison-${{ steps.resolve.outputs.sha }} diff --git a/eval/tests/test_diagnostic_trial.py b/eval/tests/test_diagnostic_trial.py new file mode 100644 index 000000000..097cc0d07 --- /dev/null +++ b/eval/tests/test_diagnostic_trial.py @@ -0,0 +1,237 @@ +"""The paid trial must stay at two bounded calls and publish private-safe results.""" + +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import time +from types import SimpleNamespace + +import pytest +import yaml + +from workflow_bench import diagnostic_experiments, diagnostic_trial +from workflow_bench.session_diagnostics import diagnose_session + +ROOT = Path(__file__).resolve().parents[2] +SHA = "a" * 40 + + +@pytest.fixture +def trial_plan(tmp_path, monkeypatch): + runtime = tmp_path / "candidate" + runtime.mkdir() + output = tmp_path / "release-evaluation" / "trial" + monkeypatch.setattr(sys, "argv", [ + "prepare", "--tasks", str(ROOT / "eval/workflow_bench/tasks.scenarios.yaml"), + "--repository", str(runtime), "--runtime", str(runtime), "--runtime-sha", SHA, + "--harness-sha", SHA, "--model", diagnostic_trial.MODEL, "--repeats", "1", + "--variants", "no_graph", "mcp_eager", "--task-ids", diagnostic_trial.TASK, + "--max-budget-usd", "5", "--out", str(output), + ]) + diagnostic_experiments.main() + monkeypatch.setattr(diagnostic_trial, "_head", lambda repository: SHA) + return output / "plan.json", runtime + + +def fake_runner(calls, *, codes=(0, 0), missing=None): + events = [ + {"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "one", "name": "Read", + "input": {"file_path": "PRIVATE"}}]}}, + {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "one", "content": "PRIVATE"}]}}, + ] + diagnostics = diagnose_session(events, [1, 2], 3) + + def run(command, *, cwd, env, check): + calls.append((command, env)) + output = Path(command[command.index("--out") + 1]) + output.mkdir(parents=True) + row = { + "task": diagnostic_trial.TASK, "arm": command[command.index("--arms") + 1], "run": 0, + "model": diagnostic_trial.MODEL, "cost_usd": 0.1, "resolved": True, "ok": True, + "session_diagnostics": [diagnostics] if missing != "diagnostics" else [], + } + if missing != "results": + (output / "results.jsonl").write_text(json.dumps(row) + "\n") + if missing != "ledger": + Path(env["GITNEXUS_BENCH_PROVIDER_USAGE"]).write_text(json.dumps({ + "status": "success", "provider": "openai", "response_id": "private-id", + "native_usage": {"prompt_tokens": 10, "completion_tokens": 2}, "prompt": "PRIVATE", + }) + "\n") + return SimpleNamespace(returncode=codes[len(calls) - 1]) + return run + + +@pytest.mark.parametrize("codes", [(0, 0), (1, 0), (0, 1)]) +def test_trial_executes_exactly_two_serial_budgeted_cells_and_retains_failures(trial_plan, monkeypatch, codes): + path, runtime = trial_plan + calls = [] + monkeypatch.setattr(diagnostic_trial.subprocess, "run", fake_runner(calls, codes=codes)) + assert diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") == int(any(codes)) + assert len(calls) == 2 + assert [command[command.index("--arms") + 1] for command, _ in calls] == ["baseline", "baseline_nomcp"] + for command, _ in calls: + for flag, value in (("--max-budget-usd", "5.0"), ("--workers", "1"), ("--runs", "1"), + ("--timeout", "900"), ("--model", diagnostic_trial.MODEL), ("--claude-bin", "/pinned/claude")): + assert command[command.index(flag) + 1] == value + assert command[:3] == [sys.executable, "-m", "workflow_bench.runner"] + assert len({env["GITNEXUS_BENCH_PROVIDER_USAGE"] for _, env in calls}) == 2 + receipt = json.loads((path.parent / "execution.json").read_text()) + assert [cell["exit_code"] for cell in receipt["cells"]] == list(codes) + assert all(cell["measurements_present"] for cell in receipt["cells"]) + assert receipt["status"] == ("failed" if any(codes) else "completed") + with pytest.raises(ValueError, match="repeat paid work"): + diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") + assert len(calls) == 2 + + +@pytest.mark.parametrize("field,value", [("model", "other"), ("workers", 2), ("repeats", 3), + ("max_budget_usd_per_cell", 10), ("planned_cli_budget_usd", 20), + ("release_gate_eligible", True), ("execution", "started"), ("seed", 1)]) +def test_policy_changes_fail_before_any_paid_process(trial_plan, monkeypatch, field, value): + path, runtime = trial_plan + plan = json.loads(path.read_text()) + plan[field] = value + path.write_text(json.dumps(plan)) + calls = [] + monkeypatch.setattr(diagnostic_trial.subprocess, "run", fake_runner(calls)) + with pytest.raises(ValueError): + diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") + assert calls == [] + + +@pytest.mark.parametrize("change", ["third_cell", "extra_argument", "manifest_hash", "extra_task", "ledger", "revision"]) +def test_cell_tampering_fails_before_any_paid_process(trial_plan, monkeypatch, change): + path, runtime = trial_plan + plan = json.loads(path.read_text()) + cell = plan["cells"][0] + manifest = path.parent / "cell-0000.tasks.json" + if change == "third_cell": + plan["cells"].append(cell.copy()) + elif change == "extra_argument": + cell["argv"] += ["--runs", "10"] + elif change == "manifest_hash": + manifest.write_text(manifest.read_text() + " ") + elif change == "extra_task": + document = json.loads(manifest.read_text()) + document["tasks"].append(document["tasks"][0].copy()) + manifest.write_text(json.dumps(document)) + cell["task_manifest_sha256"] = hashlib.sha256(manifest.read_bytes()).hexdigest() + elif change == "ledger": + cell["env"]["GITNEXUS_BENCH_PROVIDER_USAGE"] = "/tmp/reused-ledger" + else: + plan["runtime_sha"] = "b" * 40 + path.write_text(json.dumps(plan)) + calls = [] + monkeypatch.setattr(diagnostic_trial.subprocess, "run", fake_runner(calls)) + with pytest.raises(ValueError): + diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") + assert calls == [] + + +def test_interrupted_trial_cannot_be_reexecuted_without_a_new_plan(trial_plan, monkeypatch): + path, runtime = trial_plan + calls = [] + + def crash(*args, **kwargs): + calls.append(args) + raise OSError("synthetic launch failure") + + monkeypatch.setattr(diagnostic_trial.subprocess, "run", crash) + with pytest.raises(OSError): + diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") + with pytest.raises(FileExistsError): + diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") + assert len(calls) == 1 + + +@pytest.mark.parametrize("missing", ["results", "ledger", "diagnostics"]) +def test_missing_measurements_cannot_make_the_trial_green(trial_plan, monkeypatch, missing): + path, runtime = trial_plan + calls = [] + monkeypatch.setattr(diagnostic_trial.subprocess, "run", fake_runner(calls, missing=missing)) + assert diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") == 1 + assert len(calls) == 2 + + +def workflow(): + return yaml.safe_load((ROOT / ".github/workflows/release-evaluation.yml").read_text()) + + +@pytest.mark.parametrize("override", [{}, {"TRIAL": "false", "GITHUB_RUN_ATTEMPT": "2"}, + {"GITHUB_RUN_ATTEMPT": "2"}, {"GITHUB_EVENT_NAME": "schedule"}, + {"RUNNER_ONLY": "true"}, {"CLI_ARM": "true"}, {"DEFERRED_ARM": "true"}, + {"MODEL": "other"}, {"EFFORT": "max"}, {"CANDIDATE_REF": "latest-rc"}]) +def test_real_startup_guard_rejects_expanded_or_repeated_paid_trials(override): + step = workflow()["jobs"]["start-runner"]["steps"][0] + env = {**os.environ, "TRIAL": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_RUN_ATTEMPT": "1", + "RUNNER_ONLY": "false", "CLI_ARM": "false", "DEFERRED_ARM": "false", "CANDIDATE_REF": SHA, + "MODEL": diagnostic_trial.MODEL, "EFFORT": "xhigh", **override} + result = subprocess.run(["bash", "-c", step["run"]], env=env, capture_output=True) + assert (result.returncode == 0) == (not override or override.get("TRIAL") == "false") + + +@pytest.mark.parametrize("attempt", ["1", "2"]) +def test_real_paid_step_routes_trial_to_bounded_executor_without_full_sweep(tmp_path, attempt): + steps = workflow()["jobs"]["evaluate"]["steps"] + script = next(step["run"] for step in steps if step.get("name") == "Run repeated paired agent sessions") + script = re.sub(r"\$\{\{.*?\}\}", str(int(time.time()) + 7200), script) + binary = tmp_path / "uv" + binary.write_text(f"#!{sys.executable}\nimport json,os,sys\nwith open(os.environ['CAPTURE'],'a') as f: f.write(json.dumps(sys.argv[1:])+'\\n')\n") + binary.chmod(0o755) + capture = tmp_path / "calls.jsonl" + result = subprocess.run(["bash", "-c", script], capture_output=True, env={ + **os.environ, "DIAGNOSTIC_TRIAL": "true", "CAPTURE": str(capture), "PATH": f"{tmp_path}:{os.defpath}", + "RUNNER_TEMP": str(tmp_path), "GITHUB_WORKSPACE": str(tmp_path), "GITHUB_RUN_ATTEMPT": attempt, + }) + if attempt != "1": + assert result.returncode != 0 and not capture.exists() + return + assert result.returncode == 0 + calls = [json.loads(line) for line in capture.read_text().splitlines()] + assert len(calls) == 1 + assert "workflow_bench.diagnostic_trial" in calls[0] and "--execute" in calls[0] + assert "workflow_bench.runner" not in calls[0] + + +def test_trial_is_manual_only_and_cannot_publish_release_evidence(): + document = workflow() + assert document[True]["workflow_dispatch"]["inputs"]["diagnostic_trial"]["default"] is False + assert "diagnostic_trial" not in document[True]["workflow_call"]["inputs"] + steps = document["jobs"]["evaluate"]["steps"] + excluded = {"Summarize complete or incomplete measurements", "Require complete paired evidence and no task regression", + "Collect sanitized evidence and publish summary", "Summarize time, cost, and tool measurements", + "Upload sanitized release evidence", "Upload sanitized PR cost comparison"} + for step in steps: + if step.get("name") in excluded: + assert "inputs.diagnostic_trial != true" in step["if"] + upload = next(step for step in steps if step.get("name") == "Upload sanitized diagnostic trial measurements") + assert "always()" in upload["if"] and "inputs.diagnostic_trial == true" in upload["if"] + assert upload["with"]["name"].startswith("diagnostic-trial-") + assert all("/public/trial-*-diagnostics." in line for line in upload["with"]["path"].splitlines()) + assert "refs/heads/main" in document["jobs"]["evaluate"]["if"] + assert document["jobs"]["evaluate"]["environment"] == "gitnexus-evolution" + + +def test_real_trial_summary_publishes_both_cells_without_private_payloads(trial_plan, tmp_path, monkeypatch): + path, runtime = trial_plan + calls = [] + with monkeypatch.context() as scoped: + scoped.setattr(diagnostic_trial.subprocess, "run", fake_runner(calls)) + assert diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") == 0 + binary = tmp_path / "uv" + binary.write_text(f"#!{sys.executable}\nimport os,sys\na=sys.argv[1:]; os.execv(sys.executable,[sys.executable,*a[a.index('python')+1:]])\n") + binary.chmod(0o755) + summary = tmp_path / "summary.md" + step = next(step for step in workflow()["jobs"]["evaluate"]["steps"] if step.get("name") == "Summarize the diagnostic trial") + subprocess.run(["bash", "-c", step["run"]], check=True, cwd=ROOT / "eval", capture_output=True, env={ + **os.environ, "PATH": f"{tmp_path}:{os.defpath}", "RUNNER_TEMP": str(tmp_path), "GITHUB_STEP_SUMMARY": str(summary), + }) + body = summary.read_text() + assert "Two-cell diagnostic trial" in body and "planned $10" in body + assert body.count("Observed cells: 1") == 2 + assert "PRIVATE" not in body and "private-id" not in body + assert len(list((tmp_path / "release-evaluation/public").glob("trial-*-diagnostics.json"))) == 2 diff --git a/eval/tests/test_ec2_workflows.py b/eval/tests/test_ec2_workflows.py index 3bfe5a0ad..235f57181 100644 --- a/eval/tests/test_ec2_workflows.py +++ b/eval/tests/test_ec2_workflows.py @@ -322,6 +322,12 @@ def test_shared_lifecycle_jobs_stay_identical_across_both_ec2_workflows(): renamed = {"watch-evaluate-pickup": "watch-evolve-pickup", "evaluate": "evolve"} for job in ("start-runner", "check-runner", "runner-ready", "stop-runner"): expected = {key: value for key, value in evaluation[job].items() if key != "name"} + if job == "start-runner": + guard, *lifecycle = expected["steps"] + assert guard["name"] == "Validate diagnostic trial authorization scope" + assert "uses" not in guard and "secrets." not in json.dumps(guard) + # Input rejection precedes EC2 startup; compare every lifecycle step. + expected["steps"] = lifecycle if job != "stop-runner": preview = "(github.event_name == 'pull_request_target' && inputs.candidate_pr != '') ||\n " assert expected["if"].count(preview) == 1 diff --git a/eval/workflow_bench/README.md b/eval/workflow_bench/README.md index 3ca916555..54c544277 100644 --- a/eval/workflow_bench/README.md +++ b/eval/workflow_bench/README.md @@ -1030,6 +1030,28 @@ or successful task completion. The $10 planned total uses CLI accounting, not a provider invoice cap: an in-flight request or different provider accounting can exceed it. Preparing the plan does not authorize or execute the trial. +After authorizing those two paid cells, dispatch the trusted main workflow with +`diagnostic_trial=true` and the full reviewed candidate SHA: + +```bash +gh workflow run release-evaluation.yml --ref main \ + -f diagnostic_trial=true -f candidate_ref=FULL_CANDIDATE_SHA \ + -f model=gpt-5.6-sol -f effort=xhigh +``` + +This mode runs `trivial-status-json-alias` once per arm, serially, with a $5 +CLI-reported budget stop and a 900-second session timeout. It rejects additional +CLI/deferred arms, other models/efforts, runner-only mode, and repeat attempts. +The full stable/candidate sweep is skipped. The existing EC2 lock, containment +preflight, stop deadline, evidence reserve, and verified shutdown still apply. + +The Actions summary and `diagnostic-trial-*` artifact contain sanitized +measurements for both cells, including failures. Raw provider ledgers remain on +the runner. Trial artifacts cannot approve a release. Missing result rows, +session diagnostics, or provider ledgers fail the trial instead of producing a +green run without measurements. Review the recorded spend before authorizing a +new dispatch after failure; the executor never retries or resumes a used plan. + | Possible cause | Measurement | What still needs an experiment | | --- | --- | --- | | Missing, ambiguous or incomplete resolution | Tool outcomes, UNKNOWN risk, lower-bound/stale/truncated responses, next source actions | Verify the intended target exists at the indexed source revision | diff --git a/eval/workflow_bench/diagnostic_trial.py b/eval/workflow_bench/diagnostic_trial.py new file mode 100644 index 000000000..b46e95115 --- /dev/null +++ b/eval/workflow_bench/diagnostic_trial.py @@ -0,0 +1,122 @@ +"""Execute the fixed, explicitly authorized two-cell telemetry trial once.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +from typing import Any + +MODEL = "gpt-5.6-sol" +TASK = "trivial-status-json-alias" +BUDGET_USD = 5.0 +HARNESS_ROOT = Path(__file__).resolve().parents[2] + + +def _head(repository: Path) -> str: + return subprocess.run( + ["git", "-C", str(repository), "rev-parse", "HEAD"], + check=True, capture_output=True, text=True, timeout=60, + ).stdout.strip() + + +def _validated_cells(plan_path: Path, runtime: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: + plan = json.loads(plan_path.read_text()) + policy = { + "schema": "gitnexus.diagnostic-experiments/v1", "execution": "not_started", + "model": MODEL, "effort": "xhigh", "repeats": 1, "workers": 1, "seed": 3762, + "max_budget_usd_per_cell": BUDGET_USD, "planned_cli_budget_usd": 10.0, + "release_gate_eligible": False, + } + if not isinstance(plan, dict) or any(plan.get(key) != value for key, value in policy.items()): + raise ValueError("trial plan does not match the authorized two-cell policy") + if plan.get("harness_sha") != _head(HARNESS_ROOT) or plan.get("runtime_sha") != _head(runtime): + raise ValueError("trial checkout does not match its pinned revision") + cells = plan.get("cells") + if not isinstance(cells, list) or len(cells) != 2 or not all(isinstance(c, dict) for c in cells): + raise ValueError("trial must contain exactly two cells") + if {cell.get("variant") for cell in cells} != {"no_graph", "mcp_eager"}: + raise ValueError("trial requires one no-graph and one eager-MCP cell") + root = plan_path.parent + for index, cell in enumerate(cells): + cell_id = f"cell-{index:04d}" + arm = "baseline" if cell["variant"] == "mcp_eager" else "baseline_nomcp" + manifest = root / f"{cell_id}.tasks.json" + output = root / "results" / cell_id + ledger = root / "results" / f"{cell_id}.provider.jsonl" + expected = [ + "uv", "run", "--locked", "python", "-m", "workflow_bench.runner", + "--tasks", str(manifest), "--runs", "1", "--workers", "1", "--arms", arm, + "--mcp-tool-loading", "eager", "--gitnexus-root", str(runtime), + "--model", MODEL, "--effort", "xhigh", "--include-expensive", + "--max-budget-usd", str(BUDGET_USD), "--out", str(output), + ] + if (cell.get("cell_id") != cell_id or cell.get("task") != TASK or cell.get("argv") != expected + or cell.get("cwd") != str(HARNESS_ROOT / "eval") or cell.get("env") != { + "GITNEXUS_BENCH_SWEEP_ID": cell_id, "GITNEXUS_BENCH_PROVIDER_USAGE": str(ledger), + }): + raise ValueError("trial cell has unexpected arguments or paths") + if manifest.is_symlink() or hashlib.sha256(manifest.read_bytes()).hexdigest() != cell.get("task_manifest_sha256"): + raise ValueError("trial task manifest changed after preparation") + document = json.loads(manifest.read_text()) + tasks = document.get("tasks") if isinstance(document, dict) else None + if not isinstance(tasks, list) or len(tasks) != 1 or not isinstance(tasks[0], dict) or tasks[0].get("id") != TASK: + raise ValueError("trial manifest must contain only the authorized task") + if output.exists() or output.is_symlink() or ledger.exists() or ledger.is_symlink(): + raise ValueError("trial results already exist; refusing to repeat paid work") + return plan, cells + + +def execute_trial(plan_path: Path, runtime: Path, claude_bin: str) -> int: + plan_path, runtime = plan_path.resolve(), runtime.resolve() + plan, cells = _validated_cells(plan_path, runtime) + receipt = plan_path.parent / "execution.json" + state: dict[str, Any] = { + "schema": "gitnexus.diagnostic-trial-execution/v1", "status": "running", + "harness_sha": plan["harness_sha"], "runtime_sha": plan["runtime_sha"], + "planned_cli_budget_usd": 10.0, "budget_is_invoice_cap": False, "cells": [], + } + # A crash can happen after a paid request but before its result is saved. + # Never resume/retry this plan automatically, even when no result exists. + with receipt.open("x") as stream: + stream.write(json.dumps(state, indent=2) + "\n") + failed = False + for cell in cells: + command = [sys.executable, *cell["argv"][4:], "--timeout", "900", "--claude-bin", claude_bin] + item: dict[str, Any] = {"cell_id": cell["cell_id"], "variant": cell["variant"], "status": "started"} + state["cells"].append(item) + receipt.write_text(json.dumps(state, indent=2) + "\n") + completed = subprocess.run(command, cwd=cell["cwd"], env={**os.environ, **cell["env"]}, check=False) + output = plan_path.parent / "results" / cell["cell_id"] / "results.jsonl" + ledger = Path(cell["env"]["GITNEXUS_BENCH_PROVIDER_USAGE"]) + try: + rows = [json.loads(line) for line in output.read_text().splitlines() if line.strip()] + measured = (len(rows) == 1 and isinstance(rows[0], dict) and rows[0].get("task") == TASK + and rows[0].get("arm") == cell["arm"] and bool(rows[0].get("session_diagnostics")) + and ledger.is_file() and ledger.stat().st_size > 0) + except (OSError, ValueError): + measured = False + item.update(status="finished", exit_code=completed.returncode, measurements_present=measured) + failed |= completed.returncode != 0 or not measured + receipt.write_text(json.dumps(state, indent=2) + "\n") + state["status"] = "failed" if failed else "completed" + receipt.write_text(json.dumps(state, indent=2) + "\n") + return int(failed) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--execute", action="store_true", required=True, help="Authorize these two paid cells") + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--runtime", type=Path, required=True) + parser.add_argument("--claude-bin", required=True) + args = parser.parse_args() + raise SystemExit(execute_trial(args.plan, args.runtime, args.claude_bin)) + + +if __name__ == "__main__": + main() From 97ec905b5dadac9482e79a90bedcf2c2971b4fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Fri, 30 Oct 2026 20:47:29 +0000 Subject: [PATCH 03/26] fix(storage): preserve fresh registry entries (#3777) Recheck prune candidates under the registry lock so stale inspections cannot remove a completed re-registration. Preserve transient errors and unreadable registry state. Refs #3515 Co-authored-by: Gergo Magyar --- gitnexus/src/storage/repo-manager.ts | 31 ++- ...o-manager-registry-validation-race.test.ts | 243 ++++++++++++++++-- 2 files changed, 246 insertions(+), 28 deletions(-) diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index c6a362bff..c3c731418 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -63,6 +63,7 @@ import { LIST_STORAGE_REQUIREMENTS, requireDeletableStoragePath, StorageDeletionError, + type StorageInspection, validateConfiguredStoragePath, } from './storage-resolver.js'; @@ -1745,6 +1746,12 @@ export const listRegisteredRepos = async (opts?: { const prunedSlots = new Set(); const registrySlotKey = (entry: RegistryEntry): string => `${canonicalizePath(entry.path)}\0${canonicalizePath(entry.storagePath)}`; + const isSafelyPrunable = (inspection: StorageInspection): boolean => + !isTransientStorageInspection(inspection) && + (inspection.state === 'missing' || + inspection.state === 'empty' || + (inspection.state === 'unowned' && + inspection.reason === 'Storage directory contains data but no valid ownership metadata.')); const inspections = await mapPool(entries, inspectRegisteredStorage, 8); for (const [entry, inspection] of entries.map( (entry, i) => [entry, inspections[i]] as [RegistryEntry, (typeof inspections)[number]], @@ -1754,12 +1761,7 @@ export const listRegisteredRepos = async (opts?: { (!LIST_STORAGE_REQUIREMENTS.requireCodeIndexDB || inspection.hasCodeIndexDB); if (meetsRequirements) { valid.push(entry); - } else if ( - inspection.state === 'missing' || - inspection.state === 'empty' || - (inspection.state === 'unowned' && - inspection.reason === 'Storage directory contains data but no valid ownership metadata.') - ) { + } else if (isSafelyPrunable(inspection)) { // A missing/empty directory or a registry slot with no ownership // metadata is not a usable index. Removing only the registry row is // safe; the storage directory itself is never deleted here. @@ -1786,15 +1788,22 @@ export const listRegisteredRepos = async (opts?: { // only then. The validation walk above is read-only and can touch several // files per entry, so holding the global lock across it would serialize // every `gitnexus augment` behind unrelated registry work for no benefit. - // Re-read inside the lock and drop only the same provably-absent storage - // slots from that fresh snapshot, so a concurrent re-registration of the - // same repository path into another slot survives. + // Reinspect only deletion candidates under the lock: even the same slot + // with identical metadata may have been rebuilt and re-registered since + // the initial inspection (#3515). Other slots keep the lock-free read path. if (prunedSlots.size > 0) { try { await withRegistryLock(async () => { - const fresh = await readRegistry(); + const fresh = await readRegistryStrictIfPresent(); + if (!fresh) return; + const candidates = fresh.filter((entry) => prunedSlots.has(registrySlotKey(entry))); + const rechecked = await mapPool(candidates, inspectRegisteredStorage, 8); + const confirmedSlots = new Set( + candidates.filter((_, i) => isSafelyPrunable(rechecked[i])).map(registrySlotKey), + ); + if (confirmedSlots.size === 0) return; await writeRegistry( - fresh.filter((entry) => !prunedSlots.has(registrySlotKey(entry))), + fresh.filter((entry) => !confirmedSlots.has(registrySlotKey(entry))), 1, ); }); diff --git a/gitnexus/test/unit/repo-manager-registry-validation-race.test.ts b/gitnexus/test/unit/repo-manager-registry-validation-race.test.ts index eec94d89c..4c97eb8a6 100644 --- a/gitnexus/test/unit/repo-manager-registry-validation-race.test.ts +++ b/gitnexus/test/unit/repo-manager-registry-validation-race.test.ts @@ -15,8 +15,10 @@ import { listRegisteredRepos, readRegistry, registerRepo, + type RegistryEntry, type RepoMeta, } from '../../src/storage/repo-manager.js'; +import type { StorageInspection } from '../../src/storage/storage-resolver.js'; import { createTempDir } from '../helpers/test-db.js'; describe('listRegisteredRepos validation persistence', () => { @@ -72,25 +74,232 @@ describe('listRegisteredRepos validation persistence', () => { const releaseInspectionPromise = new Promise((resolve) => { releaseInspection = resolve; }); - storageResolverCtx.inspectRegisteredStorage.mockImplementationOnce(async (entry: any) => { - inspectionStarted(); - await releaseInspectionPromise; - return { - repoPath: entry.path, - storagePath: entry.storagePath, - state: 'missing', - hasCodeIndexDB: false, - }; - }); + storageResolverCtx.inspectRegisteredStorage.mockImplementationOnce( + async (entry: RegistryEntry) => { + inspectionStarted(); + await releaseInspectionPromise; + return { + repoPath: entry.path, + storagePath: entry.storagePath, + state: 'missing', + hasCodeIndexDB: false, + }; + }, + ); const validatingRead = listRegisteredRepos({ validate: true }); - await inspectionStartedPromise; - await registerRepo(repoPath, meta, { name: 'repo', storagePath: replacementStoragePath }); - releaseInspection(); + try { + await inspectionStartedPromise; + await registerRepo(repoPath, meta, { name: 'repo', storagePath: replacementStoragePath }); + releaseInspection(); - await expect(validatingRead).resolves.toEqual([]); - await expect(readRegistry()).resolves.toMatchObject([ - { path: repoPath, storagePath: replacementStoragePath }, - ]); + await expect(validatingRead).resolves.toEqual([]); + await expect(readRegistry()).resolves.toMatchObject([ + { path: repoPath, storagePath: replacementStoragePath }, + ]); + } finally { + releaseInspection(); + await validatingRead.catch(() => undefined); + } + }); + + it.each(['updated', 'unchanged'] as const)( + 'keeps a concurrent same-slot re-registration with %s metadata', + async (metadata) => { + const repoPath = tmpRepo.dbPath; + const storagePath = path.join(repoPath, '.gitnexus'); + const meta: RepoMeta = { + repoPath, + storagePath, + lastCommit: 'initial-commit', + indexedAt: '2026-09-05T00:00:00.000Z', + stats: { files: 1, nodes: 1 }, + }; + await registerRepo(repoPath, meta, { name: 'repo', storagePath }); + + let inspectionStarted!: () => void; + let releaseInspection!: () => void; + const started = new Promise((resolve) => { + inspectionStarted = resolve; + }); + const released = new Promise((resolve) => { + releaseInspection = resolve; + }); + storageResolverCtx.inspectRegisteredStorage + .mockResolvedValue({ + repoPath, + storagePath, + state: 'owned', + hasCodeIndexDB: true, + } satisfies StorageInspection) + .mockImplementationOnce(async (): Promise => { + inspectionStarted(); + await released; + return { repoPath, storagePath, state: 'missing', hasCodeIndexDB: false }; + }); + + const validatingRead = listRegisteredRepos({ validate: true }); + try { + await started; + const replacement = + metadata === 'updated' + ? { ...meta, lastCommit: 'rebuilt-commit', indexedAt: '2026-09-06T00:00:00.000Z' } + : meta; + await registerRepo(repoPath, replacement, { name: 'repo', storagePath }); + releaseInspection(); + + await expect(validatingRead).resolves.toEqual([]); + await expect(readRegistry()).resolves.toMatchObject([ + { + path: repoPath, + storagePath, + lastCommit: replacement.lastCommit, + indexedAt: replacement.indexedAt, + }, + ]); + } finally { + releaseInspection(); + await validatingRead.catch(() => undefined); + } + }, + ); + + it('still persists pruning when the registered storage remains missing', async () => { + const repoPath = tmpRepo.dbPath; + const storagePath = path.join(repoPath, '.gitnexus'); + await registerRepo(repoPath, { + repoPath, + lastCommit: 'initial-commit', + indexedAt: '2026-09-05T00:00:00.000Z', + stats: { files: 1, nodes: 1 }, + }); + storageResolverCtx.inspectRegisteredStorage.mockResolvedValue({ + repoPath, + storagePath, + state: 'missing', + hasCodeIndexDB: false, + } satisfies StorageInspection); + + await expect(listRegisteredRepos({ validate: true })).resolves.toEqual([]); + await expect(readRegistry()).resolves.toEqual([]); + }); + + it.each([ + { label: 'owned', state: 'owned', hasCodeIndexDB: true }, + { + label: 'temporarily unreadable', + state: 'invalid_storage', + hasCodeIndexDB: false, + reason: 'EIO: Storage directory could not be inspected.', + }, + { + label: 'empty with a transient database error', + state: 'empty', + hasCodeIndexDB: false, + reason: 'EIO: LadybugDB directory could not be inspected.', + }, + ] satisfies Array< + Pick & { label: string } + >)( + 'does not rewrite the registry when the candidate becomes $label', + async ({ label: _label, ...inspection }) => { + const repoPath = tmpRepo.dbPath; + const storagePath = path.join(repoPath, '.gitnexus'); + const registryPath = path.join(tmpHome.dbPath, 'registry.json'); + const original = JSON.stringify([{ name: 'repo', path: repoPath, storagePath }]); + // Compact formatting makes an otherwise identical atomic rewrite visible. + await fs.writeFile(registryPath, original); + storageResolverCtx.inspectRegisteredStorage + .mockResolvedValue({ repoPath, storagePath, ...inspection } satisfies StorageInspection) + .mockResolvedValueOnce({ + repoPath, + storagePath, + state: 'missing', + hasCodeIndexDB: false, + } satisfies StorageInspection); + + await expect(listRegisteredRepos({ validate: true })).resolves.toEqual([]); + expect(await fs.readFile(registryPath, 'utf8')).toBe(original); + }, + ); + + it('keeps an initially empty slot when its database inspection failed transiently', async () => { + const repoPath = tmpRepo.dbPath; + const storagePath = path.join(repoPath, '.gitnexus'); + const registryPath = path.join(tmpHome.dbPath, 'registry.json'); + const entry = { name: 'repo', path: repoPath, storagePath }; + const original = JSON.stringify([entry]); + await fs.writeFile(registryPath, original); + storageResolverCtx.inspectRegisteredStorage.mockResolvedValue({ + repoPath, + storagePath, + state: 'empty', + hasCodeIndexDB: false, + reason: 'EIO: LadybugDB directory could not be inspected.', + } satisfies StorageInspection); + + await expect(listRegisteredRepos({ validate: true })).resolves.toEqual([entry]); + expect(await fs.readFile(registryPath, 'utf8')).toBe(original); + }); + + it.each(['corrupt', 'deleted'] as const)( + 'does not overwrite a registry that becomes %s during validation', + async (change) => { + const repoPath = tmpRepo.dbPath; + const storagePath = path.join(repoPath, '.gitnexus'); + const registryPath = path.join(tmpHome.dbPath, 'registry.json'); + const corrupt = '{"truncated":'; + await fs.writeFile( + registryPath, + JSON.stringify([{ name: 'repo', path: repoPath, storagePath }]), + ); + storageResolverCtx.inspectRegisteredStorage.mockImplementation( + async (): Promise => { + if (change === 'corrupt') await fs.writeFile(registryPath, corrupt); + else await fs.rm(registryPath, { force: true }); + return { repoPath, storagePath, state: 'missing', hasCodeIndexDB: false }; + }, + ); + + await expect(listRegisteredRepos({ validate: true })).resolves.toEqual([]); + if (change === 'corrupt') expect(await fs.readFile(registryPath, 'utf8')).toBe(corrupt); + else await expect(fs.access(registryPath)).rejects.toMatchObject({ code: 'ENOENT' }); + }, + ); + + it('preserves an unrelated registration completed during validation', async () => { + const repoPath = tmpRepo.dbPath; + const storagePath = path.join(repoPath, '.gitnexus'); + const addedRepoPath = path.join(tmpHome.dbPath, 'new-repo'); + const registryPath = path.join(tmpHome.dbPath, 'registry.json'); + await fs.writeFile( + registryPath, + JSON.stringify([{ name: 'gone', path: repoPath, storagePath }]), + ); + storageResolverCtx.inspectRegisteredStorage + .mockResolvedValue({ + repoPath, + storagePath, + state: 'missing', + hasCodeIndexDB: false, + } satisfies StorageInspection) + .mockImplementationOnce(async (): Promise => { + await registerRepo( + addedRepoPath, + { + repoPath: addedRepoPath, + lastCommit: 'new-commit', + indexedAt: '2026-09-06T00:00:00.000Z', + stats: { files: 1, nodes: 1 }, + }, + { name: 'new-repo' }, + ); + return { repoPath, storagePath, state: 'missing', hasCodeIndexDB: false }; + }); + + await expect(listRegisteredRepos({ validate: true })).resolves.toEqual([]); + const persisted = await readRegistry(); + expect(persisted).toHaveLength(1); + expect(persisted[0]).toMatchObject({ name: 'new-repo', path: addedRepoPath }); }); }); From bd9586e008c6a2dbcd2f15598a3a20c3f2f44f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Fri, 30 Oct 2026 21:06:08 +0000 Subject: [PATCH 04/26] fix: recover duplicate candidates and expose test inventory gaps (#3743) * fix: recover missed test scenarios Run the full core test inventory and retrieve identical callback bodies across setup contexts. Separate assertion evidence from context and retain conservative input, path and runtime provenance checks. Restore prepared test inputs and report missing runtime collection. Add the audited recall corpus and inventory regressions. * Address PR review feedback (#3743) Correct counted nouns in the inventory report, document fully skipped modules, and include unhandled runner errors in the incomplete-collection warning. * Address PR review feedback (#3743) Force-stage explicit temporary fixtures despite inherited Git ignores, and isolate recorded-file collection-error accounting in the inventory tests. * fix(review): simplify selection and guard mixed artifacts * feat(test): detect fixture consolidation opportunities Capture source inputs used by integration tests and compare fixture clones, executed blocks, and assertion subjects and values. Report partial overlaps with the unique checks and fixture spans that consolidation must preserve. Keep cross-language structural suggestions separate from observed equality. Use pinned jscpd with direct pair verification when longer representatives hide partial clones. Retain capture gaps and native/worker limitations. Validate with 1,245 detector regressions, 28 pipeline pair controls, and 12 manually labeled comparisons. Scan 30,698 core cases and find 3,111 opportunity groups; exclude the capture's 103 failed and 89 skipped cases. * Fix scenario-map review findings (#3743) Resolve assertion aliases by lexical binding and normalize duplicate static object keys. Capture fixture files through checked descriptors with bounded reads and guaranteed cleanup. * fix: address test duplicate audit false signals * refactor(test): reuse assertion binding analysis Reuse lexical bindings within each evidence extraction while retaining assertion-specific offsets and expansion limits. Deduplicate fixture coverage endpoints numerically without string conversion. Preserve complete assertion evidence across 29,286 static instances and weighted coverage across 10,000 generated interval cases. * Fix scenario-map review findings (#3743) Bind consolidation evidence to the assertion call accepted by the check extractor, so unsupported same-line calls cannot supply incorrect subjects or expectations. Keep source provenance private and serialized checks intact. Exercise omitted plugins alongside instrumented Vitest runs. * fix: retain input-shaped scenario controls (#3743) Compare assertion subject structure separately from exact input-bearing assertion keys, so changing literal inputs retains table-row and boundary classification. Preserve selectors, callee identity and regex values. Keep the control expectations unchanged. Add regressions for literal inputs, queried subjects and nested regex arguments; bump the fingerprint engine version so reclassified findings resurface. * test: give scenario cases unique titles Include the parameterized case index so repeated input expressions and truncated audit paths cannot collide in the CI execution inventory. * fix: scope assertion subjects to their bindings Prevent same-named declarations in different files or scopes from becoming exact assertion matches. Preserve same-binding consolidation and advance the fingerprint engine to version 5. * fix: raise full-suite planning heap budget The 30,953-case replay aborts at the default 4 GiB heap. Give only the Plan step 8 GiB and document the same local invocation. The identical four coverage slices complete in 4m34s at 8.2 GiB peak RSS, retaining all 116,409 candidate pairs. * fix: stream full-suite scenario reports The full inventory produces a 663 MiB report, exceeding V8's single-string limit. Write records incrementally and stream only required fields when reading baselines and publishing issue updates. Replay preserves all 116,398 pairs from 30,953 collected cases. The report remains partial when the trace run fails. * fix: preserve test lifecycles during capture Forced test repeats changed cache and file state. Capture ordinary runs, retain runnable siblings during tracing, and keep retry-bearing files inconclusive. Preserve failure details and mark failed captures partial. Use the original JSON serializer so test mocks cannot break fixture reads. * fix: limit trace capture to candidates Keep sibling tests in both passes for suite state, while recording only candidate task IDs. This prevents unrelated timing checks and large corpus tests from failing under tracing. * ci: budget full-suite trace collection * fix: address scenario-map feedback (#3743) --------- Co-authored-by: Gergo Magyar --- .github/workflows/test-scenario-map.yml | 19 +- gitnexus/package-lock.json | 147 ++++ gitnexus/package.json | 2 + .../mutation-scenario-map/expected.json | 4 +- .../order-dependent.case.ts | 4 +- .../fixtures/scenario-map-recall/audit.json | 566 +++++++++++++ .../scenario-map-consolidation-inputs.test.ts | 65 ++ .../scenario-map-runtime-inputs.test.ts | 49 ++ .../test/unit/scenario-map/candidates.test.ts | 18 +- gitnexus/test/unit/scenario-map/cli.test.ts | 121 ++- .../unit/scenario-map/consolidation.test.ts | 750 ++++++++++++++++++ .../scenario-map/coverage-inventory.test.ts | 76 ++ .../scenario-map/coverage-records.test.ts | 17 + .../unit/scenario-map/fingerprint.test.ts | 230 ++++++ .../scenario-map/fixture-similarity.test.ts | 365 +++++++++ .../test/unit/scenario-map/partitions.test.ts | 70 ++ .../unit/scenario-map/recall-audit.test.ts | 53 ++ .../unit/scenario-map/report-json.test.ts | 124 +++ .../test/unit/scenario-map/report.test.ts | 2 +- .../scenario-map/runtime-fixtures.test.ts | 430 ++++++++++ .../unit/scenario-map/trace-setup.test.ts | 41 + gitnexus/test/unit/scenario-map/trace.test.ts | 104 +++ .../test/unit/scenario-map/vitest-run.test.ts | 32 + gitnexus/tools/scenario-map/README.md | 226 ++++++ .../scenario-map/consolidation/assertions.ts | 419 ++++++++++ .../consolidation/fixture-clones.ts | 533 +++++++++++++ .../consolidation/fixture-sources.ts | 27 + .../scenario-map/consolidation/report.ts | 391 +++++++++ .../tools/scenario-map/consolidation/run.ts | 27 + .../consolidation/verify-controls.ts | 87 ++ gitnexus/tools/scenario-map/facet/body-key.ts | 8 + gitnexus/tools/scenario-map/facet/checks.ts | 28 +- .../scenario-map/facet/fixture-inputs.ts | 313 ++++++++ .../scenario-map/facet/fixture-profile.ts | 318 ++++++++ .../scenario-map/facet/runtime-fixtures.ts | 122 +++ .../inventory/coverage-inventory.ts | 60 ++ .../tools/scenario-map/mutation/vitest-run.ts | 7 +- .../scenario-map/path-coverage/candidates.ts | 100 ++- .../path-coverage/coverage.setup.ts | 39 +- .../path-coverage/fixture-capture.ts | 269 +++++++ .../path-coverage/fixture-plugin.ts | 53 ++ .../scenario-map/path-coverage/partitions.ts | 63 +- .../scenario-map/path-coverage/prefilter.ts | 39 +- .../scenario-map/path-coverage/records.ts | 11 +- .../path-coverage/run-coverage.ts | 62 +- .../scenario-map/path-coverage/slices.ts | 8 + gitnexus/tools/scenario-map/report/json.ts | 79 ++ .../tools/scenario-map/report/markdown.ts | 31 +- gitnexus/tools/scenario-map/report/report.ts | 10 +- gitnexus/tools/scenario-map/scenario-map.ts | 174 +++- .../scenario-map/value-flow/trace-fixtures.ts | 26 + .../scenario-map/value-flow/trace-records.ts | 33 +- .../scenario-map/value-flow/trace.setup.ts | 19 +- .../tools/scenario-map/value-flow/trace.ts | 40 +- .../tools/scenario-map/verdict/fingerprint.ts | 123 ++- .../tools/scenario-map/verdict/verdict.ts | 6 + 56 files changed, 6850 insertions(+), 190 deletions(-) create mode 100644 gitnexus/test/fixtures/scenario-map-recall/audit.json create mode 100644 gitnexus/test/integration/scenario-map-consolidation-inputs.test.ts create mode 100644 gitnexus/test/integration/scenario-map-runtime-inputs.test.ts create mode 100644 gitnexus/test/unit/scenario-map/consolidation.test.ts create mode 100644 gitnexus/test/unit/scenario-map/coverage-inventory.test.ts create mode 100644 gitnexus/test/unit/scenario-map/fixture-similarity.test.ts create mode 100644 gitnexus/test/unit/scenario-map/recall-audit.test.ts create mode 100644 gitnexus/test/unit/scenario-map/report-json.test.ts create mode 100644 gitnexus/test/unit/scenario-map/runtime-fixtures.test.ts create mode 100644 gitnexus/test/unit/scenario-map/trace-setup.test.ts create mode 100644 gitnexus/tools/scenario-map/README.md create mode 100644 gitnexus/tools/scenario-map/consolidation/assertions.ts create mode 100644 gitnexus/tools/scenario-map/consolidation/fixture-clones.ts create mode 100644 gitnexus/tools/scenario-map/consolidation/fixture-sources.ts create mode 100644 gitnexus/tools/scenario-map/consolidation/report.ts create mode 100644 gitnexus/tools/scenario-map/consolidation/run.ts create mode 100644 gitnexus/tools/scenario-map/consolidation/verify-controls.ts create mode 100644 gitnexus/tools/scenario-map/facet/body-key.ts create mode 100644 gitnexus/tools/scenario-map/facet/fixture-inputs.ts create mode 100644 gitnexus/tools/scenario-map/facet/fixture-profile.ts create mode 100644 gitnexus/tools/scenario-map/facet/runtime-fixtures.ts create mode 100644 gitnexus/tools/scenario-map/inventory/coverage-inventory.ts create mode 100644 gitnexus/tools/scenario-map/path-coverage/fixture-capture.ts create mode 100644 gitnexus/tools/scenario-map/path-coverage/fixture-plugin.ts create mode 100644 gitnexus/tools/scenario-map/report/json.ts create mode 100644 gitnexus/tools/scenario-map/value-flow/trace-fixtures.ts diff --git a/.github/workflows/test-scenario-map.yml b/.github/workflows/test-scenario-map.yml index d3fa58bc9..1e9445d92 100644 --- a/.github/workflows/test-scenario-map.yml +++ b/.github/workflows/test-scenario-map.yml @@ -226,6 +226,9 @@ jobs: ls "$dest" - name: Plan the run env: + # Full-suite Layer 2 exceeds Node's default ~4 GiB heap. Budget 8 GiB + # for this step, leaving room on the 16 GB runner for native memory. + NODE_OPTIONS: --max-old-space-size=8192 SEED: ${{ inputs.seed || github.run_id }} SHARDS: ${{ inputs.shards }} MUTATION: ${{ steps.scope.outputs.mutation }} @@ -246,6 +249,9 @@ jobs: ${{ runner.temp }}/scenario-map/scenario-run-manifest.json ${{ runner.temp }}/scenario-map/layer0.json ${{ runner.temp }}/scenario-map/layer2.json + ${{ runner.temp }}/scenario-map/consolidation.json + ${{ runner.temp }}/scenario-map/consolidation.html + ${{ runner.temp }}/scenario-map/fixtures.json ${{ runner.temp }}/scenario-map/coverage/coverage.json if-no-files-found: error retention-days: 14 @@ -261,9 +267,9 @@ jobs: needs: plan if: ${{ !cancelled() && needs.plan.outputs.artifact != '' }} runs-on: ubuntu-latest - # Locally: PDG index about 3 min (peak RSS about 8.4 GB of the runner's 16 GB), the whole - # stage about 5 min. - timeout-minutes: 60 + # Full-suite tracing runs candidate files twice, including their runnable siblings. + # The 31k-case inventory exceeded 50 minutes; leave room for both passes and uploads. + timeout-minutes: 100 continue-on-error: true permissions: contents: read @@ -283,7 +289,7 @@ jobs: name: ${{ needs.plan.outputs.artifact }} path: ${{ runner.temp }}/scenario-map - name: Run value flow - timeout-minutes: 50 + timeout-minutes: 90 env: GITNEXUS_WORKER_READY_TIMEOUT_MS: '120000' run: | @@ -660,13 +666,14 @@ jobs: import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import { shouldUpdateIssue } from './tools/scenario-map/report/issue-body.ts'; + import { readReportFields } from './tools/scenario-map/report/json.ts'; import { completenessLine, degradedLine, runLink } from './tools/scenario-map/report/markdown.ts'; const { ISSUE_NUMBER, GITHUB_OUTPUT, RUNNER_TEMP } = process.env; const reportDir = path.join(RUNNER_TEMP, 'scenario-map-report'); const issueDir = path.join(RUNNER_TEMP, 'scenario-map-issue'); - const report = JSON.parse( - readFileSync(path.join(reportDir, 'scenario-map-report.json'), 'utf8'), + const report = await readReportFields( + path.join(reportDir, 'scenario-map-report.json'), ['run', 'completeness'], ); const existing = ISSUE_NUMBER ? readFileSync(path.join(issueDir, 'existing-body.md'), 'utf8') diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index bdbf1d277..37ee5799a 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -73,6 +73,7 @@ "@babel/traverse": "^8.0.0", "@babel/types": "^8.0.0", "@mermaid-js/tiny": "12.2.0", + "@streamparser/json": "0.0.26", "@stryker-mutator/api": "10.0.0", "@stryker-mutator/core": "10.0.0", "@types/busboy": "^1.5.4", @@ -83,6 +84,7 @@ "@types/proxy-addr": "^2.0.3", "@vitest/coverage-v8": "^5.0.0", "gitnexus-shared": "file:../gitnexus-shared", + "jscpd": "5.4.2", "jsdom": "30.1.3", "tsx": "^4.0.0", "typescript": "^7.0.2", @@ -2230,6 +2232,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@streamparser/json": { + "version": "0.0.26", + "resolved": "https://registry.npmjs.org/@streamparser/json/-/json-0.0.26.tgz", + "integrity": "sha512-46597LNFI+MFdUnzX2QJWwmdTRdq0XVD+vVNJTtGVzIrnCuhG9pFo1OAzbNBqci8UJgk/X5KJZ6LcV+y7PTuDQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@stryker-mutator/api": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/@stryker-mutator/api/-/api-10.0.0.tgz", @@ -4792,6 +4801,144 @@ "js-yaml": "bin/js-yaml.mjs" } }, + "node_modules/jscpd": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/jscpd/-/jscpd-5.4.2.tgz", + "integrity": "sha512-bxZ1ObA4NyWBcM7Zh+QoGrdcMovbCxrDofFaLo2Lwgk/Axe/e45VYu8EM+mpPx2CpU/QGCoktONRnaRZl8Yq2w==", + "dev": true, + "license": "MIT", + "bin": { + "jscpd": "run-jscpd.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://opencollective.com/jscpd" + }, + "optionalDependencies": { + "jscpd-darwin-arm64": "5.4.2", + "jscpd-darwin-x64": "5.4.2", + "jscpd-linux-arm64-gnu": "5.4.2", + "jscpd-linux-arm64-musl": "5.4.2", + "jscpd-linux-x64-gnu": "5.4.2", + "jscpd-linux-x64-musl": "5.4.2", + "jscpd-windows-arm64-msvc": "5.4.2", + "jscpd-windows-x64-msvc": "5.4.2" + } + }, + "node_modules/jscpd-darwin-arm64": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/jscpd-darwin-arm64/-/jscpd-darwin-arm64-5.4.2.tgz", + "integrity": "sha512-EGoPYQZqESW2kJfxKZrUYsYWhzq2Irk6sBQQWZ7VPAGj2wwk3n8HtDP2FnJOiHWZtSz55I8zQWldzIrD55B5Vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/jscpd-darwin-x64": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/jscpd-darwin-x64/-/jscpd-darwin-x64-5.4.2.tgz", + "integrity": "sha512-/bz/rgoahqeK0p5r+KjeTTS59d7bZpMxD7hQ4CaQ0qKZWBZIyu7hRdExzJ7/KrCEJ5cmRogvXX1jhFqldYLNXw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/jscpd-linux-arm64-gnu": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/jscpd-linux-arm64-gnu/-/jscpd-linux-arm64-gnu-5.4.2.tgz", + "integrity": "sha512-YW/eX+mOYZmT/A3QEARYmJQoVQNV/cfecKCobNXZcD3mE57drpKHmJnvuL9uS85z0cUKWGCM57JVkVz6Y9oxxw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/jscpd-linux-arm64-musl": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/jscpd-linux-arm64-musl/-/jscpd-linux-arm64-musl-5.4.2.tgz", + "integrity": "sha512-jOLTO8YVRz65NaeYNeXoIf13yOPWhbM61xKXpv51DXQ2BXce6XfgPaJVeDCOM5s+8mt6aDtq0BZ8zptNeqGuIw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/jscpd-linux-x64-gnu": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/jscpd-linux-x64-gnu/-/jscpd-linux-x64-gnu-5.4.2.tgz", + "integrity": "sha512-eAmKLxx+pIOEX4lZ342CCCjL5bvf+a3gdmaaVHXiGIx2Zr3zKkF32YkfuN2sCvCui5I/wnely80XsRvVVFO47Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/jscpd-linux-x64-musl": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/jscpd-linux-x64-musl/-/jscpd-linux-x64-musl-5.4.2.tgz", + "integrity": "sha512-ers9YFscD6DrmhyjBDXo/L+yBKcclHdWTFLMulYVGLcoGRKfBFhmK08MI3ajN/GMmCpJu7rkDYKL3egdfE8OIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/jscpd-windows-arm64-msvc": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/jscpd-windows-arm64-msvc/-/jscpd-windows-arm64-msvc-5.4.2.tgz", + "integrity": "sha512-kOhVw4cVhN8T9jr3OVThB3pPpbff0ITHTwpJ/+4EGzMkbf0y6QtPpHDzRgiHJj7u1FSTBXTI6VhPhDUhT/os1g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/jscpd-windows-x64-msvc": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/jscpd-windows-x64-msvc/-/jscpd-windows-x64-msvc-5.4.2.tgz", + "integrity": "sha512-BuR5DCV7F/G5ukMEIzCrz43ea5C0t0ALcP7v+opOWefoi4aiQYCEmtoRwfXMM/zjOmNYmvJBDDfeH05U9uuzEg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/jsdom": { "version": "30.1.3", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-30.1.3.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 2abadf380..1289c1c00 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -137,6 +137,7 @@ "@babel/traverse": "^8.0.0", "@babel/types": "^8.0.0", "@mermaid-js/tiny": "12.2.0", + "@streamparser/json": "0.0.26", "@stryker-mutator/api": "10.0.0", "@stryker-mutator/core": "10.0.0", "@types/busboy": "^1.5.4", @@ -147,6 +148,7 @@ "@types/proxy-addr": "^2.0.3", "@vitest/coverage-v8": "^5.0.0", "gitnexus-shared": "file:../gitnexus-shared", + "jscpd": "5.4.2", "jsdom": "30.1.3", "tsx": "^4.0.0", "typescript": "^7.0.2", diff --git a/gitnexus/test/fixtures/mutation-scenario-map/expected.json b/gitnexus/test/fixtures/mutation-scenario-map/expected.json index a0b535607..061631521 100644 --- a/gitnexus/test/fixtures/mutation-scenario-map/expected.json +++ b/gitnexus/test/fixtures/mutation-scenario-map/expected.json @@ -302,8 +302,8 @@ "verdict": "inconclusive", "rule": "isolation-disagrees", "fingerprint": { - "verdict": "inconclusive", - "rule": "not-passing" + "verdict": "table-row", + "rule": "input-values-differ" }, "stages": { "facet": "equal", diff --git a/gitnexus/test/fixtures/mutation-scenario-map/order-dependent.case.ts b/gitnexus/test/fixtures/mutation-scenario-map/order-dependent.case.ts index 8a184da01..2a1b9559c 100644 --- a/gitnexus/test/fixtures/mutation-scenario-map/order-dependent.case.ts +++ b/gitnexus/test/fixtures/mutation-scenario-map/order-dependent.case.ts @@ -1,7 +1,7 @@ // Scenario-map controls fixture (Phase 1 U17), special pair `order-dependent` (design §5.4 full // isolation pass, KTD9): the second test relies on the counter the first test leaves behind. In -// file order both pass; run alone, the second gets `beta-1` and fails. Expected: Inconclusive -// (expected.json). +// file order both pass; run alone, the second gets `beta-1` and fails. The fingerprint sees +// different row values; mutation remains Inconclusive on isolation (expected.json). import { describe, expect, it } from 'vitest'; import { nextTicketLabel } from './src/order-dependent.js'; diff --git a/gitnexus/test/fixtures/scenario-map-recall/audit.json b/gitnexus/test/fixtures/scenario-map-recall/audit.json new file mode 100644 index 000000000..9a96d7f82 --- /dev/null +++ b/gitnexus/test/fixtures/scenario-map-recall/audit.json @@ -0,0 +1,566 @@ +{ + "revision": "f354e66ff4a9515c8e4b33c4d39a94fc56846444", + "method": "Independently generated callback candidates; source review of setup, helper targets, fixture paths, inputs and assertions. Hand-adjudicated subset, not an exhaustive semantic oracle.", + "positives": [ + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Java > detects User class with save and getName methods", + "b": "test/integration/resolvers/java.test.ts#Java cross-file binding propagation > detects User class with save and getName methods", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Kotlin > detects User class with save and getName methods", + "b": "test/integration/resolvers/kotlin.test.ts#Kotlin cross-file binding propagation > detects User class with save and getName methods", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: JavaScript > detects User class with save and getName methods", + "b": "test/integration/resolvers/javascript.test.ts#JavaScript cross-file binding propagation > detects User class with save and getName methods", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: PHP > detects User class with save and getName methods", + "b": "test/integration/resolvers/php.test.ts#PHP cross-file binding propagation > detects User class with save and getName methods", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Rust > resolves u.save() in process() to User#save via cross-file return type", + "b": "test/integration/resolvers/rust.test.ts#Rust cross-file binding propagation > resolves u.save() in process() to User#save via cross-file return type propagation", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Ruby > resolves user.save in process() to User#save via cross-file return type", + "b": "test/integration/resolvers/ruby.test.ts#Ruby cross-file binding propagation > resolves user.save in process to User#save via cross-file propagation", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/resolvers/rust-scope.test.ts#Rust scope: impl block method classification > classifies save as a function owned by User (HAS_METHOD edge)", + "b": "test/integration/resolvers/rust-scope.test.ts#Rust scope: impl block method classification > emits HAS_METHOD edge from User to save", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Kotlin > resolves u.save() in run() to User#save via cross-file return type", + "b": "test/integration/resolvers/kotlin.test.ts#Kotlin cross-file binding propagation > resolves u.save() in run() to User#save via cross-file return type propagation", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Python > resolves u.save() in run() to User#save via cross-file return type", + "b": "test/integration/resolvers/python.test.ts#Python cross-file binding propagation > resolves u.save() in run() to User#save via cross-file return type propagation", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: JavaScript > resolves u.save() in run() to User#save via cross-file return type", + "b": "test/integration/resolvers/javascript.test.ts#JavaScript cross-file binding propagation > resolves u.save() in run() to User#save via cross-file return type propagation", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: C++ > detects User class with save and get_name methods", + "b": "test/integration/resolvers/cpp.test.ts#C++ cross-file binding propagation > detects User class with save and get_name methods", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Ruby > detects User class with save and get_name methods", + "b": "test/integration/resolvers/ruby.test.ts#Ruby cross-file binding propagation > detects User class with save and get_name methods", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > bare [] prefix with spaces only \u2192 undefined", + "b": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > empty array suffix \u2192 undefined", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > bare [] prefix with spaces only \u2192 undefined", + "b": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > [] suffix with no base \u2192 undefined", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > bare [] prefix with spaces only \u2192 undefined", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > returns undefined for just brackets: []", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > empty array suffix \u2192 undefined", + "b": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > [] suffix with no base \u2192 undefined", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > empty array suffix \u2192 undefined", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > returns undefined for just brackets: []", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > [] suffix with no base \u2192 undefined", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > returns undefined for just brackets: []", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Rust > detects get_user function and process function", + "b": "test/integration/resolvers/rust.test.ts#Rust cross-file binding propagation > detects get_user and process functions", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/resolvers/java.test.ts#Java same-arity overload cross-file and chain resolution > crossFileByName() emits exactly one CALLS edge to find(String) in DbLookup", + "b": "test/integration/resolvers/java.test.ts#Java same-arity overloads (worker path) > crossFileByName() \u2192 find(String) \u2014 sequential path", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/resolvers/java.test.ts#Java same-arity overload cross-file and chain resolution > emits METHOD_IMPLEMENTS from DbLookup.find(int) \u2192 ILookup.find(int)", + "b": "test/integration/resolvers/java.test.ts#Java same-arity overloads (worker path) > METHOD_IMPLEMENTS edges match interface methods \u2014 sequential path", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/unit/pdg-mode-flip.test.ts#pdgModeMismatch \u2014 M1\u2192M2 stamp upgrade (#2082 M2, pure) > an identical resolved M2 config compares equal (steady state keeps incremental)", + "b": "test/unit/pdg-mode-flip.test.ts#pdgModeMismatch \u2014 M2\u2192M3 stamp upgrade (#2083 M3 U5, pure) > an identical resolved M3 config compares equal (steady state keeps incremental)", + "category": "import-initialization", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/unit/pdg-mode-flip.test.ts#pdgModeMismatch \u2014 M1\u2192M2 stamp upgrade (#2082 M2, pure) > an identical resolved M2 config compares equal (steady state keeps incremental)", + "b": "test/unit/pdg-mode-flip.test.ts#pdgModeMismatch \u2014 pre-#2201\u2192SSA reaching-defs solver upgrade (#2201 review R3, pure) > an identical post-#2201 stamp compares equal (no spurious re-analysis churn)", + "category": "import-initialization", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/unit/pdg-mode-flip.test.ts#pdgModeMismatch \u2014 M2\u2192M3 stamp upgrade (#2083 M3 U5, pure) > an identical resolved M3 config compares equal (steady state keeps incremental)", + "b": "test/unit/pdg-mode-flip.test.ts#pdgModeMismatch \u2014 pre-#2201\u2192SSA reaching-defs solver upgrade (#2201 review R3, pure) > an identical post-#2201 stamp compares equal (no spurious re-analysis churn)", + "category": "import-initialization", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Python > detects User class with save and get_name methods", + "b": "test/integration/resolvers/python.test.ts#Python cross-file binding propagation > detects User class with save and get_name methods", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Python > detects get_user function and run function", + "b": "test/integration/resolvers/python.test.ts#Python cross-file binding propagation > detects get_user and run functions", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Python > resolves u.get_name() in run() to User#get_name via cross-file return type", + "b": "test/integration/resolvers/python.test.ts#Python cross-file binding propagation > resolves u.get_name() in run() to User#get_name via cross-file return type propagation", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Go > detects User struct with Save and GetName methods", + "b": "test/integration/resolvers/go.test.ts#Go cross-file binding propagation > detects User struct with Save and GetName methods", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Go > resolves user.Save() in main() to User#Save via cross-file return type", + "b": "test/integration/resolvers/go.test.ts#Go cross-file binding propagation > resolves user.Save() in main() to User#Save via cross-file propagation", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: Rust > detects User struct with save and get_name methods", + "b": "test/integration/resolvers/rust.test.ts#Rust cross-file binding propagation > detects User struct with save and get_name methods", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: JavaScript > detects getUser factory and run function", + "b": "test/integration/resolvers/javascript.test.ts#JavaScript cross-file binding propagation > detects getUser and run functions", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/cross-file-binding.test.ts#Phase 9 \u2014 Cross-File Call-Result Binding: C# > detects User class with Save and GetName methods", + "b": "test/integration/resolvers/csharp.test.ts#C# cross-file binding propagation > detects User class with Save and GetName methods", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/resolvers/java.test.ts#Java same-arity overload cross-file and chain resolution > crossFileById() emits exactly one CALLS edge to find(int) in DbLookup", + "b": "test/integration/resolvers/java.test.ts#Java same-arity overloads (worker path) > crossFileById() \u2192 find(int) \u2014 sequential path", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/resolvers/javascript-object-properties.test.ts#JavaScript plain-object property access (A1/A5) > emits NOTHING when two objects share the key name", + "b": "test/integration/resolvers/javascript-object-properties.test.ts#JavaScript plain-object property access (A1/A5) > scope narrowing for multi-candidate names (R2) > keeps refusing two same-named keys in the reading file", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/resolvers/rust.test.ts#Rust qualified paths resolve against the module tree (#2730) > ignores a function-local fn of the same name in the target module (H3)", + "b": "test/integration/resolvers/rust.test.ts#Rust qualified paths resolve against the module tree (#2730) > follows a `pub use` re-export through to the original definition", + "category": "integration-same-fixture", + "basis": "Same fixture directory or shared PipelineResult, identical runPipelineFromRepo setup and matching assertions. CROSS_FILE_FIXTURES resolves to the same directory through both definitions." + }, + { + "a": "test/integration/tree-sitter-languages.test.ts#Tree-sitter multi-language parsing > unhappy path > handles empty string file path", + "b": "test/unit/ingestion-utils.test.ts#getLanguageFromFilename > unsupported > returns null for empty string", + "category": "irrelevant-setup", + "basis": "Both import getLanguageFromFilename from gitnexus-shared, pass empty string and assert toBeNull. Loading a parser in an unrelated beforeAll does not change this pure function." + }, + { + "a": "test/unit/analyze-api.test.ts#analyze API logic > rejects when job already active for different repo", + "b": "test/unit/analyze-job.test.ts#JobManager > enforces single-slot concurrency", + "category": "cleanup-context", + "basis": "Both instantiate JobManager before each test, use identical URLs and job transitions, and assert the same result. Extra mock/timer restoration happens after the assertions." + }, + { + "a": "test/unit/analyze-api.test.ts#analyze API logic > returns existing job for same repo URL", + "b": "test/unit/analyze-job.test.ts#JobManager > returns existing job for same repoUrl when active", + "category": "cleanup-context", + "basis": "Both instantiate JobManager before each test, use identical URLs and job transitions, and assert the same result. Extra mock/timer restoration happens after the assertions." + }, + { + "a": "test/unit/cobol-preprocessor.test.ts#extractCobolSymbolsWithRegex > Linkage & Data Flow > PROCEDURE DIVISION USING RETURNING excludes return value from USING list", + "b": "test/unit/cobol-preprocessor.test.ts#extractCobolSymbolsWithRegex > PROCEDURE DIVISION USING edge cases > excludes RETURNING value from USING parameter list", + "category": "same-file", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/unit/embedder.test.ts#embedder > getEmbeddingDims > returns 384 (MiniLM default)", + "b": "test/unit/http-embedder.test.ts#HTTP embedding backend > MCP embedder > returns 384 dimensions by default", + "category": "cleanup-context", + "basis": "Same getEmbeddingDims import, default environment and 384 assertion; additional afterEach restoration adds no scenario input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > array suffix (TypeScript / Java / C#) > User[] \u2192 User", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > strips array suffix: User[] \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > array suffix (TypeScript / Java / C#) > string[] \u2192 string", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > strips array suffix for primitive: string[] \u2192 string", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > Go slice prefix > []User \u2192 User", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > strips Go slice prefix: []User \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > Go slice prefix > []string \u2192 string", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > strips Go slice prefix for primitive: []string \u2192 string", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > Swift array sugar > [User] \u2192 User", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > unwraps Swift array sugar: [User] \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > generic angle-bracket containers > Array \u2192 User", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > unwraps single-arg generic angle: Array \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > generic angle-bracket containers > Vec \u2192 User (Rust)", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > unwraps single-arg generic angle: Rust Vec \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > generic angle-bracket containers > vector \u2192 User (C++)", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > unwraps single-arg generic angle: C++ vector \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > Python subscript-style generics > List[User] \u2192 User", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > unwraps Python subscript: List[User] \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > multi-argument generics \u2014 default returns last (value) arg > Map \u2192 User (default: last/value arg)", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > returns last arg by default for Map: \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > multi-argument generics \u2014 default returns last (value) arg > Map \u2192 String (pos=first: key arg)", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > returns first arg with pos=first for Map: \u2192 String", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > multi-argument generics \u2014 default returns last (value) arg > Map> \u2192 undefined (last arg is nested generic)", + "b": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > cross-bracket nesting (bracket depth fix) > Map> \u2192 undefined (default: last arg is nested generic)", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > multi-argument generics \u2014 default returns last (value) arg > Map> \u2192 String (pos=first: key arg)", + "b": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > cross-bracket nesting (bracket depth fix) > Map> \u2192 String (pos=first: key arg)", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > empty string \u2192 undefined", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > returns undefined for empty string", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/extract-element-type-from-string.test.ts#extractElementTypeFromString > edge cases \u2014 return undefined > plain type name (no container) \u2192 undefined", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > returns undefined when no brackets are present: User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/group/storage.test.ts#Group storage > getGroupDir returns correct path for group name", + "b": "test/unit/group/storage.test.ts#Group storage > getGroupDir rejects invalid names > test_getGroupDir_valid_name_returns_path", + "category": "same-file", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/unit/has-method.test.ts#findEnclosingClassId > Rust > finds enclosing impl_item for a method", + "b": "test/unit/has-method.test.ts#findEnclosingClassId > Rust > still picks struct name for plain impl Struct (no trait)", + "category": "native-parser", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/unit/index-repo-command.test.ts#indexCommand > fails when meta.json is missing and --force is not set", + "b": "test/unit/index-repo-command.test.ts#indexCommand > fails without --force when LadybugDB exists but metadata is missing", + "category": "same-file", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/unit/model/registration-table.test.ts#behavior group isolation > Impl hooks write to types.registerImpl, never to types.registerClass", + "b": "test/unit/model/registration-table.test.ts#hook behavior (real registries, no mocks) > implHook writes to types.registerImpl, NOT types.registerClass", + "category": "same-file", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/unit/scope-resolution/function-local-import-chain.test.ts#Python: a function-local import reaches the IMPORTS reason > `def f(): from m import X` is deferred", + "b": "test/unit/scope-resolution/function-local-import-chain.test.ts#Rust: a function-local `use` is NOT deferred > Python still defers on the same run", + "category": "native-parser", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/unit/security.test.ts#path traversal (isTestFilePath as proxy for path handling) > isTestFilePath returns false for nodes without a filePath", + "b": "test/unit/test-file-path.test.ts#isTestFilePath \u2014 shared predicate > returns false for nullish input rather than throwing", + "category": "reexport-context", + "basis": "local-backend.ts imports and directly re-exports the same isTestFilePath function. Both tests pass undefined, null and empty string and assert false three times." + }, + { + "a": "test/unit/server.test.ts#server error handling > createMCPServer reads version from package.json", + "b": "test/unit/server.test.ts#prompt registration > server registers detect_impact and generate_map prompts", + "category": "same-file", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > returns undefined when base is not a simple word: user model[]", + "b": "test/unit/shared-type-extractors.test.ts#extractElementTypeFromString > returns undefined for complex non-word base before []: user model[]", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > strips \"| null\": User | null \u2192 User", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > strips User | null \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > strips \"| undefined\": User | undefined \u2192 User", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > strips User | undefined \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > strips both \"| null | undefined\": User | null | undefined \u2192 User", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > strips User | null | undefined \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > strips nullable suffix: User? \u2192 User", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > strips User? \u2192 User", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > returns undefined for genuine union: User | Repo", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > refuses genuine union User | Repo \u2192 undefined", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > returns undefined for bare \"null\"", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > returns undefined for null alone", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > returns undefined for bare \"undefined\"", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > returns undefined for undefined alone", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > returns undefined for bare \"None\"", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > returns undefined for None alone", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > returns undefined for bare \"nil\"", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > returns undefined for nil alone", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > returns undefined for bare \"void\"", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > returns undefined for void alone", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > returns undefined for empty string", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > returns undefined for empty string", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/shared-type-extractors.test.ts#stripNullable > returns the type name unchanged when no nullable markers: User \u2192 User", + "b": "test/unit/type-env.test.ts#buildTypeEnv > stripNullable > passes through User unchanged", + "category": "pure-helper", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/stdout-silence.test.ts#stdout silencing > handles nested silence/restore (reference counting)", + "b": "test/unit/stdout-silence.test.ts#stdout silencing > simulates embedder + pool-adapter concurrent usage without conflict", + "category": "same-file", + "basis": "Identical scenario body in one file, using the same helpers, input data and assertions. Enclosing setup has no relevant difference." + }, + { + "a": "test/unit/type-env.test.ts#buildTypeEnv > constructor inference (Tier 1 fallback) > Rust if-let / while-let pattern bindings > extracts binding from if let Some(x) = opt via Phase 5.2 pattern binding", + "b": "test/unit/type-env.test.ts#buildTypeEnv > constructor inference (Tier 1 fallback) > Rust if-let / while-let pattern bindings > Phase 5.2: extracts binding from if let Some(x) = opt where opt: Option", + "category": "native-parser", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/type-env.test.ts#buildTypeEnv > previously-skipped limitations (now resolved) > TS instanceof narrowing: if (x instanceof User) \u2014 first-writer-wins, not block-scoped", + "b": "test/unit/type-env.test.ts#buildTypeEnv > null-check narrowing via patternOverrides (Phase C Task 7) > TS: instanceof still works alongside null-check narrowing", + "category": "native-parser", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/type-env.test.ts#buildTypeEnv > previously-skipped limitations (now resolved) > Rust for with .iter(): for user in users.iter() \u2014 call_expression iterable", + "b": "test/unit/type-env.test.ts#buildTypeEnv > container descriptor-aware type arg selection > Vec.iter() resolves to element type (User) \u2014 arity 1 always returns last", + "category": "native-parser", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/type-env.test.ts#buildTypeEnv > method-aware type arg selection (.keys() vs .values()) > Rust for key in map.keys() resolves to key type (String)", + "b": "test/unit/type-env.test.ts#buildTypeEnv > container descriptor-aware type arg selection > HashMap.keys() resolves to key type (String) via descriptor", + "category": "native-parser", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/type-env.test.ts#buildTypeEnv > method-aware type arg selection (.keys() vs .values()) > Rust for user in map.values() resolves to value type (User)", + "b": "test/unit/type-env.test.ts#buildTypeEnv > container descriptor-aware type arg selection > HashMap.values() resolves to value type (User) via descriptor", + "category": "native-parser", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + }, + { + "a": "test/unit/type-env.test.ts#buildTypeEnv > Kotlin when/is pattern binding (Phase 6) > when (x) { is User -> } binds x to User", + "b": "test/unit/type-env.test.ts#buildTypeEnv > null-check narrowing via patternOverrides (Phase C Task 7) > Kotlin: when/is still works alongside null-check narrowing", + "category": "native-parser", + "basis": "Same production function, literal inputs and expected values; source callbacks are identical. No enclosing hooks change the input." + } + ], + "negatives": [ + { + "a": "test/integration/resolvers/swift.test.ts#Swift return type inference > resolves user.save() to User#save via return type of getUser() -> User", + "b": "test/integration/resolvers/swift.test.ts#Swift return-type inference via function return type > resolves user.save() to User#save via return type of getUser()", + "audit": "distinct", + "reason": "Different fixtures: swift-return-type versus swift-return-type-inference; latter tests competing methods." + }, + { + "a": "test/unit/cursor-hook.test.ts#Cursor hook source regressions > uses npx.cmd for Windows", + "b": "test/unit/factory-plugin.test.ts#Factory hook source regressions > invokes npx.cmd directly on Windows instead of a shell", + "audit": "distinct", + "reason": "The captured source is a different integration hook implementation." + }, + { + "a": "test/unit/cursor-hook.test.ts#Cursor hook source regressions > passes pattern after end-of-options marker (--)", + "b": "test/unit/factory-plugin.test.ts#Factory hook source regressions > passes the pattern after the -- end-of-options marker", + "audit": "distinct", + "reason": "The captured source is a different integration hook implementation." + }, + { + "a": "test/unit/scope-resolution/c/c-captures.test.ts#emitCScopeCaptures \u2014 scopes > captures translation_unit as @scope.module", + "b": "test/unit/scope-resolution/cpp/cpp-captures.test.ts#emitCppScopeCaptures \u2014 scopes > captures translation_unit as @scope.module", + "audit": "distinct", + "reason": "Same snippet and assertion text, but different language parser/provider helpers." + }, + { + "a": "test/unit/scope-resolution/c/c-captures.test.ts#emitCScopeCaptures \u2014 scopes > captures struct_specifier as @scope.class", + "b": "test/unit/scope-resolution/cpp/cpp-captures.test.ts#emitCppScopeCaptures \u2014 scopes > captures struct_specifier as @scope.class", + "audit": "distinct", + "reason": "Same snippet and assertion text, but different language parser/provider helpers." + }, + { + "a": "test/unit/scope-resolution/c/c-captures.test.ts#emitCScopeCaptures \u2014 other declarations > captures typedef as @declaration.typedef", + "b": "test/unit/scope-resolution/cpp/cpp-captures.test.ts#emitCppScopeCaptures \u2014 typedef/alias declarations > captures typedef as @declaration.typedef", + "audit": "distinct", + "reason": "Same snippet and assertion text, but different language parser/provider helpers." + }, + { + "a": "test/unit/scope-resolution/c/c-captures.test.ts#emitCScopeCaptures \u2014 other declarations > captures variable with initializer as @declaration.variable", + "b": "test/unit/scope-resolution/cpp/cpp-captures.test.ts#emitCppScopeCaptures \u2014 variable declarations > captures variable with initializer", + "audit": "distinct", + "reason": "Same snippet and assertion text, but different language parser/provider helpers." + }, + { + "a": "test/unit/scope-resolution/c/c-captures.test.ts#emitCScopeCaptures \u2014 other declarations > captures macro as @declaration.macro", + "b": "test/unit/scope-resolution/cpp/cpp-captures.test.ts#emitCppScopeCaptures \u2014 macro declarations > captures #define as @declaration.macro", + "audit": "distinct", + "reason": "Same snippet and assertion text, but different language parser/provider helpers." + }, + { + "a": "test/unit/setup.test.ts#setupQoder > writes the MCP entry to ~/.qoder.json", + "b": "test/unit/setup.test.ts#setupDroid (Factory) > writes the MCP entry to ~/.factory/mcp.json under mcpServers", + "audit": "distinct", + "reason": "Different selected editors and configPath helpers: Qoder versus Factory." + }, + { + "a": "test/unit/setup.test.ts#setupQoder > leaves a corrupt ~/.qoder.json untouched", + "b": "test/unit/setup.test.ts#setupDroid (Factory) > leaves a corrupt ~/.factory/mcp.json untouched", + "audit": "distinct", + "reason": "Different selected editors and configPath helpers: Qoder versus Factory." + }, + { + "a": "test/unit/verify-workflow-run-pr-identity.test.ts#commit-fork-prebuilds.yml contract > does not call commits/{sha}/pulls (empty for fork SHAs; comments may name it)", + "b": "test/unit/verify-workflow-run-pr-identity.test.ts#pr-autofix-publish.yml contract > does not call commits/{sha}/pulls (empty for fork SHAs; comments may name it)", + "audit": "distinct", + "reason": "The captured workflow source is a different file in each suite." + } + ] +} diff --git a/gitnexus/test/integration/scenario-map-consolidation-inputs.test.ts b/gitnexus/test/integration/scenario-map-consolidation-inputs.test.ts new file mode 100644 index 000000000..50e158be5 --- /dev/null +++ b/gitnexus/test/integration/scenario-map-consolidation-inputs.test.ts @@ -0,0 +1,65 @@ +import { expect, it } from 'vitest'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; + +// Ground truth controls: fixture content changes independently of the checks below. +const BASE = 'export function calculate(value: number) { return value + 1; }\n'; +const RENAMED = 'export function compute(amount: number) { return amount + 1; }\n'; +const EXTRA = + BASE + 'export function extra(value: number) { while (value > 0) { value--; } return value; }\n'; +const UNRELATED = 'export interface Settings { enabled: boolean; label: string; }\n'; +let count = 0; + +async function ingest(source: string, extension = 'ts'): Promise { + const root = mkdtempSync(path.join(tmpdir(), 'consolidation-control-')); + try { + writeFileSync(path.join(root, `main.${extension}`), source); + count = (await runPipelineFromRepo(root, () => {}, { skipGraphPhases: true })).graph.nodeCount; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +// dupe-ok: intentional detector control; preserve fixture and capture ground truth +it('exact A', async () => { + await ingest(BASE); + expect(count).toBeGreaterThan(0); +}); +// dupe-ok: intentional detector control; preserve fixture and capture ground truth +it('exact B', async () => { + await ingest(BASE); + expect(count).toBeGreaterThan(0); +}); +// dupe-ok: intentional detector control; preserve fixture and capture ground truth +it('renamed fixture', async () => { + await ingest(RENAMED); + expect(count).toBeGreaterThan(0); +}); +// dupe-ok: intentional detector control; preserve fixture and capture ground truth +it('partial fixture and unique check', async () => { + await ingest(EXTRA); + expect(count).toBeGreaterThan(0); + expect(count).toBeGreaterThan(2); +}); +// dupe-ok: intentional detector control; preserve fixture and capture ground truth +it('different expectation', async () => { + await ingest(BASE); + expect(count).toBeGreaterThan(1); +}); +// dupe-ok: intentional detector control; preserve fixture and capture ground truth +it('unrelated assertion', async () => { + await ingest(BASE); + expect(count).toBeTypeOf('number'); +}); +// dupe-ok: intentional detector control; preserve fixture and capture ground truth +it('unrelated fixture', async () => { + await ingest(UNRELATED); + expect(count).toBeGreaterThan(0); +}); +// dupe-ok: intentional detector control; preserve fixture and capture ground truth +it('equivalent Python scenario', async () => { + await ingest('def calculate(value):\n return value + 1\n', 'py'); + expect(count).toBeGreaterThan(0); +}); diff --git a/gitnexus/test/integration/scenario-map-runtime-inputs.test.ts b/gitnexus/test/integration/scenario-map-runtime-inputs.test.ts new file mode 100644 index 000000000..992d124ff --- /dev/null +++ b/gitnexus/test/integration/scenario-map-runtime-inputs.test.ts @@ -0,0 +1,49 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +// Positive/negative controls for the scenario-map runtime fixture recorder. +import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; + +const inputs = [ + ['typescript', 'ts', 'export function twice(x: number) { return x + x; }'], + ['python', 'py', 'def twice(x):\n return x + x\n'], + ['loop', 'ts', 'export function twice(x: number) { while(x > 0) { x--; } return x; }'], +] as const; + +describe.each(inputs)('%s generated repository', (_name, extension, source) => { + let count = 0; + beforeAll(async () => { + const root = mkdtempSync(path.join(tmpdir(), 'scenario-runtime-')); + try { + writeFileSync(path.join(root, `main.${extension}`), source); + count = (await runPipelineFromRepo(root, () => {}, { skipGraphPhases: true })).graph + .nodeCount; + } finally { + rmSync(root, { recursive: true, force: true }); + } + }, 60_000); + // dupe-ok: intentional detector control; preserve fixture and capture ground truth + it('finds declarations', () => { + expect(count).toBeGreaterThan(0); + }); + // dupe-ok: intentional detector control; preserve fixture and capture ground truth + it('retains the result after fixture cleanup', () => { + expect(count).toBeGreaterThan(0); + }); +}); + +// dupe-ok: intentional detector control; preserve fixture and capture ground truth +it.each(['sync', 'promise'])('dynamic %s read survives cleanup', async (api) => { + const root = mkdtempSync(path.join(tmpdir(), 'scenario-runtime-read-')); + let text: string; + try { + const file = path.join(root, 'main.ts'); + writeFileSync(file, inputs[0][2]); + text = api === 'sync' ? readFileSync(file, 'utf8') : await readFile(file, 'utf8'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + expect(text).toContain('function twice'); +}); diff --git a/gitnexus/test/unit/scenario-map/candidates.test.ts b/gitnexus/test/unit/scenario-map/candidates.test.ts index 66f247341..2c594031e 100644 --- a/gitnexus/test/unit/scenario-map/candidates.test.ts +++ b/gitnexus/test/unit/scenario-map/candidates.test.ts @@ -80,6 +80,22 @@ describe('jaccard', () => { }); describe('findCandidates: Jaccard source', () => { + it('retrieves the same body across cleanup/import contexts and dissimilar setup coverage', () => { + const result = findCandidates([ + test('direct', [1, 2], { checksContext: 'direct-context', bodyKey: 'same-body' }), + test('reexport', [1, 2, 3, 4], { checksContext: 'reexport-context', bodyKey: 'same-body' }), + test('other', [5], { checksContext: 'other-context', bodyKey: 'different-body' }), + ]); + expect(result.pairs).toEqual([ + expect.objectContaining({ + a: 'direct', + b: 'reexport', + sources: ['body'], + identicalBlocks: false, + }), + ]); + }); + it.each([ { name: 'one block removed (199/200)', b: range(0, 199) }, { name: 'one block replaced (199/201)', b: [...range(0, 199), 500] }, @@ -147,7 +163,7 @@ describe('findCandidates: checks-and-context source', () => { test('b', [1, 2], { checksContext: 'k' }), ]); expect(result.pairs.map((pair) => pair.sources)).toEqual([['jaccard', 'checks-context']]); - expect(result.stats.bySource).toEqual({ jaccard: 0, checksContext: 0, both: 1 }); + expect(result.stats.bySource).toEqual({ jaccard: 0, checksContext: 0, both: 1, body: 0 }); }); }); diff --git a/gitnexus/test/unit/scenario-map/cli.test.ts b/gitnexus/test/unit/scenario-map/cli.test.ts index 727f9904d..e327d8fd3 100644 --- a/gitnexus/test/unit/scenario-map/cli.test.ts +++ b/gitnexus/test/unit/scenario-map/cli.test.ts @@ -93,19 +93,19 @@ describe('trace selection', () => { 'excludes configured retries from %s traces even when coverage passed', (mutation) => { const pairs = [ - { a: 'a', b: 'b' }, - { a: 'a', b: 'retry' }, - { a: 'a', b: 'failed' }, - { a: 'a', b: 'failed-retry' }, + { a: 'a.ts#a', b: 'b.ts#b' }, + { a: 'a.ts#a', b: 'retry.ts#retry' }, + { a: 'a.ts#a', b: 'failed.ts#failed' }, + { a: 'a.ts#a', b: 'failed-retry.ts#failed-retry' }, ]; expect( passingTracePairs( [ - { id: 'a', state: 'passed', retry: 0 }, - { id: 'b', state: 'passed' }, - { id: 'retry', state: 'passed', retry: 1 }, - { id: 'failed', state: 'failed' }, - { id: 'failed-retry', state: 'failed', retry: 2 }, + { id: 'a.ts#a', state: 'passed', retry: 0 }, + { id: 'b.ts#b', state: 'passed' }, + { id: 'retry.ts#retry', state: 'passed', retry: 1 }, + { id: 'failed.ts#failed', state: 'failed' }, + { id: 'failed-retry.ts#failed-retry', state: 'failed', retry: 2 }, ], pairs, mutation, @@ -116,25 +116,44 @@ describe('trace selection', () => { it('keeps non-passing coverage tests out of the shared trace without losing valid pairs', () => { const pairs = [ - { a: 'a', b: 'b' }, - { a: 'a', b: 'failed' }, - { a: 'skipped', b: 'b' }, - { a: 'pending', b: 'b' }, - { a: 'missing', b: 'b' }, + { a: 'a.ts#a', b: 'b.ts#b' }, + { a: 'a.ts#a', b: 'failed.ts#failed' }, + { a: 'skipped.ts#skipped', b: 'b.ts#b' }, + { a: 'pending.ts#pending', b: 'b.ts#b' }, + { a: 'missing.ts#missing', b: 'b.ts#b' }, ]; expect( passingTracePairs( [ - { id: 'a', state: 'passed' }, - { id: 'b', state: 'passed' }, - { id: 'failed', state: 'failed' }, - { id: 'skipped', state: 'skipped' }, - { id: 'pending', state: 'pending' }, + { id: 'a.ts#a', state: 'passed' }, + { id: 'b.ts#b', state: 'passed' }, + { id: 'failed.ts#failed', state: 'failed' }, + { id: 'skipped.ts#skipped', state: 'skipped' }, + { id: 'pending.ts#pending', state: 'pending' }, ], pairs, ), ).toEqual([pairs[0]]); }); + + it('keeps an entire file inconclusive when a runnable sibling uses retries', () => { + const pairs = [ + { a: 'retry.ts#passing', b: 'stable.ts#a' }, + { a: 'stable.ts#a', b: 'stable.ts#b' }, + ]; + expect( + passingTracePairs( + [ + { id: 'retry.ts#passing', state: 'passed' }, + { id: 'retry.ts#sibling', state: 'passed', retry: 1 }, + { id: 'stable.ts#a', state: 'passed' }, + { id: 'stable.ts#b', state: 'passed' }, + { id: 'stable.ts#skipped', state: 'skipped', retry: 1 }, + ], + pairs, + ), + ).toEqual([pairs[1]]); + }); }); const tempDir = (): string => { const dir = mkdtempSync(path.join(os.tmpdir(), 'scenario-map-u14-')); @@ -335,16 +354,9 @@ describe('runCli usage', () => { ).toEqual(['HEAD^1', 'HEAD^1', 'HEAD^1']); }); - it.each([ - { mutation: 'all' as const, kind: 'normal', mode: { kind: 'normal' as const } }, - { - mutation: 'none' as const, - kind: 'controls', - mode: { kind: 'controls' as const, fixtureRoot: 'gitnexus/test/fixtures/x' }, - }, - ])('runs every file in Layer 1 for mutation $mutation in $kind mode', ({ mutation, mode }) => { + it('runs every file in Layer 1 for a whole-suite run', () => { const files = ['test/unit/a.test.ts', 'test/unit/b.test.ts']; - expect(layer1FilesOf('/no-repo', files, mutation, mode)).toEqual(files); + expect(layer1FilesOf('/no-repo', files)).toEqual(files); }); it('exits 2 for a work directory inside the checkout', async () => { @@ -531,7 +543,18 @@ function writeTwoShardRun(work: string, replan: ShardReplan | null = null): void extraRoots: [], repeats: 1, blocks: ['gitnexus/src/a.ts#f@0'], - files: [], + files: plan.eligibility.fingerprintFiles.map((file) => ({ + file, + recorded: true, + errors: [], + unattributed: { + known: true, + executedFiles: [], + nativeEntries: [], + workerSpawns: [], + excludedCandidates: [], + }, + })), tests: tests.map((id, i) => coverageTest(id, i + 1)), unhandledErrors: [], }; @@ -620,6 +643,46 @@ function writeTwoShardRun(work: string, replan: ShardReplan | null = null): void } describe('merge stage', () => { + it('keeps a fully collected run partial when a captured test failed', async () => { + const { deps, work } = setup(); + writeTwoShardRun(work, ONE_SHARD_REPLAN); + const layout = workLayout(work); + const coverage = JSON.parse(readFileSync(layout.coverage, 'utf8')) as CoverageWorkFile; + coverage.tests[0].state = 'failed'; + coverage.tests[0].errors = ['AssertionError: unexpected capture result']; + writeFileSync(layout.coverage, JSON.stringify(coverage)); + expect(await runCli(['merge', '--work', work], deps)).toBe(1); + const report = JSON.parse(readFileSync(layout.reportJson, 'utf8')) as ScenarioMapReport; + expect(report.inventory?.collectionComplete).toBe(true); + expect(report.completeness.shards.incomplete).toContainEqual({ + shard: 'coverage tests', + reason: '1 failed; 0 unfinished', + }); + expect(report.completeness.complete).toBe(false); + }); + + it('marks missing coverage partial and reports cases independently of candidate pairs', async () => { + const { deps, work } = setup(); + writeTwoShardRun(work, ONE_SHARD_REPLAN); + const layout = workLayout(work); + const coverage = JSON.parse(readFileSync(layout.coverage, 'utf8')) as CoverageWorkFile; + coverage.files[0].recorded = false; + writeFileSync(layout.coverage, JSON.stringify(coverage)); + const code = await runCli(['merge', '--work', work], deps); + const report = JSON.parse(readFileSync(layout.reportJson, 'utf8')) as ScenarioMapReport; + expect(code).toBe(1); + expect(report.completeness.partial).toBe(true); + expect(report.completeness.shards.incomplete).toContainEqual({ + shard: 'coverage', + reason: '1 files missing or errored; 0 unhandled errors', + }); + expect(report.inventory?.tests.collected).toBe(6); + expect(report.inventory?.files.unrecorded).toEqual(['test/unit/a.test.ts']); + expect(readFileSync(layout.reportMarkdown, 'utf8')).toContain( + 'Runtime inventory: 6 test cases collected', + ); + }); + it('reports the fingerprint of every pair and no mutation verdict for a fingerprint-only run', async () => { const { deps, work } = setup(); writeTwoShardRun(work); diff --git a/gitnexus/test/unit/scenario-map/consolidation.test.ts b/gitnexus/test/unit/scenario-map/consolidation.test.ts new file mode 100644 index 000000000..7a8c38a23 --- /dev/null +++ b/gitnexus/test/unit/scenario-map/consolidation.test.ts @@ -0,0 +1,750 @@ +import { expect, it } from 'vitest'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync } from 'node:fs'; +import path from 'node:path'; +import { assertionEvidence } from '../../../tools/scenario-map/consolidation/assertions.js'; +import { + FixtureCloneIndex, + FixtureCloneCandidates, + verifyFixtureClonePairs, + runFixtureClones, + type CloneRun, +} from '../../../tools/scenario-map/consolidation/fixture-clones.js'; +import { + capturedFixtureSources, + mergeFixtureSources, + type FixtureSources, +} from '../../../tools/scenario-map/consolidation/fixture-sources.js'; +import { consolidationReport, overlap } from '../../../tools/scenario-map/consolidation/report.js'; +import { computeChecks } from '../../../tools/scenario-map/facet/checks.js'; +import { expandCases } from '../../../tools/scenario-map/inventory/cases.js'; +import { sha1 } from '../../../tools/scenario-map/util.js'; +import type { FixtureProfile } from '../../../tools/scenario-map/facet/fixture-profile.js'; +import type { + CoverageWorkFile, + CoverageTest, +} from '../../../tools/scenario-map/path-coverage/run-coverage.js'; +import type { Layer2Test } from '../../../tools/scenario-map/scenario-map.js'; + +const SOURCE = 'export function calculate(value: number) { return value + 1; }\n'; +const files: [string, string][] = [['main.ts', SOURCE]]; +const key = sha1(JSON.stringify(files)); +const sources: FixtureSources = { [key]: files }; +const profile: FixtureProfile = { + sourceKeys: [key], + roots: ['/tmp/input'], + files: ['/tmp/input/main.ts'], + contentKey: 'same', + languages: ['typescript'], + features: [['def:Function', 1]], + unresolved: [], +}; +const noClones: CloneRun = { tool: 'jscpd', version: 'test', sourceCount: 1, clones: [] }; +const atom = (value = '2', shape = 'count') => ({ + line: 1, + shape, + shapeKey: shape, + key: `${shape}:${value}`, + expected: value, +}); +const evidence = (atoms = [atom()]) => ({ atoms, unresolved: [] }); +const test = (id: string, blocks = [0, 1]): CoverageTest => ({ + id, + taskId: id, + file: 'test/input.test.ts', + fullName: id, + location: null, + state: 'passed', + concurrent: false, + attempts: 2, + blocks, + suiteBlocks: [], + provenance: { + known: true, + executedFiles: [], + nativeEntries: [], + workerSpawns: [], + excludedCandidates: [], + }, +}); +const detail = (id: string, assertions = evidence(), fixture = profile): Layer2Test => ({ + id, + file: 'test/input.test.ts', + anchor: null, + notOracleValidated: false, + dupeOk: null, + joinProblems: [], + inventoryUnresolved: [], + checksContext: null, + unsupportedForms: [], + unresolvedSelectors: [], + facet: null, + fixture, + assertionEvidence: assertions, +}); +const work = (tests: CoverageTest[]): CoverageWorkFile => ({ + schemaVersion: 1, + fixtureCaptureVersion: 2, + fixtureSources: sources, + mode: 'normal', + extraRoots: [], + repeats: 1, + blocks: [ + 'src/a.ts#one@0|1-2:1', + 'src/a.ts#two@2|3-4:1', + 'src/a.ts#miss@4|5-6:0', + 'src/b.ts#three@0|1-2:1', + ], + files: [], + tests, + unhandledErrors: [], +}); +const report = (tests: CoverageTest[], details: Layer2Test[]) => + consolidationReport(work(tests), details, new FixtureCloneIndex(sources, noClones)); + +it('requires all three axes for observed equality and groups repeated cases without expanding every pair', () => { + const tests = Array.from({ length: 100 }, (_, i) => test(String(i))); + const result = report( + tests, + tests.map((t) => detail(t.id)), + ); + expect(result.opportunities).toHaveLength(1); + expect(result.opportunities[0]).toMatchObject({ + kind: 'same-observed-input-path-assertions', + representedPairs: 4950, + path: { coverageA: 1, coverageB: 1 }, + assertions: { coverageA: 1, coverageB: 1 }, + }); +}); + +it('does not mistake shared uncovered branch ranges for shared execution', () => { + const result = report([test('a', [0, 2]), test('b', [3, 2])], [detail('a'), detail('b')]); + expect(result.opportunities).toEqual([]); +}); + +it('requires a fresh capture before treating archived read profiles as fixture inputs', () => { + const old = { ...work([test('a'), test('b')]), fixtureCaptureVersion: 1 as const }; + const result = consolidationReport( + old, + [detail('a'), detail('b')], + new FixtureCloneIndex(sources, noClones), + ); + expect(result.opportunities).toEqual([]); + expect(result.summary.excluded['unverified-input-provenance']).toBe(2); +}); + +it('reports partial shared checks and retains unique assertions and executed branches', () => { + const result = report( + [test('a', [0, 1]), test('b', [0, 3])], + [ + detail('a', evidence([atom(), atom('a', 'only-a')])), + detail('b', evidence([atom(), atom('b', 'only-b')])), + ], + ); + expect(result.opportunities[0]).toMatchObject({ + kind: 'extract-shared-scenario', + path: { coverageA: 0.5, coverageB: 0.5 }, + assertions: { shared: 1, totalA: 2, totalB: 2 }, + }); + const found = result.opportunities[0]; + expect([...found.onlyAssertionsA, ...found.onlyAssertionsB].join()).toContain('only-a'); + expect([...found.onlyAssertionsA, ...found.onlyAssertionsB].join()).toContain('only-b'); + expect([...found.onlyBlocksA, ...found.onlyBlocksB]).toContain('src/a.ts#two@2|3-4:1'); +}); + +it('keeps different expected values as parameter rows, not equal assertions', () => { + const result = report([test('a'), test('b')], [detail('a'), detail('b', evidence([atom('3')]))]); + expect(result.opportunities[0]).toMatchObject({ + kind: 'parameterize-expectations', + assertions: { shared: 0 }, + assertionShapes: { shared: 1 }, + }); +}); + +it('retains existing tables and intentional controls separately from actionable consolidation', () => { + const cases = [test('row-a'), test('row-b'), test('control'), test('separate')]; + const details = [ + { ...detail('row-a'), declarationKey: 'table' }, + { ...detail('row-b', evidence([atom('3')])), declarationKey: 'table' }, + { ...detail('control'), declarationKey: 'control', dupeOk: 'detector positive control' }, + { ...detail('separate'), declarationKey: 'separate' }, + ]; + const result = report(cases, details); + expect(result.opportunities.some((o) => o.review === 'already-parameterized')).toBe(true); + expect(result.opportunities.some((o) => o.review === 'intentional-keep')).toBe(true); + expect(result.opportunities.some((o) => o.review === 'actionable')).toBe(true); + expect(result.summary.actionableOpportunities).toBeLessThan(result.summary.opportunities); + expect(result.summary.representedPairs).toBe(6); +}); + +it('keeps equal observed table rows actionable and separates keep reasons from evidence gaps', () => { + const equalRows = report( + [test('a'), test('b')], + ['a', 'b'].map((id) => ({ + ...detail(id), + declarationKey: 'same-table', + })), + ); + expect(equalRows.opportunities[0].review).toBe('actionable'); + const kept = report( + [test('a'), test('b')], + ['a', 'b'].map((id) => ({ + ...detail(id), + dupeOk: 'control', + })), + ); + expect(kept.opportunities[0]).toMatchObject({ + review: 'intentional-keep', + kind: 'same-observed-input-path-assertions', + limitations: [], + }); +}); + +it('rejects disjoint checks, failed cases, concurrent paths, missing inputs, and unresolved expectations', () => { + expect( + report([test('a'), test('b')], [detail('a'), detail('b', evidence([atom('2', 'different')]))]) + .opportunities, + ).toEqual([]); + for (const patch of [{ state: 'failed' as const }, { concurrent: true }]) + expect( + report([test('a'), { ...test('b'), ...patch }], [detail('a'), detail('b')]).opportunities, + ).toEqual([]); + expect( + report( + [test('a'), test('b')], + [detail('a'), detail('b', evidence(), { ...profile, unresolved: ['missing file'] })], + ).opportunities, + ).toEqual([]); + expect( + report( + [test('a'), test('b')], + [detail('a'), detail('b', evidence([{ ...atom(), key: null, expected: null }]))], + ).opportunities, + ).toEqual([]); +}); + +it('counts repeated assertions as a multiset and handles empty evidence honestly', () => { + expect(overlap(['a', 'a', 'b'], ['a', 'b'])).toMatchObject({ + shared: 2, + totalA: 3, + totalB: 2, + coverageB: 1, + }); + expect(overlap([], [])).toMatchObject({ shared: 0, jaccard: 0 }); +}); + +it('merges immutable source tables and rejects altered source hashes', () => { + expect(mergeFixtureSources([sources, sources])).toEqual(sources); + expect(() => mergeFixtureSources([{ [key]: [['main.ts', 'changed']] }])).toThrow('hash mismatch'); + expect(capturedFixtureSources([])).toEqual({}); +}); + +it('measures clone coverage using weighted byte spans instead of whole matching lines', () => { + const otherFiles: [string, string][] = [['main.ts', SOURCE.replace('+', '-')]]; + const otherKey = sha1(JSON.stringify(otherFiles)); + const run: CloneRun = { + ...noClones, + sourceCount: 2, + clones: [ + { + a: { source: key, file: 'main.ts', start: 0, end: 20, startLine: 1, endLine: 1 }, + b: { source: otherKey, file: 'main.ts', start: 0, end: 20, startLine: 1, endLine: 1 }, + kind: 'similar', + similarity: 0.5, + tokens: 12, + }, + ], + }; + const index = new FixtureCloneIndex({ ...sources, [otherKey]: otherFiles }, run); + const found = index.compare(profile, { + ...profile, + sourceKeys: [otherKey], + contentKey: 'different', + }); + expect(found?.coverageA).toBeGreaterThan(0); + expect(found?.coverageA).toBeLessThan(0.5); + expect(found?.kind).toBe('clone'); +}); + +it('runs the pinned external clone engine on renamed and partial fixture functions', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'fixture-clone-test-')); + const renamed: [string, string][] = [ + ['main.ts', SOURCE.replaceAll('calculate', 'compute').replaceAll('value', 'amount')], + ]; + const longer: [string, string][] = [ + [ + 'main.ts', + SOURCE + '\nexport function extra(x: number) { while (x > 0) { x--; } return x; }\n', + ], + ]; + const renamedKey = sha1(JSON.stringify(renamed)), + longerKey = sha1(JSON.stringify(longer)); + try { + const all = { ...sources, [renamedKey]: renamed, [longerKey]: longer }; + const run = await runFixtureClones(all, root); + expect(run.version).toContain('5.4.2'); + const index = new FixtureCloneIndex(all, run); + expect( + index.compare(profile, { ...profile, sourceKeys: [renamedKey], contentKey: 'renamed' }) + ?.coverageA, + ).toBeGreaterThan(0.5); + const partial = index.compare(profile, { + ...profile, + sourceKeys: [longerKey], + contentKey: 'longer', + }); + expect(partial?.coverageA).toBeGreaterThan(0.5); + expect(partial?.coverageB).toBeLessThan(0.8); + expect(partial?.unmatchedB.some((span) => span.end > SOURCE.length)).toBe(true); + // The external report may connect both to a representative, omitting this pair. + const transitive = index.compare( + { ...profile, sourceKeys: [renamedKey], contentKey: 'renamed' }, + { ...profile, sourceKeys: [longerKey], contentKey: 'longer' }, + ); + expect(transitive?.coverageA).toBeGreaterThan(0.5); + expect(transitive?.coverageB).toBeLessThan(0.8); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +it('verifies contained representative spans without projecting offsets or treating near misses as transitive', async () => { + const root = mkdtempSync(path.join(tmpdir(), 'fixture-contained-test-')); + const longer: [string, string][] = [ + ['main.ts', SOURCE + 'export function extra(x: number) { while (x > 0) x--; return x; }'], + ]; + const decoy: [string, string][] = [['decoy.ts', '// unrelated wrapper\n' + longer[0][1]]]; + const longerKey = sha1(JSON.stringify(longer)), + decoyKey = sha1(JSON.stringify(decoy)); + const all = { ...sources, [longerKey]: longer, [decoyKey]: decoy }; + const b = { ...profile, sourceKeys: [longerKey], contentKey: 'longer' }; + const span = (source: string, file: string, start: number, end: number) => ({ + source, + file, + start, + end, + startLine: 1, + endLine: 2, + }); + try { + const exported = await runFixtureClones(all, root); + // The full corpus can choose the same representative with DIFFERENT bounds. + // These representative links retrieve a candidate, never manufactured clone spans. + const run: CloneRun = { + ...exported, + clones: [ + { + a: span(key, 'main.ts', 0, SOURCE.length - 1), + b: span(decoyKey, 'decoy.ts', 21, 21 + SOURCE.length - 1), + kind: 'renamed', + similarity: 1, + tokens: 13, + }, + { + a: span(longerKey, 'main.ts', 0, SOURCE.length + 10), + b: span(decoyKey, 'decoy.ts', 21, 21 + SOURCE.length + 10), + kind: 'renamed', + similarity: 1, + tokens: 17, + }, + ], + }; + expect(new FixtureCloneIndex(all, run).compare(profile, b)).toBeNull(); + const candidates = new FixtureCloneCandidates(run); + consolidationReport( + { ...work([test('a'), test('b')]), fixtureSources: all }, + [detail('a'), detail('b', evidence(), b)], + new FixtureCloneIndex(all, run), + (a, b) => candidates.consider(a, b), + ); + expect(candidates.pairs()).toEqual([[key, longerKey].sort()]); + const verified = await verifyFixtureClonePairs( + all, + run, + candidates.pairs(), + path.join(root, 'pairs'), + ); + const result = new FixtureCloneIndex(all, verified).compare(profile, b); + expect(result?.coverageA).toBeGreaterThan(0.8); + expect(result?.coverageB).toBeLessThan(0.8); + expect(result?.unmatchedB.some((s) => s.end > SOURCE.length)).toBe(true); + expect(verified.verifiedPairs).toBe(1); + const nearMiss = new FixtureCloneCandidates({ + ...run, + clones: run.clones.map((c) => ({ ...c, kind: 'similar', similarity: 0.9 })), + }); + nearMiss.consider(profile, b); + expect(nearMiss.pairs()).toEqual([]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +function assertions(source: string) { + return expandCases([{ path: 'gitnexus/test/unit/controls.test.ts', source }]).sites.flatMap( + (site) => + (site.instances ?? []).map((instance) => { + const subject = { record: site.record, instance, fullName: site.fullName }; + return assertionEvidence(subject, computeChecks(subject)); + }), + ); +} + +it('does not consolidate assertions on same-named file-level constants with different values', () => { + const found = expandCases( + [1, 2].map((value) => ({ + path: `gitnexus/test/unit/constant-${value}.test.ts`, + source: `const result = ${value}; it('rule', () => expect(result).toBeGreaterThan(0));`, + })), + ).sites.map((site) => { + const subject = { + record: site.record, + instance: site.instances?.[0] ?? null, + fullName: site.fullName, + }; + return assertionEvidence(subject, computeChecks(subject)); + }); + expect(found[0].atoms[0].key).toBeTruthy(); + expect(found[1].atoms[0].key).not.toBe(found[0].atoms[0].key); + expect( + report([test('a'), test('b')], [detail('a', found[0]), detail('b', found[1])]).opportunities, + ).toEqual([]); +}); + +it.each([ + 'const result = compute();', + 'let result = 1; result++;', + 'const original = 1; const result = original;', + 'const { result } = { result: 1 };', + 'const result = { count: 1 }; result.count++;', +])('case %#: keeps enclosing value origins scoped: %s', (declaration) => { + const found = assertions(` + describe('outer', () => { + ${declaration} + it('direct', () => { expect(result).toBeDefined(); }); + it('alias', () => { const actual = result; expect(actual).toBeDefined(); }); + }); + describe('other', () => { + ${declaration} + it('same name', () => { expect(result).toBeDefined(); }); + }); + `); + expect(found[0].atoms[0].key).toBeTruthy(); + expect(found[1].atoms[0].key).toBe(found[0].atoms[0].key); + expect(found[2].atoms[0].key).not.toBe(found[0].atoms[0].key); +}); + +it('retains partial shared checks without matching different file-level bindings', () => { + const found = expandCases( + [1, 2].map((value) => ({ + path: `gitnexus/test/unit/partial-${value}.test.ts`, + source: `const result = ${value}; + it('rule', () => { + expect(result).toBeGreaterThan(0); + expect(graph.nodes.length).toBe(2); + });`, + })), + ).sites.map((site) => { + const subject = { + record: site.record, + instance: site.instances?.[0] ?? null, + fullName: site.fullName, + }; + return assertionEvidence(subject, computeChecks(subject)); + }); + const [opportunity] = report( + [test('a'), test('b')], + [detail('a', found[0]), detail('b', found[1])], + ).opportunities; + expect(opportunity).toMatchObject({ + kind: 'extract-shared-scenario', + assertions: { shared: 1, totalA: 2, totalB: 2 }, + }); + expect(opportunity.onlyAssertionsA).toHaveLength(1); + expect(opportunity.onlyAssertionsB).toHaveLength(1); +}); + +it('preserves consolidation of checks against the same suite-level pipeline result', () => { + const [a, b] = assertions(` + let count = 0; + async function ingest() { count = (await runPipeline()).graph.nodeCount; } + it('a', async () => { await ingest(); expect(count).toBeGreaterThan(0); }); + it('b', async () => { await ingest(); expect(count).toBeGreaterThan(0); }); + `); + expect( + report([test('a'), test('b')], [detail('a', a), detail('b', b)]).opportunities[0], + ).toMatchObject({ assertions: { shared: 1, coverageA: 1, coverageB: 1 } }); +}); + +it('does not conflate same-named file-local assertion helpers from different engines', () => { + const found = ['fingerprint', 'mutation'].map((engine) => { + const [site] = expandCases([ + { + path: `gitnexus/test/unit/${engine}.test.ts`, + source: ` + import { compute } from './${engine}-engine.js'; + const verdictOf = () => compute(); + it('rule', () => { expect(verdictOf().rule).toBe('same'); }); + `, + }, + ]).sites; + const subject = { record: site.record, instance: site.instances![0], fullName: site.fullName }; + return assertionEvidence(subject, computeChecks(subject)); + }); + expect(found[0].atoms[0].key).not.toBeNull(); + expect(found[0].atoms[0].key).not.toBe(found[1].atoms[0].key); +}); + +it('matches copied helpers with the same import and preserves unresolved closure identity', () => { + const evidenceFor = (file: string, closure: boolean) => { + const [site] = expandCases([ + { + path: `gitnexus/test/unit/${file}.test.ts`, + source: ` + import { compute } from './engine.js'; + let state = 1; + const run = (value) => compute(${closure ? 'state' : 'value'}); + it('rule', () => expect(run(1).rule).toBe('same')); + `, + }, + ]).sites; + const subject = { + record: site.record, + instance: site.instances?.[0] ?? null, + fullName: site.fullName, + }; + return assertionEvidence(subject, computeChecks(subject)); + }; + const a = evidenceFor('a', false), + b = evidenceFor('b', false); + expect(a.atoms[0].key).not.toBeNull(); + expect(a.atoms[0].key).toBe(b.atoms[0].key); + expect(evidenceFor('a', true).atoms[0].key).not.toBe(evidenceFor('b', true).atoms[0].key); +}); + +it('resolves asserted const origins and distinguishes different queried quantities', () => { + const found = assertions(` + it('a', () => { const n = graph.nodes.length; expect(n).toBe(2); }); + it('b', () => { const m = graph.nodes.length; expect(m).toBe(2); }); + it('c', () => { const n = graph.edges.length; expect(n).toBe(2); }); + it('d', () => { let n = graph.nodes.length; n++; expect(n).toBe(2); }); + `); + expect(found[0].atoms[0].key).toBeTruthy(); + expect(found[1].atoms[0].key).toBe(found[0].atoms[0].key); + expect(found[2].atoms[0].key).not.toBe(found[0].atoms[0].key); + expect(found[3].atoms[0].key).toBeNull(); +}); + +it('keeps statement order and callback bindings independent across assertions in one test', () => { + const [combined, direct] = assertions(` + it('combined', () => { + expect(count).toBe(2); + const count = graph.nodes.length; + expect(count).toBe(2); + expect(items.map(item => item.name)).toEqual(items.map(expected => expected.name)); + expect(items.map(value => value.name)).toEqual([]); + expect(items.map(other => other.name)).toEqual([]); + }); + it('direct', () => { expect(graph.nodes.length).toBe(2); }); + `); + const [before, after, dynamic, first, second] = combined.atoms; + expect(before.key).toBeNull(); + expect(after.key).toBeTruthy(); + expect(after.key).toBe(direct.atoms[0].key); + expect(dynamic.key).toBeNull(); + expect(dynamic.observed).toBe(first.observed); + expect(first.key).toBeTruthy(); + expect(first.key).toBe(second.key); +}); + +it('retains exact expected values, negation, selectors, and row values', () => { + const found = assertions(` + it.each([2, 3])('row %i', (n) => { expect(graph.nodes.length).toBe(n); }); + it('literal', () => { expect(graph.nodes.length).toBe(2); }); + it('negative', () => { expect(graph.nodes.length).not.toBe(2); }); + it('selector', () => { expect(graph.get('Function')).toHaveLength(2); }); + it('other selector', () => { expect(graph.get('Class')).toHaveLength(2); }); + `); + expect(found[0].atoms[0].key).toBe(found[2].atoms[0].key); + expect(found[0].atoms[0].key).not.toBe(found[1].atoms[0].key); + expect(found[2].atoms[0].key).not.toBe(found[3].atoms[0].key); + expect(found[4].atoms[0].key).not.toBe(found[5].atoms[0].key); +}); + +it('canonicalizes object and signed numeric expectations and rejects unknown expected values', () => { + const found = assertions(` + it.each([{ b: 2, a: -1 }])('row', (value) => { expect(result).toEqual(value); }); + it('literal', () => { expect(result).toEqual({ a: -1, b: 2 }); }); + it('unknown', () => { expect(result).toEqual(calculateExpected()); }); + it('shadow', () => { const undefined = calculateExpected(); expect(result).toBe(undefined); }); + `); + expect(found[0].atoms[0].expected).toBe(found[1].atoms[0].expected); + expect(found[2].atoms[0].key).toBeNull(); + expect(found[3].atoms[0].key).toBeNull(); +}); + +it('does not hide unknown checks or extra repeated assertions in partial overlaps', () => { + const a = evidence([atom(), atom(), { ...atom(), key: null, expected: null }]); + const result = report([test('a'), test('b')], [detail('a', a), detail('b')]); + const found = result.opportunities[0]; + expect(found.assertions.shared).toBe(1); + expect([found.assertions.totalA, found.assertions.totalB].sort()).toEqual([1, 3]); + expect(found.onlyAssertionsA.length + found.onlyAssertionsB.length).toBe(2); + expect(found.kind).not.toBe('same-observed-input-path-assertions'); +}); + +it('resolves const query chains and lexically bound callback parameters without conflating selectors', () => { + const found = assertions(` + it('a', () => { const calls = getRelationships(result, 'CALLS'); const edge = calls.find(c => c.target === 'save'); expect(edge).toBeDefined(); }); + it('b', () => { const relations = getRelationships(result, 'CALLS'); const match = relations.find(e => e.target === 'save'); expect(match).toBeDefined(); }); + it('c', () => { const calls = getRelationships(result, 'CALLS'); const edge = calls.find(c => c.target === 'getName'); expect(edge).toBeDefined(); }); + it('d', () => { const calls = getRelationships(result, 'IMPORTS'); const edge = calls.find(c => c.target === 'save'); expect(edge).toBeDefined(); }); + `); + expect(found[0].atoms[0].key).toBeTruthy(); + expect(found[1].atoms[0].key).toBe(found[0].atoms[0].key); + expect(found[2].atoms[0].key).not.toBe(found[0].atoms[0].key); + expect(found[3].atoms[0].key).not.toBe(found[0].atoms[0].key); +}); + +it('retains template text when tracing asserted query chains', () => { + const a = assertions( + "it('a', () => { const paths = items.map(e => `${e.source} -> ${e.target}`); expect(paths.some(p => p.includes('service'))).toBe(true); });", + )[0]; + const b = assertions( + "it('b', () => { const paths = items.map(e => `${e.source} => ${e.target}`); expect(paths.some(p => p.includes('service'))).toBe(true); });", + )[0]; + expect(a.atoms[0].key).toBeTruthy(); + expect(a.atoms[0].key).not.toBe(b.atoms[0].key); +}); + +it('does not infer clone families through near-miss similarity', () => { + const thirdFiles: [string, string][] = [['main.ts', SOURCE.replace('+ 1', '* 2')]]; + const third = sha1(JSON.stringify(thirdFiles)); + const middleFiles: [string, string][] = [['main.ts', SOURCE.replace('+ 1', '- 1')]]; + const middle = sha1(JSON.stringify(middleFiles)); + const span = (source: string) => ({ + source, + file: 'main.ts', + start: 0, + end: 20, + startLine: 1, + endLine: 1, + }); + const run: CloneRun = { + ...noClones, + sourceCount: 3, + clones: [ + { a: span(key), b: span(middle), kind: 'similar', similarity: 0.9, tokens: 12 }, + { a: span(middle), b: span(third), kind: 'similar', similarity: 0.9, tokens: 12 }, + ], + }; + const index = new FixtureCloneIndex( + { ...sources, [middle]: middleFiles, [third]: thirdFiles }, + run, + ); + expect( + index.compare(profile, { ...profile, sourceKeys: [third], contentKey: 'third' }), + ).toBeNull(); +}); + +it('surfaces one shared check in a partial scenario and retains all other expectations', () => { + const found = report( + [test('a'), test('b')], + [ + detail('a', evidence([atom(), atom('a', 'unique-a'), atom('b', 'unique-b')])), + detail( + 'b', + evidence([atom(), atom('c', 'unique-c'), atom('d', 'unique-d'), atom('e', 'unique-e')]), + ), + ], + ).opportunities[0]; + expect(found.assertions).toMatchObject({ shared: 1 }); + expect(found.onlyAssertionsA.length + found.onlyAssertionsB.length).toBe(5); +}); + +it('keeps negative zero distinct and locates the real assertion on a shared line', () => { + const found = assertions(` + it('a', () => { helper(result).toBe(99); expect(result).toBe(-0); }); + it('b', () => { expect(result).toBe(0); }); + `); + expect(found[0].atoms[0].expected).toContain('"text":"-0"'); + expect(found[0].atoms[0].key).not.toBe(found[1].atoms[0].key); +}); + +it.each([ + ['duplicate modifier', 'expect(x).not.not.toBe(2);'], + ['conditional', 'if (flag) expect(x).toBe(2);'], + ['loop', 'for (const value of values) expect(x).toBe(2);'], + ['nested function', 'const helper = () => expect(x).toBe(2);'], + ['unsupported expect arguments', "expect(x, 'message', 'extra').toBe(2);"], +])('uses the supported assertion after a same-line %s', (_kind, prefix) => { + const [mixed, direct] = assertions(` + it('mixed', () => { ${prefix} expect(y).toBe(3); }); + it('direct', () => { expect(y).toBe(3); }); + `); + expect(mixed.atoms).toHaveLength(1); + expect(mixed.atoms[0].observed).toBe('y'); + expect(mixed.atoms[0].key).toBeTruthy(); + expect(mixed.atoms[0].key).toBe(direct.atoms[0].key); + expect(mixed.unresolved.length).toBeGreaterThan(0); +}); + +it('reports missing assertion provenance instead of guessing a call for detached checks', () => { + const [site] = expandCases([ + { + path: 'gitnexus/test/unit/provenance.test.ts', + source: "it('case', () => expect(result).toBe(3));", + }, + ]).sites; + const subject = { + record: site.record, + instance: site.instances?.[0] ?? null, + fullName: site.fullName, + }; + const checks = computeChecks(subject); + expect(assertionEvidence(subject, checks).atoms[0].key).toBeTruthy(); + const detached = { ...checks, checks: checks.checks.map((check) => ({ ...check })) }; + expect(JSON.stringify(detached)).toBe(JSON.stringify(checks)); + expect(assertionEvidence(subject, detached)).toEqual({ + atoms: [], + unresolved: ['1: assertion call not located'], + }); +}); + +it('resolves assertion aliases by lexical binding without importing nested declarations', () => { + const found = assertions(` + it('parameter', (value) => { function helper() { const value = graph.nodes.length; } expect(value).toBe(2); }); + it('direct', () => { expect(graph.nodes.length).toBe(2); }); + it('outer alias', () => { const value = graph.nodes.length; function helper() { const value = graph.edges.length; } expect(value).toBe(2); }); + it('nested block', (value) => { { const value = graph.nodes.length; } expect(value).toBe(2); }); + it.each([7])('row', (value) => { function helper() { const value = 99; } expect(result).toBe(value); }); + it('literal row', () => { expect(result).toBe(7); }); + it('callback', () => { const value = 'outer'; expect(items.map(value => value.name)).toEqual([]); }); + it('renamed callback', () => { expect(items.map(item => item.name)).toEqual([]); }); + `); + expect(found[0].atoms[0].key).toBeNull(); + expect(found[2].atoms[0].key).toBe(found[1].atoms[0].key); + expect(found[2].atoms[0].key).toBeTruthy(); + expect(found[3].atoms[0].key).toBeNull(); + // The shared row parser still treats redeclarations conservatively; never import 99. + expect(found[4].atoms[0].key).toBeNull(); + expect(found[5].atoms[0].key).toBeTruthy(); + expect(found[6].atoms[0].key).toBe(found[7].atoms[0].key); + expect(found[6].atoms[0].key).toBeTruthy(); +}); + +it('uses the last static value of duplicate object keys for literals and rows', () => { + const found = assertions(` + it('duplicates', () => { expect(result).toEqual({ a: 1, a: 2, nested: { x: 3, x: 4 } }); }); + it('effective value', () => { expect(result).toEqual({ nested: { x: 4 }, a: 2 }); }); + it.each([{ a: 1, a: 2, nested: { x: 3, x: 4 } }])('row', (value) => { expect(result).toEqual(value); }); + it('numeric aliases', () => { expect(result).toEqual({ 1: 2, '1': 3 }); }); + it('numeric value', () => { expect(result).toEqual({ '1': 3 }); }); + it('different value', () => { expect(result).toEqual({ a: 1, nested: { x: 4 } }); }); + `); + expect(found[0].atoms[0].key).toBeTruthy(); + expect(found[0].atoms[0].key).toBe(found[1].atoms[0].key); + expect(found[2].atoms[0].key).toBe(found[1].atoms[0].key); + expect(found[3].atoms[0].key).toBe(found[4].atoms[0].key); + expect(found[5].atoms[0].key).not.toBe(found[1].atoms[0].key); +}); diff --git a/gitnexus/test/unit/scenario-map/coverage-inventory.test.ts b/gitnexus/test/unit/scenario-map/coverage-inventory.test.ts new file mode 100644 index 000000000..a40e5bd36 --- /dev/null +++ b/gitnexus/test/unit/scenario-map/coverage-inventory.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { coverageInventory } from '../../../tools/scenario-map/inventory/coverage-inventory.js'; +import type { CoverageWorkFile } from '../../../tools/scenario-map/path-coverage/run-coverage.js'; + +const provenance = { + known: false, + executedFiles: [], + nativeEntries: [], + workerSpawns: [], + excludedCandidates: [], +}; +const work = ( + state: 'passed' | 'skipped', + recorded: boolean, + errors: string[] = [], +): CoverageWorkFile => ({ + schemaVersion: 1, + mode: 'normal', + extraRoots: [], + repeats: 1, + blocks: [], + unhandledErrors: [], + files: [{ file: 'a.test.ts', recorded, errors, unattributed: provenance }], + tests: [ + { + id: 'a.test.ts#a', + taskId: 'a', + file: 'a.test.ts', + fullName: 'a', + location: null, + state, + concurrent: false, + attempts: recorded ? 2 : 0, + blocks: [], + suiteBlocks: [], + provenance, + }, + ], +}); + +describe('runtime inventory accounting', () => { + it('does not certify collection after an unhandled runner error', () => { + const input = work('passed', true); + input.unhandledErrors.push('worker exited unexpectedly'); + expect(coverageInventory(input, ['a.test.ts']).collectionComplete).toBe(false); + }); + + it('counts cases independently of files and exposes prefilter omissions', () => { + const result = coverageInventory(work('passed', true), ['a.test.ts', 'b.test.ts']); + expect(result.tests).toEqual({ collected: 1, passed: 1, failed: 0, skipped: 0, pending: 0 }); + expect(result.files.notScheduled).toEqual(['b.test.ts']); + expect(result.collectionComplete).toBe(false); + }); + + it('accounts for fully skipped benchmarks without pretending they executed', () => { + const result = coverageInventory(work('skipped', false), ['a.test.ts']); + expect(result.collectionComplete).toBe(true); + expect(result.tests).toMatchObject({ collected: 1, passed: 0, skipped: 1 }); + }); + + it('reports collection errors even when the file has an execution record', () => { + const result = coverageInventory(work('passed', true, ['collection failed']), ['a.test.ts']); + expect(result.files.unrecorded).toEqual([]); + expect(result.files.errors).toEqual(['a.test.ts']); + expect(result.collectionComplete).toBe(false); + }); + + it.each([ + { state: 'passed' as const, errors: [] }, + { state: 'skipped' as const, errors: ['collection failed'] }, + ])('does not hide missing records or collection errors: $state $errors', ({ state, errors }) => { + const result = coverageInventory(work(state, false, errors), ['a.test.ts']); + expect(result.files.unrecorded).toEqual(['a.test.ts']); + expect(result.collectionComplete).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/scenario-map/coverage-records.test.ts b/gitnexus/test/unit/scenario-map/coverage-records.test.ts index 40d6a01ee..ac1d812a8 100644 --- a/gitnexus/test/unit/scenario-map/coverage-records.test.ts +++ b/gitnexus/test/unit/scenario-map/coverage-records.test.ts @@ -247,6 +247,23 @@ describe('attribution (design §5.2)', () => { }); describe('buildWorkFile: block ids', () => { + it('retains a failed test’s diagnostics even when its coverage window is missing', () => { + const result = work( + [], + [ + rt('failed', 'fails', 3, { + state: 'failed', + errors: ['AssertionError: expected 1 to equal 0'], + }), + ], + ); + expect(result.tests[0]).toMatchObject({ + state: 'failed', + errors: ['AssertionError: expected 1 to equal 0'], + attempts: 0, + }); + }); + const record = new FileRun() .load() .start('t1', ['f']) diff --git a/gitnexus/test/unit/scenario-map/fingerprint.test.ts b/gitnexus/test/unit/scenario-map/fingerprint.test.ts index 7a021bf80..40fa2f79d 100644 --- a/gitnexus/test/unit/scenario-map/fingerprint.test.ts +++ b/gitnexus/test/unit/scenario-map/fingerprint.test.ts @@ -5,6 +5,9 @@ * dupe-ok keep list, the orientation and determinism. */ import { describe, expect, it } from 'vitest'; +import { assertionEvidence } from '../../../tools/scenario-map/consolidation/assertions.js'; +import { computeChecks } from '../../../tools/scenario-map/facet/checks.js'; +import { expandCases } from '../../../tools/scenario-map/inventory/cases.js'; import type { InputFacet } from '../../../tools/scenario-map/facet/input-facet.js'; import { FINGERPRINT_ENGINE_VERSION, @@ -127,6 +130,233 @@ const CASES: { rule: FingerprintRule; verdict: FingerprintVerdict; edit: WorldEd ]; describe('fingerprint rules', () => { + it('does not confirm unresolved file-level subjects through equal coarse checks and clean traces', () => { + const evidence = expandCases( + [1, 2].map((value) => ({ + path: `gitnexus/test/unit/constant-${value}.test.ts`, + source: `const result = ${value}; it('rule', () => expect(result).toBeGreaterThan(0));`, + })), + ).sites.map((site) => { + const subject = { + record: site.record, + instance: site.instances?.[0] ?? null, + fullName: site.fullName, + }; + return assertionEvidence(subject, computeChecks(subject)); + }); + expect( + verdictOf({ + a: { checks: 'same-check', assertionEvidence: evidence[0] }, + b: { checks: 'same-check', assertionEvidence: evidence[1] }, + }), + ).toMatchObject({ verdict: 'possible-duplicate', rule: 'input-unresolved' }); + }); + + it.each([ + ['roundScore(2.4)', 'roundScore(2.5)'], + ['roundScore({ scores: [2.4] })', 'roundScore({ scores: [2.5] })'], + ])('keeps changed literal inputs to %s as table rows', (left, right) => { + const evidence = expandCases([ + { + path: 'gitnexus/test/unit/input-shapes.test.ts', + source: `import { roundScore } from './scores.js'; + it('left', () => { const actual = ${left}; expect(actual).toBe(2); }); + it('right', () => { const actual = ${right}; expect(actual).toBe(3); });`, + }, + ]).sites.map((site) => { + const subject = { + record: site.record, + instance: site.instances![0], + fullName: site.fullName, + }; + return assertionEvidence(subject, computeChecks(subject)); + }); + expect(evidence.every((item) => item.atoms[0].key !== null)).toBe(true); + expect(evidence[0].atoms[0].key).not.toBe(evidence[1].atoms[0].key); + const inputs = { + a: { checks: 'same-check', assertionEvidence: evidence[0], facet: facet(['2.4', '2']) }, + b: { checks: 'same-check', assertionEvidence: evidence[1], facet: facet(['2.5', '3']) }, + }; + expect(verdictOf(inputs)).toMatchObject({ verdict: 'table-row', rule: 'input-values-differ' }); + }); + + it.each([ + [ + "getNodes(result).find(n => n.name === 'save')", + "getNodes(result).find(n => n.name === 'load')", + ], + ["getNodes(result, 'CALLS')", "getNodes(result, 'IMPORTS')"], + ['getNodes(/save/)', 'getNodes(/load/)'], + ['getNodes({ patterns: [/save/] })', 'getNodes({ patterns: [/load/] })'], + ['getNodes(2).length', 'getEdges(2).length'], + ['getNodes(2).length', 'getNodes(2).size'], + ])('case %#: preserves queried subjects when comparing %s', (left, right) => { + const evidence = expandCases([ + { + path: 'gitnexus/test/unit/queried-subjects.test.ts', + source: `import { getNodes, getEdges } from './graph.js'; + it('left', () => { const actual = ${left}; expect(actual).toBeDefined(); }); + it('right', () => { const actual = ${right}; expect(actual).toBeDefined(); });`, + }, + ]).sites.map((site) => { + const subject = { + record: site.record, + instance: site.instances![0], + fullName: site.fullName, + }; + return assertionEvidence(subject, computeChecks(subject)); + }); + expect( + verdictOf({ + a: { checks: 'coarse', assertionEvidence: evidence[0] }, + b: { checks: 'coarse', assertionEvidence: evidence[1] }, + }), + ).toMatchObject({ verdict: 'distinct', rule: 'checks-differ' }); + }); + + it('does not promote different assertion subjects or literals through a coarse checks key', () => { + const assertions = (shapeKey: string, expected: string, known = true) => ({ + atoms: [ + { + line: 1, + shape: 'expect().toContain()', + shapeKey, + key: known ? `${shapeKey}:${expected}` : null, + expected, + }, + ], + unresolved: known ? [] : ['asserted local origin is unresolved'], + }); + const a = { checks: 'coarse', facet: null, assertionEvidence: assertions('nodes', 'greet') }; + for (const b of [assertions('text', 'greet'), assertions('nodes', 'function twice', false)]) { + expect( + verdictOf({ + a, + b: { checks: 'coarse', checksContext: 'other', facet: null, assertionEvidence: b }, + }), + ).toMatchObject({ verdict: 'distinct', rule: 'checks-differ' }); + } + expect(verdictOf({ a, b: { ...a, checksContext: 'other' } }).verdict).toBe( + 'possible-duplicate', + ); + }); + + it('retains existing table rows outside the actionable pair list without hiding other declarations', () => { + const result = computeFingerprints({ + tests: [ + testInput(A, { declarationKey: 'same-table', facet: null }), + testInput(B, { declarationKey: 'same-table', facet: null }), + ], + pairs: [pairInput()], + }); + expect(result.pairs).toEqual([]); + expect(result.kept[0].keep[0].reason).toContain('already parameterized'); + expect( + verdictOf({ a: { declarationKey: 'first' }, b: { declarationKey: 'second' } }).verdict, + ).toBe('duplicate'); + }); + it('leaves proven duplicate rows actionable and uncertain helper equivalence unresolved', () => { + const identical = computeFingerprints({ + tests: [ + testInput(A, { declarationKey: 'same-table' }), + testInput(B, { declarationKey: 'same-table' }), + ], + pairs: [pairInput()], + }); + expect(identical.pairs[0]?.verdict).toBe('duplicate'); + expect(identical.kept).toEqual([]); + const evidence = (file: string) => ({ + atoms: [ + { + line: 1, + shape: 'expect(run()).toBe(1)', + shapeKey: file, + key: file, + expected: '1', + observed: `()`, + }, + ], + unresolved: [], + }); + expect( + verdictOf({ + a: { assertionEvidence: evidence('a') }, + b: { assertionEvidence: evidence('b') }, + }).verdict, + ).toBe('possible-duplicate'); + }); + it.each(['a', 'b'] as const)( + 'does not compare an assertion-only key on side %s with a legacy combined key', + (side) => { + const pair = verdictOf({ [side]: { checks: 'checks-toBe' } }); + expect([pair.verdict, pair.rule, pair.evidence.checks]).toEqual([ + 'possible-duplicate', + 'checks-unknown', + 'missing', + ]); + }, + ); + + it('keeps equal assertions with different setup as a review lead, not different checks', () => { + const pair = verdictOf({ + a: { checks: 'same-expectation' }, + b: { checks: 'same-expectation', checksContext: 'cleanup-context' }, + }); + expect([pair.verdict, pair.rule, pair.evidence.checks]).toEqual([ + 'possible-duplicate', + 'context-differs', + 'same', + ]); + }); + + it('cannot confirm a duplicate when assertion-only keys exist but context is missing', () => { + const pair = verdictOf({ + a: { checks: 'same-expectation', checksContext: null }, + b: { checks: 'same-expectation', checksContext: null }, + }); + expect([pair.verdict, pair.rule]).toEqual(['possible-duplicate', 'context-unknown']); + }); + + it('retains known input differences when setup also differs', () => { + const pair = verdictOf({ + a: { checks: 'same-expectation' }, + b: { + checks: 'same-expectation', + checksContext: 'other-setup', + facet: facet(['Number[]', 'Number']), + }, + }); + expect([pair.verdict, pair.rule]).toEqual(['table-row', 'input-values-differ']); + }); + + it.each(['args-differ', 'receiver-differs'] as const)( + 'retains a proven runtime value difference (%s) when setup also differs', + (reason) => { + const pair = verdictOf({ + a: { checks: 'same-expectation' }, + b: { checks: 'same-expectation', checksContext: 'other-setup' }, + pair: { trace: { status: 'veto', reasons: [reason] } }, + }); + expect([pair.verdict, pair.rule]).toEqual(['table-row', 'runtime-values-differ']); + }, + ); + + it('does not prove distinctness from incomplete native/worker paths', () => { + const pair = verdictOf({ + a: { provenance: { known: true, nativeEntries: ['tree-sitter.node'], workerSpawns: [] } }, + pair: { identicalBlocks: false }, + }); + expect([pair.verdict, pair.rule]).toEqual(['possible-duplicate', 'path-incomplete']); + }); + + it.each(['asymmetric-sites', 'asymmetric-counts'] as const)( + 'does not call a cold/warm trace difference (%s) different input values', + (reason) => { + const pair = verdictOf({ pair: { trace: { status: 'veto', reasons: [reason] } } }); + expect([pair.verdict, pair.rule]).toEqual(['possible-duplicate', 'trace-unknown']); + }, + ); + it.each(CASES)('rule $rule decides $verdict', ({ rule, verdict, edit }) => { const decided = verdictOf(edit); expect([decided.verdict, decided.rule]).toEqual([verdict, rule]); diff --git a/gitnexus/test/unit/scenario-map/fixture-similarity.test.ts b/gitnexus/test/unit/scenario-map/fixture-similarity.test.ts new file mode 100644 index 000000000..550142873 --- /dev/null +++ b/gitnexus/test/unit/scenario-map/fixture-similarity.test.ts @@ -0,0 +1,365 @@ +import { describe, expect, it } from 'vitest'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { + FixtureProfiler, + compareFixtureProfiles, + fixtureSimilarityReport, +} from '../../../tools/scenario-map/facet/fixture-profile.js'; +import { + FixtureInputs, + withFixtureIdentity, +} from '../../../tools/scenario-map/facet/fixture-inputs.js'; +import { expandCases } from '../../../tools/scenario-map/inventory/cases.js'; +import { findCandidates } from '../../../tools/scenario-map/path-coverage/candidates.js'; +import { + computeFingerprints, + type FingerprintTestInput, +} from '../../../tools/scenario-map/verdict/fingerprint.js'; +import type { + FacetSources, + FacetSubject, + InputFacet, +} from '../../../tools/scenario-map/facet/input-facet.js'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { layer1Files } from '../../../tools/scenario-map/path-coverage/prefilter.js'; +import { isLanguageAvailable } from '../../../src/core/tree-sitter/parser-loader.js'; + +const ROOT = 'gitnexus/test/fixtures'; +const FILE = 'gitnexus/test/integration/example.test.ts'; +const sources = (files: Record): FacetSources => ({ + read: (file) => files[file] ?? null, + list: (dir) => + Object.keys(files) + .filter((file) => file.startsWith(`${dir}/`)) + .sort(), +}); +const subjects = (source: string): FacetSubject[] => + expandCases([{ path: FILE, source }]).sites.flatMap((site) => + (site.instances ?? []).map((instance, index) => ({ + id: `${site.fullName}:${index}`, + file: site.file, + record: site.record, + instance, + executedFiles: [], + })), + ); +const FIXTURES = { + [`${ROOT}/ts/main.ts`]: 'export function double(x: number) { return x + x; }', + [`${ROOT}/renamed/main.ts`]: + '// renamed\nexport function twice(value: number) { return value + value; }', + [`${ROOT}/py/main.py`]: 'def twice(value):\n return value + value\n', + [`${ROOT}/loop/main.ts`]: + 'export function twice(value: number) { while (value > 0) { value--; } return value; }', +}; + +// Exercise the real native providers, including standalone COBOL and embedded Vue scripts. +const LANGUAGE_FIXTURES: Record = { + typescript: ['ts', 'function twice(x: number) { return x + x; }'], + javascript: ['js', 'function twice(x) { return x + x; }'], + python: ['py', 'def twice(x):\n return x + x\n'], + go: ['go', 'package main\nfunc twice(x int) int { return x + x }'], + rust: ['rs', 'fn twice(x: i32) -> i32 { return x + x; }'], + java: ['java', 'class Sample { static int twice(int x) { return x + x; } }'], + csharp: ['cs', 'class Sample { static int Twice(int x) { return x + x; } }'], + c: ['c', 'int twice(int x) { return x + x; }'], + cpp: ['cpp', 'int twice(int x) { return x + x; }'], + kotlin: ['kt', 'fun twice(x: Int): Int { return x + x }'], + php: ['php', ' Int { return x + x }'], + dart: ['dart', 'int twice(int x) { return x + x; }'], + zig: ['zig', 'fn twice(x: i32) i32 { return x + x; }'], + 'objective-c': [ + 'm', + '@interface Sample\n- (int)twice:(int)x;\n@end\n@implementation Sample\n- (int)twice:(int)x { return x + x; }\n@end', + ], + vue: [ + 'vue', + '', + ], + cobol: [ + 'cbl', + ' IDENTIFICATION DIVISION.\n PROGRAM-ID. SAMPLE.\n PROCEDURE DIVISION.\n MAIN-PARA.\n DISPLAY "HI".\n STOP RUN.\n', + ], +}; + +it.each(Object.entries(LANGUAGE_FIXTURES))( + 'profiles real %s fixtures with the ingestion provider', + async (language, [extension, content]) => { + const root = `${ROOT}/language-${language}`; + const profiler = new FixtureProfiler(sources({ [`${root}/main.${extension}`]: content })); + const result = await profiler.profile([root]); + if ( + language !== SupportedLanguages.Cobol && + !isLanguageAvailable(language as SupportedLanguages, `main.${extension}`) + ) { + expect(result.contentKey).not.toBeNull(); + expect(result.unresolved.length).toBeGreaterThan(0); + return; + } + expect(result.unresolved).toEqual([]); + expect(result.languages).toEqual([language]); + expect(result.features.some(([feature]) => feature.startsWith('def:'))).toBe(true); + expect(result.contentKey).not.toBeNull(); + }, +); + +describe('fixture similarity', () => { + it('retrieves renamed and cross-language fixtures, without conflating a loop with straight-line code', async () => { + const profiler = new FixtureProfiler(sources(FIXTURES)); + const [ts, renamed, py, loop] = await Promise.all( + ['ts', 'renamed', 'py', 'loop'].map((name) => profiler.profile([`${ROOT}/${name}`])), + ); + expect(compareFixtureProfiles(ts, renamed)).toEqual({ + kind: 'structure', + score: 1, + crossLanguage: false, + }); + expect(compareFixtureProfiles(ts, py)).toEqual({ + kind: 'structure', + score: 1, + crossLanguage: true, + }); + expect(compareFixtureProfiles(ts, loop)).toBeNull(); + expect(new Set([ts.contentKey, renamed.contentKey, py.contentKey]).size).toBe(3); + const candidate = (id: string, fixture: typeof ts, checks = 'assertion') => ({ + id, + blocks: [id.charCodeAt(0)], + concurrent: false, + checksContext: id, + bodyKey: id, + checks, + fixture, + }); + const found = findCandidates([ + candidate('a', ts), + candidate('b', py), + candidate('c', loop), + candidate('d', py, 'different'), + ]); + expect(found.pairs).toHaveLength(1); + expect(found.pairs[0]).toMatchObject({ + a: 'a', + b: 'b', + sources: ['fixture'], + fixtureSimilarity: { crossLanguage: true }, + }); + expect(found.stats.bySource.fixture).toBe(1); + const report = fixtureSimilarityReport( + [ + { id: 'a', fixture: ts }, + { id: 'b', fixture: py }, + { id: 'another', fixture: ts }, + ], + found.pairs, + ); + expect(report.summary).toMatchObject({ + tests: 3, + profiles: 2, + matches: 1, + crossLanguageMatches: 1, + }); + expect(report.profiles.find((entry) => entry.profile === ts)?.tests).toEqual(['a', 'another']); + expect(findCandidates([candidate('a', ts), candidate('b', py)], () => false).pairs).toEqual([]); + }); + + it('keeps file multiplicity, layout, and non-source assets in exact input identity', async () => { + const profiler = new FixtureProfiler( + sources({ + [`${ROOT}/a/main.ts`]: FIXTURES[`${ROOT}/ts/main.ts`], + [`${ROOT}/b/main.ts`]: FIXTURES[`${ROOT}/ts/main.ts`], + [`${ROOT}/c/nested/main.ts`]: FIXTURES[`${ROOT}/ts/main.ts`], + [`${ROOT}/d/main.ts`]: FIXTURES[`${ROOT}/ts/main.ts`], + [`${ROOT}/d/package.json`]: '{"type":"module"}', + [`${ROOT}/e/one.ts`]: FIXTURES[`${ROOT}/ts/main.ts`], + [`${ROOT}/e/two.ts`]: FIXTURES[`${ROOT}/ts/main.ts`], + }), + ); + const [a, b, c, d, e] = await Promise.all( + ['a', 'b', 'c', 'd', 'e'].map((name) => profiler.profile([`${ROOT}/${name}`])), + ); + expect(compareFixtureProfiles(a, b)?.kind).toBe('content'); + expect(new Set([a.contentKey, c.contentKey, d.contentKey, e.contentKey]).size).toBe(4); + expect(compareFixtureProfiles(a, e)).toBeNull(); + }); + + it('reports missing, malformed, and oversized fixtures instead of matching partial profiles', async () => { + const profiler = new FixtureProfiler( + sources({ + ...FIXTURES, + [`${ROOT}/broken/main.ts`]: 'export function () { return (', + [`${ROOT}/large/main.ts`]: ' '.repeat(200_001), + }), + ); + const baseline = await profiler.profile([`${ROOT}/ts`]); + for (const root of ['missing', 'broken', 'large']) { + const result = await profiler.profile([`${ROOT}/${root}`]); + if (root === 'broken') { + expect(result.contentKey).not.toBeNull(); + expect(compareFixtureProfiles(result, result)?.kind).toBe('content'); + } else expect(result.contentKey).toBeNull(); + expect(result.unresolved.length).toBeGreaterThan(0); + expect(compareFixtureProfiles(result, baseline)).toBeNull(); + } + }); +}); + +it('retains fixture partners in diff selection even when their setup and callbacks differ', () => { + const root = mkdtempSync(path.join(tmpdir(), 'fixture-similarity-')); + try { + const files = ['test/integration/a.test.ts', 'test/integration/b.test.ts']; + files.forEach((file, i) => { + const target = path.join(root, 'gitnexus', file); + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync( + target, + `import path from 'node:path'; + beforeAll(() => setup(${i})); + it('test', () => { const content${i} = readFileSync(path.join(__dirname, '..', 'fixtures', '${i}.ts')); expect(content${i}).toBeDefined(); });`, + ); + mkdirSync(path.join(root, 'gitnexus/test/fixtures'), { recursive: true }); + writeFileSync( + path.join(root, `gitnexus/test/fixtures/${i}.ts`), + `function f${i}() { return 1; }`, + ); + }); + expect(layer1Files(root, files, new Set([files[0]]))).toEqual({ files, skipped: [] }); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +describe('fixture input discovery', () => { + it('does not resolve an unknown helper/callback argument using a shadowed outer constant', () => { + const inputs = new FixtureInputs(sources(FIXTURES)); + const [subject] = subjects(` + const root = '${ROOT}/ts'; + const load = (root) => runPipelineFromRepo(root); + it('unknown', () => { load(dynamicRoot()); values.forEach((root) => readFileSync(root)); }); + `); + expect(inputs.resolve(subject).roots).toEqual([]); + expect(inputs.resolve(subject).unresolved).toHaveLength(2); + }); + it('follows imported roots and local setup constants without collecting assertion strings', async () => { + const inputs = new FixtureInputs( + sources({ + ...FIXTURES, + 'gitnexus/test/integration/helpers.ts': `import path from 'node:path'; export const FIXTURES = path.resolve(__dirname, '..', 'fixtures');`, + }), + ); + const [subject] = subjects(` + import path from 'node:path'; import { FIXTURES } from './helpers.js'; + describe('suite', () => { + beforeAll(() => { const root = path.join(FIXTURES, 'ts'); runPipelineFromRepo(root); }); + it('one', () => { expect(result).toContain('${ROOT}/py'); }); + }); + `); + expect(inputs.resolve(subject)).toEqual({ roots: [`${ROOT}/ts`], unresolved: [] }); + expect((await inputs.profile(subject))?.languages).toEqual(['typescript']); + }); + + it('resolves per-row directories and both calls to an imported helper', () => { + const inputs = new FixtureInputs( + sources({ + ...FIXTURES, + 'gitnexus/test/integration/helpers.ts': `import path from 'node:path'; export function analyzeFixture(name) { return runPipelineFromRepo(path.join(__dirname, '..', 'fixtures', name)); }`, + }), + ); + const rows = subjects(`import path from 'node:path'; it.each(['ts','py'])('row %s', (name) => { + const root = path.join(__dirname, '..', 'fixtures', name); runPipelineFromRepo(root); expect(result).toBeDefined(); + });`); + expect(rows.map((row) => inputs.resolve(row).roots)).toEqual([[`${ROOT}/ts`], [`${ROOT}/py`]]); + const [helper] = subjects(`import { analyzeFixture } from './helpers.js'; it('both', () => { + analyzeFixture('ts'); analyzeFixture('py'); expect(result).toBeDefined(); + });`); + expect(inputs.resolve(helper)).toEqual({ roots: [`${ROOT}/py`, `${ROOT}/ts`], unresolved: [] }); + }); + + it('handles file URL inputs and exposes dynamically chosen inputs', () => { + const inputs = new FixtureInputs(sources(FIXTURES)); + const [known, unknown] = + subjects(`it('url', () => { readFileSync(new URL('../fixtures/ts/main.ts', import.meta.url)); }); + it('dynamic', () => { runPipelineFromRepo(selectFixture()); });`); + expect(inputs.resolve(known)).toEqual({ roots: [`${ROOT}/ts/main.ts`], unresolved: [] }); + expect(inputs.resolve(unknown).unresolved).toEqual([ + expect.stringContaining('unresolved fixture input'), + ]); + }); + + it('does not crash on spread arguments and resolves aliased path and ingestion imports', () => { + const inputs = new FixtureInputs(sources(FIXTURES)); + const [spread, alias] = subjects(` + import { join as combine } from 'node:path'; import { runPipelineFromRepo as ingest } from '../../src/core/ingestion/pipeline.js'; + it('spread', () => { runPipelineFromRepo(...args); }); + it('alias', () => { ingest(combine(__dirname, '..', 'fixtures', 'ts')); }); + `); + expect(inputs.resolve(spread).unresolved).toHaveLength(1); + expect(inputs.resolve(alias)).toEqual({ roots: [`${ROOT}/ts`], unresolved: [] }); + }); + + it('handles bigint rows and keeps hook constants separate from test-local shadowing', () => { + const inputs = new FixtureInputs(sources(FIXTURES)); + const rows = + subjects(`import path from 'node:path'; const root = path.join(__dirname, '..', 'fixtures', 'ts'); + beforeAll(() => runPipelineFromRepo(root)); + it.each([1n, 2n])('bigint %s', (n) => { const root = 'not-a-fixture'; expect(n).toBeDefined(); });`); + expect(rows.map((row) => inputs.resolve(row))).toEqual([ + { roots: [`${ROOT}/ts`], unresolved: [] }, + { roots: [`${ROOT}/ts`], unresolved: [] }, + ]); + }); +}); + +it.each([ + ['ts', 'py', 'input-shape-differs'], + ['broken', 'broken', 'input-unresolved'], +])( + 'does not confirm %s/%s fixtures despite equal paths, assertions, and clean traces', + async (left, right, rule) => { + const profiler = new FixtureProfiler( + sources({ ...FIXTURES, [`${ROOT}/broken/main.ts`]: 'function () {' }), + ); + const [a, b] = await Promise.all( + [left, right].map((name) => profiler.profile([`${ROOT}/${name}`])), + ); + const facet: InputFacet = { + fingerprint: 'same', + parts: { language: '-', source: [], fixtures: [], data: [], calls: [], order: [] }, + discarded: [], + unresolved: [], + skeleton: [], + }; + const test = (id: string, profile: typeof a): FingerprintTestInput => ({ + id, + fingerprintEligible: true, + passed: true, + dupeOk: null, + joinProblems: [], + inventoryUnresolved: [], + provenance: { known: true, nativeEntries: [], workerSpawns: [] }, + concurrent: false, + checksContext: 'same', + checks: 'same', + unsupportedForms: [], + unresolvedSelectors: [], + facet: withFixtureIdentity(facet, profile), + }); + const result = computeFingerprints({ + tests: [test('a', a), test('b', b)], + pairs: [ + { + a: 'a', + b: 'b', + group: 'g', + identicalBlocks: true, + pathUnattributable: false, + trace: { status: 'clean', reasons: [] }, + }, + ], + }); + expect(result.pairs[0].verdict).not.toBe('duplicate'); + expect(result.pairs[0].rule).toBe(rule); + }, +); diff --git a/gitnexus/test/unit/scenario-map/partitions.test.ts b/gitnexus/test/unit/scenario-map/partitions.test.ts index eccda8ddf..6e2404afe 100644 --- a/gitnexus/test/unit/scenario-map/partitions.test.ts +++ b/gitnexus/test/unit/scenario-map/partitions.test.ts @@ -5,6 +5,7 @@ * glob matching, the split, the merge of partial runs, the partitioned runner and the build output * copied into a prepared package. */ +import { execFileSync } from 'node:child_process'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -182,6 +183,73 @@ describe('copyBuildOutput', () => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); }); + it('gives inventory guards a private index without sharing Git configuration or writes', () => { + const base = mkdtempSync(path.join(os.tmpdir(), 'scenario-map-git-inventory-')); + temps.push(base); + const checkout = path.join(base, 'checkout'); + const prepared = path.join(base, 'prepared'); + for (const root of [checkout, prepared]) { + mkdirSync(path.join(root, 'gitnexus'), { recursive: true }); + writeFileSync(path.join(root, 'gitnexus', 'case.ts'), 'export const value = 1;'); + } + writeFileSync(path.join(checkout, 'public-manifest.json'), '{}'); + writeFileSync(path.join(checkout, '.env'), 'fixture-only-private-value'); + writeFileSync(path.join(prepared, 'gitnexus', 'case.ts'), 'export const value = 2;'); + execFileSync('git', ['init', '--quiet', checkout]); + const git = (root: string, ...args: string[]) => + execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim(); + git(checkout, 'add', '--force', '--', 'gitnexus/case.ts', 'public-manifest.json', '.env'); + git(checkout, 'remote', 'add', 'origin', 'https://example.com/source.git'); + copyBuildOutput('normal', path.join(checkout, 'gitnexus'), path.join(prepared, 'gitnexus')); + expect(git(prepared, 'rev-parse', '--show-toplevel')).toBe(prepared.split(path.sep).join('/')); + expect(git(prepared, 'ls-files', '--stage')).toBe( + git(checkout, 'ls-files', '--stage', '--', 'gitnexus/case.ts', 'public-manifest.json'), + ); + expect(git(prepared, 'remote')).toBe(''); + expect(existsSync(path.join(prepared, '.env'))).toBe(false); + expect(readFileSync(path.join(prepared, 'public-manifest.json'), 'utf8')).toBe('{}'); + expect(readFileSync(path.join(prepared, 'gitnexus', 'case.ts'), 'utf8')).toContain('value = 2'); + git(prepared, 'update-index', '--force-remove', 'gitnexus/case.ts'); + expect(git(checkout, 'ls-files').split('\n')).toEqual([ + '.env', + 'gitnexus/case.ts', + 'public-manifest.json', + ]); + }); + + it('preserves repository contract fixtures needed during test collection', () => { + const base = mkdtempSync(path.join(os.tmpdir(), 'scenario-map-contract-fixtures-')); + temps.push(base); + const checkout = path.join(base, 'checkout', 'gitnexus'); + const prepared = path.join(base, 'prepared', 'gitnexus'); + mkdirSync(checkout, { recursive: true }); + mkdirSync(prepared, { recursive: true }); + const fixtures = [ + '.github/workflows/ci-tests.yml', + '.claude/skills/gitnexus-plan/SKILL.md', + 'gitnexus-factory-plugin/.factory-plugin/plugin.json', + 'Dockerfile.cli', + 'render.yaml', + 'AGENTS.md', + 'CLAUDE.md', + 'eval/workflow_bench/run-evolution.sh', + 'gitnexus/prebuilds/bundled/manifest.json', + ]; + for (const file of fixtures) { + const target = path.join(checkout, '..', file); + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync(target, file); + } + writeFileSync(path.join(checkout, '..', '.env'), 'private'); + execFileSync('git', ['init', '--quiet', path.dirname(checkout)]); + execFileSync('git', ['add', '--force', '--', ...fixtures], { cwd: path.dirname(checkout) }); + copyBuildOutput('normal', checkout, prepared); + expect(fixtures.map((file) => readFileSync(path.join(prepared, '..', file), 'utf8'))).toEqual( + fixtures, + ); + expect(existsSync(path.join(prepared, '..', '.env'))).toBe(false); + }); + it('copies vendor/ and dist/ that the checkout has into a normal-mode copy only', () => { const base = mkdtempSync(path.join(os.tmpdir(), 'scenario-map-build-output-')); temps.push(base); @@ -218,6 +286,8 @@ describe('copyBuildOutput', () => { mkdirSync(dir, { recursive: true }); writeFileSync(path.join(dir, 'registry-query.cjs'), plugin); } + execFileSync('git', ['init', '--quiet', path.dirname(checkout)]); + execFileSync('git', ['add', '--', ...hooks], { cwd: path.dirname(checkout) }); copyBuildOutput('normal', checkout, prepared); expect( hooks.map((plugin) => diff --git a/gitnexus/test/unit/scenario-map/recall-audit.test.ts b/gitnexus/test/unit/scenario-map/recall-audit.test.ts new file mode 100644 index 000000000..7e9f53b90 --- /dev/null +++ b/gitnexus/test/unit/scenario-map/recall-audit.test.ts @@ -0,0 +1,53 @@ +/** Source-adjudicated pairs from the PR #3686 audit. Retrieval is not a duplicate verdict. */ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { bodyKeyOf } from '../../../tools/scenario-map/facet/body-key.js'; +import { + checksContextKey, + fsModuleReader, + fsSnapshotReader, +} from '../../../tools/scenario-map/facet/checks.js'; +import { loadStaticInventory } from '../../../tools/scenario-map/inventory/cases.js'; +import { findCandidates } from '../../../tools/scenario-map/path-coverage/candidates.js'; +import { layer1Files } from '../../../tools/scenario-map/path-coverage/prefilter.js'; + +const root = fileURLToPath(new URL('../../../../', import.meta.url)); +const audit = JSON.parse( + readFileSync(new URL('../../fixtures/scenario-map-recall/audit.json', import.meta.url), 'utf8'), +) as { + positives: { a: string; b: string; category: string; basis: string }[]; +}; +const ids = new Set(audit.positives.flatMap(({ a, b }) => [a, b])); +const files = [...new Set([...ids].map((id) => id.slice(0, id.indexOf('#'))))]; +const inventory = loadStaticInventory(root, files); +const options = { modules: fsModuleReader(root), snapshots: fsSnapshotReader(root) }; +const sites = new Map(inventory.sites.map((site) => [`${site.file}#${site.fullName}`, site])); + +describe('manual recall audit: candidate retrieval', () => { + it('includes the audited files despite differing hooks and re-export contexts', () => { + expect(layer1Files(root, files).skipped).toEqual([]); + }); + + it.each(audit.positives)( + 'case %#: retrieves $a and $b without relying on warm setup coverage', + ({ a, b }) => { + const tests = [a, b].map((id) => { + const site = sites.get(id); + expect(site, `audited test moved or disappeared: ${id}`).toBeDefined(); + if (!site) throw new Error(id); + return { + id, + blocks: [], // Do not let Jaccard hide a static retrieval regression. + concurrent: false, + checksContext: checksContextKey( + { record: site.record, instance: site.instances?.[0] ?? null, fullName: site.fullName }, + options, + ), + bodyKey: bodyKeyOf(site.record), + }; + }); + expect(findCandidates(tests).pairs).toHaveLength(1); + }, + ); +}); diff --git a/gitnexus/test/unit/scenario-map/report-json.test.ts b/gitnexus/test/unit/scenario-map/report-json.test.ts new file mode 100644 index 000000000..e5ed5c452 --- /dev/null +++ b/gitnexus/test/unit/scenario-map/report-json.test.ts @@ -0,0 +1,124 @@ +/** Full-suite reports exceed V8's string limit; each record must survive streamed I/O. */ +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { compareWithBaseline } from '../../../tools/scenario-map/report/baseline.js'; +import { readReportFields, writeReportJson } from '../../../tools/scenario-map/report/json.js'; +import { buildReport } from '../../../tools/scenario-map/report/report.js'; +import { reportInput } from './report-fixtures.js'; + +const directories: string[] = []; +function output(): string { + const directory = mkdtempSync(path.join(os.tmpdir(), 'scenario-report-json-')); + directories.push(directory); + return path.join(directory, 'report.json'); +} + +afterEach(() => { + vi.restoreAllMocks(); + for (const directory of directories.splice(0)) + rmSync(directory, { recursive: true, force: true }); +}); + +describe('report JSON', () => { + it('preserves every field and record without serializing a whole collection', async () => { + const report = buildReport(reportInput()); + const pair = report.fingerprint.pairs[0]; + report.fingerprint.pairs = Array.from({ length: 100 }, (_, i) => ({ + ...pair, + a: `test-${i}`, + details: ['escaped \"quote\", \\slash, \nnewline and unicode 🧪'], + })); + report.fingerprint.kept = report.fingerprint.pairs.map((entry) => ({ + ...entry, + keep: [{ test: entry.a, reason: 'intentional control' }], + })); + report.fingerprint.pairs[0].details = Array(1_000).fill('nested record detail'); + const file = output(); + const stringify = JSON.stringify; + // Model the V8 cap at a small scale, including a collection nested inside one record. + const limit = 16_384; + expect(stringify(report.fingerprint.pairs).length).toBeGreaterThan(limit); + expect(stringify(report.fingerprint.kept).length).toBeGreaterThan(limit); + expect(stringify(report.fingerprint.pairs[0]).length).toBeGreaterThan(limit); + const capped = vi.spyOn(JSON, 'stringify').mockImplementation((value, replacer, space) => { + const text = stringify(value, replacer, space); + if (text !== undefined && text.length > limit) throw new RangeError('Invalid string length'); + return text; + }); + await writeReportJson(file, report); + capped.mockRestore(); + expect(JSON.parse(readFileSync(file, 'utf8'))).toEqual(report); + expect(readdirSync(path.dirname(file))).toEqual(['report.json']); + }); + + it('leaves the prior report intact and removes the temporary file on serialization failure', async () => { + const report = buildReport(reportInput()); + const file = output(); + writeFileSync(file, '{"previous":true}'); + vi.spyOn(JSON, 'stringify').mockImplementation(() => { + throw new Error('cannot serialize'); + }); + await expect(writeReportJson(file, report)).rejects.toThrow('cannot serialize'); + expect(readFileSync(file, 'utf8')).toBe('{"previous":true}'); + expect(readdirSync(path.dirname(file))).toEqual(['report.json']); + }); + + it('propagates filesystem errors', async () => { + const file = output(); + await expect(writeReportJson(path.dirname(file), buildReport(reportInput()))).rejects.toThrow(); + expect(readdirSync(path.dirname(file))).toEqual([]); + }); + + it.each(['streamed', 'legacy'] as const)( + 'reads a %s baseline without retaining findings', + async (format) => { + const report = buildReport(reportInput()); + const file = output(); + if (format === 'streamed') await writeReportJson(file, report); + else writeFileSync(file, JSON.stringify(report, null, 2)); + const baseline = await readReportFields(file, ['schemaVersion', 'run', 'continuity']); + expect(baseline).toEqual({ + schemaVersion: report.schemaVersion, + run: report.run, + continuity: report.continuity, + }); + expect(compareWithBaseline(report.continuity, baseline)).toEqual( + compareWithBaseline(report.continuity, report), + ); + expect(await readReportFields(file, ['run', 'completeness'])).toEqual({ + run: report.run, + completeness: report.completeness, + }); + }, + ); + + it('preserves UTF-8 and escapes across input chunks while skipping large unselected fields', async () => { + const file = output(); + const run = { id: '🧪\n"\\é'.repeat(30_000) }; + writeFileSync(file, JSON.stringify({ ignored: Array(100).fill(run), run, completeness: null })); + expect(await readReportFields(file, ['run', 'completeness'])).toEqual({ + run, + completeness: null, + }); + }); + + it.each([ + ['empty', ''], + ['whitespace', ' \n\t'], + ['truncated selected field', '{"run":{"id":"1"}'], + ['truncated ignored field', '{"run":{"id":"1"},"ignored":[1,'], + ['invalid ignored field', '{"run":{"id":"1"},"ignored":!}'], + ['trailing garbage', '{"run":{"id":"1"}} invalid'], + ['second document', '{"run":{"id":"1"}} {}'], + ])('rejects %s instead of trusting metadata parsed before the error', async (_name, source) => { + const file = output(); + writeFileSync(file, source); + await expect(readReportFields(file, ['run'])).rejects.toThrow(); + }); + + it('propagates a missing file so merge can mark the baseline unreadable', async () => { + await expect(readReportFields(output(), ['run'])).rejects.toThrow(/ENOENT/); + }); +}); diff --git a/gitnexus/test/unit/scenario-map/report.test.ts b/gitnexus/test/unit/scenario-map/report.test.ts index 5fb8434de..39001e98b 100644 --- a/gitnexus/test/unit/scenario-map/report.test.ts +++ b/gitnexus/test/unit/scenario-map/report.test.ts @@ -354,7 +354,7 @@ describe('renderMarkdown', () => { { scope: 'a whole-suite run that skipped files', overrides: { layer1Files: 0 }, - lines: ['Layer 1 ran 0 of them; every test of the other 1 has a checks-and-context key'], + lines: ['Layer 1 ran 0 of them; the other 1 file was not compared.'], }, { scope: 'a diff-scoped run', diff --git a/gitnexus/test/unit/scenario-map/runtime-fixtures.test.ts b/gitnexus/test/unit/scenario-map/runtime-fixtures.test.ts new file mode 100644 index 000000000..ad44666d1 --- /dev/null +++ b/gitnexus/test/unit/scenario-map/runtime-fixtures.test.ts @@ -0,0 +1,430 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import { + mkdtempSync, + mkdirSync, + readFile, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { readFile as readAsync } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { FixtureCapture } from '../../../tools/scenario-map/path-coverage/fixture-capture.js'; +import { fixtureCapturePlugin } from '../../../tools/scenario-map/path-coverage/fixture-plugin.js'; +import { + CoverageAttributor, + CoverageInterner, + type RawCoverageRecord, + type WindowDelta, +} from '../../../tools/scenario-map/path-coverage/records.js'; +import { + profileRuntimeFixtures, + runtimeFixtureFor, +} from '../../../tools/scenario-map/facet/runtime-fixtures.js'; +import { compareFixtureProfiles } from '../../../tools/scenario-map/facet/fixture-profile.js'; +import { buildWorkFile } from '../../../tools/scenario-map/path-coverage/run-coverage.js'; +import { + mergeCoverageSlices, + type CoverageSliceFile, +} from '../../../tools/scenario-map/path-coverage/slices.js'; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +const repo = (files: Record) => { + const root = mkdtempSync(path.join(tmpdir(), 'runtime-fixture-')); + roots.push(root); + for (const [name, content] of Object.entries(files)) { + const file = path.join(root, name); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, content); + } + return root; +}; +const delta = (fixtures: string[] = []): WindowDelta => ({ + blocks: new Set(), + scripts: new Set(), + natives: new Set(), + spawns: [], + fixtures, +}); +const record = (capture: FixtureCapture): RawCoverageRecord => { + const owner = new CoverageAttributor(); + owner.testStart('test', ['file'], false, delta()); + owner.testEnd('test', delta(capture.drain())); + return { + ...owner.record('/prep/gitnexus/test/input.test.ts', new CoverageInterner([], new Set())), + fixtureSnapshots: capture.snapshots, + }; +}; +const profile = async (raw: RawCoverageRecord, index = 0, concurrent = false) => { + const result = runtimeFixtureFor( + raw, + raw.tests[index], + (await profileRuntimeFixtures([raw])).get(raw), + concurrent, + ); + if (result === null) throw new Error('expected captured fixture'); + return result; +}; +const TS = 'export function twice(x: number) { return x + x; }'; +const PY = 'def twice(x):\n return x + x\n'; + +it('preserves reads and repository snapshots when the test replaces JSON.stringify', () => { + const root = repo({ 'main.ts': TS }); + const capture = new FixtureCapture(); + const restore = capture.install(); + const stringify = vi.spyOn(JSON, 'stringify').mockImplementation(() => { + throw new Error('test-controlled serialization failure'); + }); + let read: string; + try { + read = readFileSync(path.join(root, 'main.ts'), 'utf8'); + capture.repository(root); + } finally { + stringify.mockRestore(); + restore(); + } + expect(read).toBe(TS); + expect(Object.values(capture.snapshots).map((snapshot) => snapshot.files)).toEqual([ + [['main.ts', TS]], + [['main.ts', TS]], + ]); +}); + +it('does not capture a replacement installed after the file metadata check', () => { + const root = repo({ 'main.ts': TS }); + const file = path.join(root, 'main.ts'); + const original = fs.lstatSync; + let replaced = false; + const stat = vi.spyOn(fs, 'lstatSync').mockImplementation(((...args) => { + const result = Reflect.apply(original, fs, args); + if (args[0] === file && !replaced) { + replaced = true; + fs.renameSync(file, `${file}.old`); + writeFileSync(file, 'replacement contents must not be retained'); + } + return result; + }) as typeof fs.lstatSync); + const capture = new FixtureCapture(); + try { + capture.repository(root); + } finally { + stat.mockRestore(); + } + expect(replaced).toBe(true); + expect(JSON.stringify(capture.snapshots)).not.toContain('replacement contents'); + expect(Object.values(capture.snapshots)[0].unresolved.join()).toMatch(/changed|unreadable/); +}); + +it('bounds reads when a file grows after opening and closes the descriptor', () => { + const root = repo({ 'main.ts': TS }); + const original = fs.fstatSync; + let grew = false; + const stat = vi.spyOn(fs, 'fstatSync').mockImplementation(((...args) => { + const result = Reflect.apply(original, fs, args); + if (!grew) { + grew = true; + fs.appendFileSync(path.join(root, 'main.ts'), 'x'.repeat(250_000)); + } + return result; + }) as typeof fs.fstatSync); + const read = vi.spyOn(fs, 'readSync'); + const close = vi.spyOn(fs, 'closeSync'); + const capture = new FixtureCapture(); + try { + capture.repository(root); + expect(grew).toBe(true); + expect(read.mock.results.reduce((n, result) => n + result.value, 0)).toBe( + Buffer.byteLength(TS) + 1, + ); + expect(close).toHaveBeenCalledOnce(); + expect(Object.values(capture.snapshots)[0]).toMatchObject({ + files: [], + unresolved: ['main.ts: changed during capture'], + }); + } finally { + stat.mockRestore(); + read.mockRestore(); + close.mockRestore(); + } +}); + +it('closes the descriptor when reading fails and marks the snapshot incomplete', () => { + const root = repo({ 'main.ts': TS }); + const read = vi.spyOn(fs, 'readSync').mockImplementation(() => { + throw new Error('read failed'); + }); + const close = vi.spyOn(fs, 'closeSync'); + const capture = new FixtureCapture(); + try { + capture.repository(root); + expect(close).toHaveBeenCalledOnce(); + expect(Object.values(capture.snapshots)[0]).toMatchObject({ + files: [], + unresolved: ['main.ts: unreadable'], + }); + } finally { + read.mockRestore(); + close.mockRestore(); + } +}); + +it('retains generated, deleted multilingual input contents and distinguishes control flow', async () => { + const profiles = []; + for (const files of [ + { 'main.ts': TS }, + { 'main.py': PY }, + { 'main.ts': 'export function twice(x: number) { while(x > 0) { x--; } return x; }' }, + ]) { + const root = repo(files); + const capture = new FixtureCapture(); + capture.repository(root); + const raw = record(capture); + rmSync(root, { recursive: true }); + profiles.push(await profile(JSON.parse(JSON.stringify(raw)))); + } + expect(compareFixtureProfiles(profiles[0], profiles[1])).toMatchObject({ + kind: 'structure', + crossLanguage: true, + }); + expect(compareFixtureProfiles(profiles[0], profiles[2])).toBeNull(); + expect(profiles.map((p) => p.unresolved)).toEqual([[], [], []]); + expect(profiles[0].contentKey).not.toBe(profiles[1].contentKey); +}); + +it('keeps input versions at the same path and ignores random temporary root names', async () => { + const first = repo({ 'main.ts': TS }); + const second = repo({ 'main.ts': TS }); + const capture = new FixtureCapture(); + capture.repository(first); + const a = record(capture); + capture.repository(second); + const b = record(capture); + writeFileSync(path.join(first, 'main.ts'), TS.replace('x + x', 'x + 1')); + capture.repository(first); + const c = record(capture); + expect((await profile(a)).contentKey).toBe((await profile(b)).contentKey); + expect((await profile(a)).contentKey).not.toBe((await profile(c)).contentKey); +}); + +it('compares repeat inputs by content rather than temporary root names', async () => { + const capture = new FixtureCapture(); + capture.repository(repo({ 'main.ts': TS })); + const cold = capture.drain(); + capture.repository(repo({ 'main.ts': TS })); + const raw = record(capture); + raw.tests[0].all.fixtures = [...cold, ...(raw.tests[0].warm.fixtures ?? [])]; + expect((await profile(raw)).unresolved).toEqual([]); + capture.repository(repo({ 'main.ts': TS.replace('x + x', 'x + 1') })); + raw.tests[0].all.fixtures.push(...capture.drain()); + expect((await profile(raw)).unresolved).toContain( + 'runtime fixture inputs changed between attempts', + ); +}); + +it('filters module loading and production housekeeping from direct-read capture', () => { + const capture = new FixtureCapture(); + expect( + capture.accepts( + 'Error\n at read (/repo/tools/scenario-map/path-coverage/fixture-capture.ts:1:1)\n at helper (/repo/test/helpers/fixture.ts:8:1)', + ), + ).toBe(true); + expect( + capture.accepts( + 'Error\n at load (/repo/src/core/parser-loader.ts:8:1)\n at test (/repo/test/example.ts:9:1)', + ), + ).toBe(false); + expect(capture.accepts('Error\n at load (/repo/node_modules/vite/test.js:8:1)')).toBe(false); +}); + +it('marks bounded captures incomplete instead of hashing a truncated repository as complete', async () => { + const root = repo( + Object.fromEntries(Array.from({ length: 201 }, (_, i) => [`file${i}.txt`, 'sample'])), + ); + const capture = new FixtureCapture(); + capture.repository(root); + const p = await profile(record(capture)); + expect(p.contentKey).toBeNull(); + expect(p.unresolved).toContain('runtime repository capture limit'); + expect(p.files.length).toBeLessThanOrEqual(200); +}); + +it('bounds large returned buffers before allocating a decoded copy', () => { + const capture = new FixtureCapture(); + const buffer = Buffer.alloc(200_000); + buffer.toString = () => { + throw new Error('must not decode oversized input'); + }; + expect(() => capture.read('/tmp/large.ts', buffer)).not.toThrow(); + expect(Object.values(capture.snapshots)).toMatchObject([ + { files: [], unresolved: ['oversized text'] }, + ]); +}); + +it('captures real sync, promise, callback and URL reads without changing results or errors', async () => { + const root = repo({ 'main.ts': TS }); + const file = path.join(root, 'main.ts'); + const capture = new FixtureCapture(); + const restore = capture.install(); + try { + expect(readFileSync(file, 'utf8')).toBe(TS); + expect((await readAsync(pathToFileURL(file))).toString()).toBe(TS); + expect( + await new Promise((resolve, reject) => + readFile(file, 'utf8', (error, text) => (error ? reject(error) : resolve(text))), + ), + ).toBe(TS); + expect(() => readFileSync(path.join(root, 'missing.ts'))).toThrow(); + await expect(readAsync(path.join(root, 'missing.ts'))).rejects.toThrow(); + } finally { + restore(); + } + const p = await profile(record(capture)); + expect(p.languages).toEqual(['typescript']); + expect(p.files).toEqual([file]); + expect(p.contentKey).toBeNull(); + expect(p.unresolved).toContain('test read may be assertion output; input role is unverified'); +}); + +it('does not turn an identical generated output into proof of identical fixture inputs', async () => { + const capture = new FixtureCapture(); + capture.read( + '/tmp/setup-home/.claude.json', + '{"mcpServers":{"gitnexus":{"command":"gitnexus.cmd"}}}', + ); + const p = await profile(record(capture)); + expect(p.contentKey).toBeNull(); + expect(p.unresolved).toContain('test read may be assertion output; input role is unverified'); + // Keep the read available for diagnosis without promoting it to an ingestion input. + expect(p.files).toEqual(['/tmp/setup-home/.claude.json']); +}); + +it('reports excluded, missing, oversized and symbolic inputs without leaking contents', async () => { + const root = repo({ + '.env': 'never-retain-this', + 'large.ts': 'x'.repeat(200_000), + 'main.ts': TS, + }); + if (process.platform !== 'win32') + symlinkSync(path.join(root, 'main.ts'), path.join(root, 'alias.ts')); + const capture = new FixtureCapture(); + capture.repository(root); + capture.repository(path.join(root, 'missing')); + const raw = record(capture); + expect(JSON.stringify(raw)).not.toContain('never-retain-this'); + const p = await profile(raw); + expect(p.contentKey).toBeNull(); + expect(p.unresolved.join()).toMatch(/environment/); + expect(p.unresolved.join()).toMatch(/size limit/); + if (process.platform !== 'win32') expect(p.unresolved.join()).toMatch(/symbolic/); + expect(p.unresolved.join()).toMatch(/unreadable/); +}); + +it('attributes nested setup to its suite and retains warm-attempt and concurrent gaps', async () => { + const capture = new FixtureCapture(); + const attributor = new CoverageAttributor(); + const inner = repo({ 'main.ts': TS }); + capture.inSuite('inner', () => capture.repository(inner)); + attributor.testStart('a', ['file', 'outer', 'inner'], false, delta(capture.drain())); + attributor.testEnd('a', delta()); + attributor.testStart('b', ['file', 'outer'], false, delta()); + attributor.testEnd('b', delta()); + const raw = { + ...attributor.record('/prep/gitnexus/test/input.test.ts', new CoverageInterner([], new Set())), + fixtureSnapshots: capture.snapshots, + }; + expect((await profile(raw)).unresolved).toEqual([]); + expect(runtimeFixtureFor(raw, raw.tests[1], undefined, false)).toBeNull(); + expect((await profile(raw, 0, true)).contentKey).toBeNull(); + const profiles = await profileRuntimeFixtures([raw]); + const work = buildWorkFile({ + mode: { kind: 'normal' }, + preparedRoot: '/prep', + repeats: 1, + unhandledErrors: [], + raws: [raw], + fixtureProfiles: profiles, + modules: [ + { + file: 'test/input.test.ts', + errors: [], + tests: raw.tests.map((t) => ({ + taskId: t.taskId, + fullName: t.taskId, + location: undefined, + concurrent: false, + state: 'passed', + })), + }, + ], + }); + expect(work.tests[0].runtimeFixture?.languages).toEqual(['typescript']); + expect(work.tests[1].runtimeFixture).toBeNull(); + raw.tests[0].all.fixtures = ['missing']; + expect((await profile(raw)).unresolved.join()).toContain('changed between attempts'); +}); + +it('leaves old records compatible and refuses missing snapshot evidence', async () => { + const capture = new FixtureCapture(); + capture.repository(repo({ 'main.ts': TS })); + const raw = record(capture); + expect(runtimeFixtureFor(raw, raw.tests[0], undefined, false)?.contentKey).toBeNull(); + delete raw.fixtureSnapshots; + expect(runtimeFixtureFor(raw, raw.tests[0], undefined, false)).toBeNull(); +}); + +it('preserves capture versions and rejects mixed instrumented and archived coverage slices', () => { + const slice = (index: number, captured: boolean): CoverageSliceFile => ({ + schemaVersion: 1, + index, + of: 2, + layer1: [], + mutation: 'none', + changed: null, + revision: null, + work: { + schemaVersion: 1, + ...(captured ? { fixtureCaptureVersion: 1 } : {}), + mode: 'normal', + extraRoots: [], + repeats: 1, + blocks: [], + files: [], + tests: [], + unhandledErrors: [], + }, + }); + expect( + mergeCoverageSlices([[], []], [slice(0, true), slice(1, true)]).fixtureCaptureVersion, + ).toBe(1); + expect( + mergeCoverageSlices([[], []], [slice(0, false), slice(1, false)]).fixtureCaptureVersion, + ).toBeUndefined(); + expect(() => mergeCoverageSlices([[], []], [slice(0, true), slice(1, false)])).toThrow( + 'not the run', + ); +}); + +it('instruments the pipeline entry with a source map and rejects entry point drift', async () => { + const plugin = fixtureCapturePlugin('/prep/src/core/ingestion/pipeline.ts'); + const transform = plugin.transform; + if (typeof transform !== 'function') throw new Error('missing transform'); + const result = await Reflect.apply(transform, {}, [ + 'export const runPipelineFromRepo = async (root: string) => { return root; };', + '/prep/src/core/ingestion/pipeline.ts', + ]); + expect(result.code).toContain('globalThis["__gitnexusScenarioFixtureCapture"]?.(root)'); + expect(JSON.parse(result.map).sources).toEqual(['/prep/src/core/ingestion/pipeline.ts']); + expect(() => + Reflect.apply(transform, {}, [ + 'export const changed = 1;', + '/prep/src/core/ingestion/pipeline.ts', + ]), + ).toThrow('entry point'); +}); diff --git a/gitnexus/test/unit/scenario-map/trace-setup.test.ts b/gitnexus/test/unit/scenario-map/trace-setup.test.ts new file mode 100644 index 000000000..5f43d3edc --- /dev/null +++ b/gitnexus/test/unit/scenario-map/trace-setup.test.ts @@ -0,0 +1,41 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { FIXTURE_CAPTURE_KEY } from '../../../tools/scenario-map/path-coverage/fixture-capture.js'; + +const previousCapture = Reflect.get(globalThis, FIXTURE_CAPTURE_KEY); + +afterEach(() => { + vi.doUnmock('vitest'); + vi.resetModules(); + vi.restoreAllMocks(); + if (previousCapture === undefined) Reflect.deleteProperty(globalThis, FIXTURE_CAPTURE_KEY); + else Reflect.set(globalThis, FIXTURE_CAPTURE_KEY, previousCapture); +}); + +it('captures pipeline fixtures only while a candidate test is active', async () => { + let current: { id: string } | undefined; + // Reload the setup module even when the detector already installed it for this worker. + vi.resetModules(); + const { TraceFixtureRoots } = + await import('../../../tools/scenario-map/value-flow/trace-fixtures.js'); + const capture = vi.spyOn(TraceFixtureRoots.prototype, 'capture').mockImplementation(() => {}); + vi.doMock('vitest', () => ({ + TestRunner: { getCurrentTest: () => current }, + inject: () => ({ taskIds: ['candidate'], preparedRoot: '/prepared', extraRoots: [], wrap: [] }), + aroundAll: vi.fn(), + aroundEach: vi.fn(), + })); + await import('../../../tools/scenario-map/value-flow/trace.setup.js'); + const hook = Reflect.get(globalThis, FIXTURE_CAPTURE_KEY) as (root: string) => void; + + hook('/suite-setup'); + expect(capture).not.toHaveBeenCalled(); + current = { id: 'sibling' }; + hook('/sibling'); + expect(capture).not.toHaveBeenCalled(); + current = { id: 'candidate' }; + hook('/candidate'); + expect(capture).toHaveBeenCalledExactlyOnceWith('/candidate'); + current = undefined; + hook('/suite-teardown'); + expect(capture).toHaveBeenCalledTimes(1); +}); diff --git a/gitnexus/test/unit/scenario-map/trace.test.ts b/gitnexus/test/unit/scenario-map/trace.test.ts index d76e245fa..2a30a3c9f 100644 --- a/gitnexus/test/unit/scenario-map/trace.test.ts +++ b/gitnexus/test/unit/scenario-map/trace.test.ts @@ -16,6 +16,7 @@ import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { afterEach, describe, expect, it } from 'vitest'; +import { TraceFixtureRoots } from '../../../tools/scenario-map/value-flow/trace-fixtures.js'; import type { PrepareOptions, PrepareReport } from '../../../tools/mutation/prepare/prepare.js'; import type { OutcomeClass } from '../../../tools/mutation/runner/classify-outcome.js'; import { scriptPath } from '../../../tools/scenario-map/path-coverage/records.js'; @@ -208,6 +209,52 @@ describe('serializeValue', () => { expect(serializeValue([Math], new Map([[Math, 'Math']])).text).toBe('[[Math]]'); }); + it('compares captured repository Dirents independently of their temporary root', async () => { + const fs = await import('node:fs'); + const roots = [tempDir(), tempDir()]; + for (const root of roots) fs.writeFileSync(path.join(root, 'main.ts'), 'export const n = 1;'); + const entries = roots.map((root) => fs.readdirSync(root, { withFileTypes: true })); + const names = new Map(roots.map((root) => [root.replace(/\\/g, '/'), 'fixture:0:same-bytes'])); + expect(serializeValue(entries[0], new Map(), names)).toEqual( + serializeValue(entries[1], new Map(), names), + ); + // Arbitrary strings and lookalike objects can be meaningful application inputs. + expect(serializeValue(roots[0], new Map(), names)).not.toEqual( + serializeValue(roots[1], new Map(), names), + ); + expect(serializeValue({ parentPath: roots[0] }, new Map(), names)).not.toEqual( + serializeValue({ parentPath: roots[1] }, new Map(), names), + ); + expect(serializeValue(entries[0])).not.toEqual(serializeValue(entries[1])); + names.set(roots[1].replace(/\\/g, '/'), 'fixture:0:different-bytes'); + expect(serializeValue(entries[0], new Map(), names)).not.toEqual( + serializeValue(entries[1], new Map(), names), + ); + }); + + it('keeps fixture bytes, layout, aliases, and incomplete captures visible in trace identity', () => { + const a = tempDir(), + b = tempDir(); + for (const root of [a, b]) writeFileSync(path.join(root, 'main.ts'), 'export const n = 1;'); + const left = new TraceFixtureRoots(), + right = new TraceFixtureRoots(); + left.capture(a); + right.capture(b); + const key = (roots: TraceFixtureRoots, root: string) => + roots.roots.get(root.replace(/\\/g, '/')); + expect(key(left, a)).toBe(key(right, b)); + left.capture(b); + expect(key(left, a)).not.toBe(key(left, b)); + writeFileSync(path.join(b, 'main.ts'), 'export const n = 2;'); + right.capture(b); + expect(key(left, a)).not.toBe(key(right, b)); + writeFileSync(path.join(b, 'huge.ts'), 'x'.repeat(200_000)); + right.capture(b); + expect(key(right, b)).toBeUndefined(); + left.clear(); + expect(left.roots.size).toBe(0); + }); + it('serialises an Intl instance by its resolved options', () => { expect(serializeValue(new Intl.NumberFormat('en-US')).text).toMatch( /^Intl\.NumberFormat\(\{"locale":"en-US",/, @@ -978,6 +1025,63 @@ async function runWith( } describe('runTrace', () => { + it('retains failed-test diagnostics while keeping an invalid run inconclusive', async () => { + const baseline: VitestRunResult = { + ...vitestResult(['killed-by-test:assertion', 'survived']), + status: 'killed', + outcome: 'killed-by-test:assertion', + }; + baseline.tests[0].message = 'expected an existing index'; + const { result } = await runWith(fakeRun([], [], { baseline })); + expect(result.baseline?.failures).toEqual([ + { id: A, outcome: 'killed-by-test:assertion', message: 'expected an existing index' }, + ]); + expect(result.issues.length).toBeGreaterThan(0); + expect(result.pairs[0].status).toBe('inconclusive'); + }); + + it('runs sibling tests in both passes while comparing only the candidate tests', async () => { + const manifestRoot = tempDir(); + writeTree(manifestRoot, ''); + const sibling = `${PACKAGE_FILE}#suite setup in an earlier test`; + const baseline = vitestResult(['survived', 'survived']); + baseline.tests.push({ + ...baseline.tests[0], + id: sibling, + taskId: 'sibling-task', + fullName: 'suite setup in an earlier test', + }); + const fake = fakeRun([ROUND('[2]')], [ROUND('[2]')], { baseline }); + const skipped = `${PACKAGE_FILE}#platform-specific test`; + const unrelated = 'test/unit/unrelated.test.ts#unrelated'; + const manifest = buildRunManifest({ + preparedRoot: manifestRoot, + mode: { kind: 'normal' }, + tests: [A, B, sibling, skipped, unrelated], + }); + const result = await runTrace( + { + repoRoot: tempDir(), + workDir: tempDir(), + manifest, + pairs: [{ a: A, b: B }], + // A candidate erroneously marked skipped must still fail closed if it does not run. + skippedTestIds: [skipped, A], + opaqueCallees: [], + }, + fake.deps, + ); + expect(fake.calls.map((call) => [...call.testIds].sort())).toEqual([ + [A, B, sibling].sort(), + [A, B, sibling].sort(), + ]); + expect(result.tests.map((test) => test.id)).toEqual([A, B]); + expect(result.pairs.map((pair) => pair.status)).toEqual(['clean']); + expect(fake.provided.find(([key]) => key === TRACE_KEY)?.[1]).toMatchObject({ + taskIds: ['t_0', 't_1'], + }); + }); + it('runs a baseline and a traced run of the same tests and vetoes differing calls', async () => { const fake = fakeRun([ROUND('[2.4]')], [ROUND('[2.5]')]); const { result, workDir } = await runWith(fake, ['Array.join', 'Map.set']); diff --git a/gitnexus/test/unit/scenario-map/vitest-run.test.ts b/gitnexus/test/unit/scenario-map/vitest-run.test.ts index 277d41fe8..a879b532d 100644 --- a/gitnexus/test/unit/scenario-map/vitest-run.test.ts +++ b/gitnexus/test/unit/scenario-map/vitest-run.test.ts @@ -39,6 +39,38 @@ function fakeCreateVitest(seen: CliOptions[]): CreateVitest { } describe('runVitest worker bound', () => { + it.each([true, false])( + 'forwards plugins only when supplied (instrumented: %s)', + async (instrumented) => { + const plugin = { name: 'fixture-capture-regression' }; + const plugins = instrumented ? [plugin] : undefined; + const create: CreateVitest = async (_options, overrides) => { + expect(overrides.plugins).toEqual(plugins); + return { + projects: [], + close: async () => {}, + collectTests: async () => { + throw new Error('unused'); + }, + runTestSpecifications: async () => { + throw new Error('unused'); + }, + }; + }; + await expect( + runVitest( + { + packageDir: tempDir(), + testIds: ['test/unit/a.test.ts#a'], + vitest: { configFile: 'vitest.config.ts', project: 'p' }, + workDir: tempDir(), + ...(instrumented ? { plugins } : {}), + }, + create, + ), + ).rejects.toThrow(/expected one Vitest project named p/); + }, + ); it.each([ ['the given bound', 3, 3], ['no bound (Vitest default)', undefined, undefined], diff --git a/gitnexus/tools/scenario-map/README.md b/gitnexus/tools/scenario-map/README.md new file mode 100644 index 000000000..edf1a3518 --- /dev/null +++ b/gitnexus/tools/scenario-map/README.md @@ -0,0 +1,226 @@ +# Scenario-map fixture comparison + +The `plan` stage discovers source fixtures read by tests and adds fixture similarity +to candidate retrieval. Run it through the existing CLI: + +```sh +cd gitnexus +npm run test:scenario-map -- plan --work /tmp/gitnexus-scenario-map --mutation none +``` + +This runs coverage for the test inventory. To reuse coverage slices from the same +checkout, add `--coverage /path/to/coverage-slices`. +All slices must use the same capture version; mixing old coverage with newly instrumented +coverage is rejected because instrumentation changes V8 block offsets. + +## Evidence + +- `fixtures.json`: fixture paths, languages, extraction gaps, counted features, + associated tests, and similarity matches. Profiles are deduplicated across tests. +- `coverage/candidates.json`: pairs retrieved through the `fixture` source, with the score, + content/structure match kind, and whether their languages differ. +- `layer2.json`: per-test fixture evidence and input identity used by later stages. +- Fresh coverage records also contain `runtimeFixture` profiles. Raw records retain + bounded input snapshots, so profiling does not depend on temporary files surviving. + +Discovery follows constants, imported test helpers, setup hooks, static string rows, +path joins, and file URLs for ingestion, file-read, and copy calls. It reads source +corpora anywhere under `gitnexus/test/`, including integration-local fixtures. +It never executes a helper to discover its inputs. Diff-scoped runs retain these +tests until their fixture profiles can be compared. + +Fresh coverage also captures generated and dynamically selected inputs. A coverage-only +Vite transform snapshots the repository passed to `runPipelineFromRepo` before ingestion, +including runs that dispatch parsing to workers. It preserves relative file layout and +non-source assets. Pass-through wrappers retain the returned contents of synchronous, +promise, and callback `readFile` calls made directly by tests or test helpers, including +file URLs. Production files on disk are not modified by instrumentation. + +Reads made by a test can also be assertions over generated output. Their contents +remain available for diagnosis, but their input role is marked unverified and they +cannot establish fixture identity or a consolidation lead. Repository-entry snapshots +establish which files were supplied to ingestion; this does not prove that every other +input was observed. Re-capture archived version-1 coverage before evaluating runtime +fixture consolidation: version 2 records this distinction. + +Snapshots follow coverage windows: the last test attempt and its suite setup feed that +test's profile. Setup hooks carry their suite identity across async calls, so nested +setup does not leak into sibling suites. Changed input contents across attempts, +concurrent tests, and reads outside attributable windows remain explicit gaps. Temporary +directory names do not change input identity. The planner parses each distinct captured +input through the same language providers used for static fixtures, after the test ends. + +The profiler reuses GitNexus's language providers for declarations, reference kinds, +call forms, imports, and control-flow graphs. Counts preserve multiplicity. Three +rounds of directed, labeled CFG-neighbor refinement give a topology signature +independent of block IDs and source positions. This is a structural heuristic, +not a graph-isomorphism or program-equivalence proof. + +Two tests become fixture candidates only when their assertion-only keys match and +their fixture profiles have weighted multiset Jaccard similarity of at least 0.90. +CFG signatures have weight 4; other features have weight 1. Exact content matches +also retrieve candidates. File profiles and pair scores are cached within a run. + +Similarity omits identifier spelling and literal values. **Exact fixture text and +relative file layout are separately included in the input identity**, including +non-source assets. A similar fixture cannot make different inputs a confirmed +duplicate. Cross-language cases often provide intentional provider coverage; +a match is a review lead, not a recommendation to remove either test. + +## Consolidation opportunities, including partial matches + +The `plan` stage also writes `consolidation.html` and `consolidation.json`. These +compare three independent axes: captured fixture contents, executed JavaScript +blocks, and individual assertions. A fixture match alone does not create a lead. +The HTML report can be filtered by test name or recommendation. It shows both +directions of overlap and the assertions, blocks, and fixture spans that remain +unique. Groups represent repeated evidence without enumerating every test pair. + +The default HTML view shows actionable opportunities. Cases from one test declaration +(including existing `it.each` and `describe.each` rows) and tests annotated with +`// dupe-ok: ` are retained under separate review filters. The JSON keeps every +group and reports `actionableOpportunities` separately. Detector controls use this +annotation so their intentional overlap remains available for acceptance checks. +The fingerprint report likewise retains existing table pairs in its `kept` list. +The CI plan artifact includes `consolidation.html`, `consolidation.json`, and `fixtures.json`. + +Fixture text is compared locally with the pinned **jscpd 5.4.2** development +dependency. Exact, consistently renamed, and near-miss fragments contribute to +byte overlap; comments and whitespace are ignored by the clone engine. The +coverage denominator conservatively includes all non-whitespace captured bytes. +Overlapping fragments count once; near-miss scores discount their contribution. +Representative exact/renamed clone links form families, so a missing direct link +does not hide another member. When representative spans overlap with different +boundaries, pairs that pass the assertion and execution gates are compared directly +in a separate two-input scan. This avoids inventing byte mappings or losing a partial +match when the corpus chooses a longer representative. The raw pair reports and +verification count are retained. Near-miss links are never treated as transitive. +No source is uploaded and no embedding model is downloaded. + +Source snapshots are content-addressed in `coverage.json` and merged across slices. +The external engine's raw report and exported inputs are under `fixture-clones/`. +The report records captured/scanned file counts and formats: unsupported formats +and fragments below the 12-token threshold can be missed. Source identities still +recognize identical complete inputs independently of tokenization. + +Across languages, GitNexus provider features supply a separate, weaker +`share-language-scenario` suggestion. Full matches require 0.90 weighted Jaccard; +partial matches require shared CFG topology and 0.90 feature containment in one +direction. These are structural scenario leads, not cross-language semantic proofs. +Each language remains a test case when consolidating into a shared harness. + +Assertions retain exact expected literals, row values, snapshots, negation and +selectors. Single preceding `const` bindings and scoped callback parameters are +traced, so `find(edge => edge.target === 'save')` does not equal a `getName` query. +Unknown expected values or local origins cannot become equal assertion atoms. +File- and suite-level values retain their declaration's file and scope identity. +Repeated reads of the same binding can match; equivalence between separate bindings +remains unproven because scope frames do not establish their values or immutability. +File-local callable identities also remain distinct across files. Fingerprint verdicts +use this detailed evidence to reject unrelated assertion subjects or literal expectations +even when the coarse matcher shape agrees. +Unknown checks remain in overlap denominators. This comparison does not execute +arbitrary expected-value expressions or establish general helper semantic equivalence. Copied helpers with identical bodies and resolved imports share identity; helpers that close over other bindings remain scope-specific, and comparisons between those bindings stay unresolved. + +Default retrieval requires 0.50 fixture containment and 0.50 executed-block +containment in either direction, plus 0.25 exact-assertion containment. Fully known +assertions with different expected values can instead qualify through 0.80 +predicate-shape containment, producing parameterization leads. Thresholds and +missing evidence are included in the JSON. This supports sharing a subset of +checks while retaining the remaining assertions and fixture branches. + +Executed blocks exclude uncovered V8 ranges. They describe observed coverage, +not call order or every possible execution. Native work, spawned workers, input +capture gaps, unsupported checks and inventory joins stay visible as limitations. +Even a complete match is a statement about the captured run; consolidation output +does not authorize deleting tests or bypass existing duplicate-confirmation gates. + +Run a complete capture and generate a report (or use the existing coverage-slice +workflow to distribute the capture): + +```sh +node --import tsx tools/scenario-map/scenario-map.ts coverage \ + --index 0 --of 1 --work /tmp/scenario-coverage --mutation none --concurrency 2 +NODE_OPTIONS=--max-old-space-size=8192 node --import tsx tools/scenario-map/scenario-map.ts plan \ + --coverage /tmp/scenario-coverage --work /tmp/scenario-plan --mutation none +node --import tsx tools/scenario-map/consolidation/verify-controls.ts \ + /tmp/scenario-plan/consolidation.json +``` + +Full-suite planning needs more than Node's default heap. The CI Plan step uses +the same 8 GiB budget; allow additional RAM for native allocations and subprocesses. + +The merge stage streams `scenario-map-report.json` one record at a time. Full-suite +reports can exceed Node's single-string limit; use a streaming JSON reader instead +of `JSON.parse(readFileSync(..., 'utf8'))`. The report schema and all candidate +evidence are preserved. Baseline comparison and the CI issue update use +`report/json.ts`'s `readReportFields` to retain only the fields they need while +validating the whole file. A failed runtime trace still produces a partial report +and cannot confirm duplicates. + +Capture preserves the suite's normal lifecycle and does not add per-test repeats: +repeating a destructive test or a cache assertion within one suite changes its input. +Tracing includes runnable siblings in each candidate file in both passes, preserving +setup performed by earlier tests; only candidate tests are recorded and compared. Candidate +task IDs come from the baseline collection. Siblings run without trace stack capture, +argument serialization or per-test fixture snapshots, so unrelated timing checks do not +pay that instrumentation cost. A missing candidate trace remains inconclusive. Explicitly skipped +siblings stay out. If a runnable sibling uses retries, the file's pairs remain inconclusive. +Failed captures retain their error details in `coverage.json` and keep the report +partial even when every file was collected. + +The acceptance command checks all 28 pairs among eight real pipeline controls: +15 exact/renamed/partial/cross-language opportunities and 13 negative pairs. It +also checks that unique evidence and runtime limitations survive in the report. + +## Capture and proof limits + +Archived coverage without runtime snapshots still uses static discovery. Unresolved static +paths (including generated inputs) require a fresh coverage run to obtain runtime evidence. +Missing parsers, parse errors, lossy text +decoding, and extraction limits are recorded as unresolved. Partial profiles do +not retrieve structural matches. Identical, completely read text can still retrieve +an exact-content candidate when parsing fails; that failure remains unresolved +for duplicate confirmation. Incomplete reads never establish input identity. Existing +path, context, assertion, runtime-value, and passing-test checks still apply. + +Runtime capture retrieves review candidates; it does not establish exhaustive input +coverage. Streams, file-descriptor reads, inputs generated inside child processes, and +repository mutations after the entry snapshot are not fully observed. Runtime evidence +cannot clear static input gaps or confirm duplicates by itself. A complete static profile +corroborated by identical runtime contents retains its existing confirmation checks. +Skipped or failed tests cannot establish duplicate safety. + +Value tracing identifies native `Dirent` directory metadata relative to repository +roots captured in that test attempt. The identity includes exact bytes, relative layout, +and separate ordinals for multiple repositories. Incomplete captures, arbitrary path +strings, relative file names, and lookalike objects are not normalized. This removes +temporary-root noise from directory-entry comparisons without relaxing native/worker +or unresolved-input gates. + +Structural fingerprints inherit provider behavior, including parser or CFG instability; +exact text and external clone spans are separate evidence. The comparison uses pre-resolution features. It does not prove cross-file binding, +data-flow, operator, or runtime equivalence. Providers without CFG support contribute +their available capture features. Identical counts can produce false-positive +review leads, and different provider representations can miss similar fixtures. + +Limits per profile: 200 files, fewer than 200,000 characters per file, 2,000,000 +characters total, and 2,000 lines per CFG function. Runtime capture additionally bounds +each test file to 20 MB of unique snapshot text and 2,000 events; repository snapshots +visit at most 1,000 entries and 30 directory levels. Symlinks are reported as incomplete; +`.git`, `.gitnexus`, and `node_modules` directories are omitted. Helper traversal is bounded to +five calls and constant evaluation to eight steps. `.env` files other than +`.env.example` are excluded and reported as incomplete. + +Validate with: + +```sh +npm test -- test/unit/scenario-map --maxWorkers=2 +npm run typecheck:tests +# Fresh real-pipeline controls, including generated repositories deleted in setup: +node --import tsx tools/scenario-map/path-coverage/run-coverage.ts \ + --file test/integration/scenario-map-runtime-inputs.test.ts \ + --out /tmp/scenario-map-runtime-controls --concurrency 1 +``` + +Existing table declarations retain uncertain or differing rows outside the default queue. Proven duplicate rows remain actionable in the fingerprint report; exact observed fixture/path/assertion matches remain actionable in the consolidation report. Keep reasons are separate from evidence gaps and do not change an opportunity’s evidence classification. diff --git a/gitnexus/tools/scenario-map/consolidation/assertions.ts b/gitnexus/tools/scenario-map/consolidation/assertions.ts new file mode 100644 index 000000000..4d10ff6ca --- /dev/null +++ b/gitnexus/tools/scenario-map/consolidation/assertions.ts @@ -0,0 +1,419 @@ +import * as t from '@babel/types'; +import traverse, { type Binding } from '@babel/traverse'; +import { generate } from '@babel/generator'; +import { resolveName } from '../../test-dupes/test-dupes.js'; +import type { RowValue } from '../types.js'; +import { + abstractNode, + assertionCallOf, + buildScope, + rowValueOf, + type CheckSubject, + type InstanceChecks, + type Scope, +} from '../facet/checks.js'; +import { sha1, sortedUnique } from '../util.js'; + +export interface AssertionAtom { + line: number; + shape: string; + shapeKey: string; + /** Subject structure with literal imported-call inputs abstracted; selectors remain exact. */ + subjectShapeKey?: string; + /** Exact expected literals/snapshot and subject shape, or null for unresolved values. */ + key: string | null; + expected: string | null; + observed?: string | null; +} +export interface AssertionEvidence { + atoms: AssertionAtom[]; + unresolved: string[]; +} + +function canonicalRow(row: RowValue): unknown | undefined { + if (row.kind === 'unresolved') return undefined; + if (row.kind === 'array') { + const items = row.items.map(canonicalRow); + return items.includes(undefined) ? undefined : { kind: 'array', items }; + } + if (row.kind === 'object') { + const values = new Map(); + for (const entry of row.entries) { + const value = canonicalRow(entry.value); + if (value === undefined) return undefined; + values.set(entry.key, value); + } + const entries = [...values].map(([key, value]) => ({ key, value })); + return { kind: 'object', entries: entries.sort((a, b) => a.key.localeCompare(b.key)) }; + } + return row; +} + +function literal(node: t.Node, scope: Scope): unknown | undefined { + if (t.isTSAsExpression(node) || t.isTSNonNullExpression(node) || t.isTSTypeAssertion(node)) + return literal(node.expression, scope); + const row = rowValueOf(node, scope); + if (row !== null) return row === 'unknown' ? undefined : canonicalRow(row); + if (t.isStringLiteral(node)) return { kind: 'string', value: node.value }; + if (t.isNumericLiteral(node)) return { kind: 'number', text: String(node.value) }; + if (t.isBooleanLiteral(node)) return { kind: 'boolean', value: node.value }; + if (t.isNullLiteral(node)) return { kind: 'null' }; + if (t.isBigIntLiteral(node)) return { kind: 'bigint', text: String(node.value) }; + if (t.isRegExpLiteral(node)) return { kind: 'regexp', pattern: node.pattern, flags: node.flags }; + if ( + t.isIdentifier(node, { name: 'undefined' }) && + !scope.locals.has('undefined') && + resolveName('undefined', scope.record.chain) === undefined + ) + return { kind: 'undefined' }; + if (t.isUnaryExpression(node) && ['-', '+', '!'].includes(node.operator)) { + if (t.isNumericLiteral(node.argument) && node.operator !== '!') { + const value = node.operator === '-' ? -node.argument.value : +node.argument.value; + return { kind: 'number', text: Object.is(value, -0) ? '-0' : String(value) }; + } + if (t.isBigIntLiteral(node.argument) && node.operator === '-') + return { kind: 'bigint', text: String(-BigInt(node.argument.value)) }; + return undefined; + } + if ( + t.isCallExpression(node) && + t.isMemberExpression(node.callee) && + !node.callee.computed && + t.isIdentifier(node.callee.object, { name: 'expect' }) && + t.isIdentifier(node.callee.property) && + [ + 'objectContaining', + 'arrayContaining', + 'stringContaining', + 'stringMatching', + 'closeTo', + ].includes(node.callee.property.name) + ) { + const args = node.arguments.map((arg) => literal(arg, scope)); + return args.includes(undefined) ? undefined : { asymmetric: node.callee.property.name, args }; + } + if (t.isTemplateLiteral(node) && node.expressions.length === 0) + return { kind: 'string', value: node.quasis[0]?.value.cooked ?? node.quasis[0]?.value.raw }; + if (t.isArrayExpression(node)) { + const items = node.elements.map((item) => + item === null ? { kind: 'hole' } : literal(item, scope), + ); + return items.includes(undefined) ? undefined : { kind: 'array', items }; + } + if (t.isObjectExpression(node)) { + const values = new Map(); + for (const item of node.properties) { + if (!t.isObjectProperty(item) || item.computed) return undefined; + const key = t.isIdentifier(item.key) + ? item.key.name + : t.isStringLiteral(item.key) || t.isNumericLiteral(item.key) + ? String(item.key.value) + : null; + const value = literal(item.value, scope); + if (key === null || value === undefined) return undefined; + values.set(key, value); + } + const entries = [...values].map(([key, value]) => ({ key, value })); + return { kind: 'object', entries: entries.sort((a, b) => a.key.localeCompare(b.key)) }; + } + return undefined; +} + +/** Expand preceding immutable const bindings in their lexical scope. */ +function localOrigins( + subject: CheckSubject, + scope: Scope, +): (node: t.Node, before: number) => t.Node { + const references = new WeakSet(); + const declarations = new WeakMap(); + traverse(t.file(t.program([t.expressionStatement(subject.record.fn)])), { + Identifier(path) { + if (path.isReferencedIdentifier()) references.add(path.node); + const binding = path.scope.getBinding(path.node.name); + if (binding) declarations.set(path.node, binding); + }, + }); + let nextBinding = 0; + let visited = 0; + const expand = ( + node: t.Node, + before: number, + seen = new Set(), + bindings = new Map(), + ): t.Node => { + const unknown = (): t.Identifier => { + const marker = t.identifier('_unknown'); + marker.name = ''; + return marker; + }; + if (++visited > 5000 || seen.size > 32) { + scope.unresolved.add('assertion origin expansion limit reached'); + return unknown(); + } + const declaration = declarations.get(node); + if (references.has(node) && declaration) { + const bound = bindings.get(declaration); + if (bound) return bound; + // The inventory binds only the test callback's own parameters to table rows. + if (declaration.kind === 'param' && declaration.scope.block === subject.record.fn) + return node; + const decl = declaration.path.node; + if ( + declaration.kind !== 'const' || + !declaration.constant || + !t.isVariableDeclarator(decl) || + !t.isIdentifier(decl.id) || + !decl.init || + (decl.end ?? Infinity) >= before || + seen.has(declaration) + ) + return unknown(); + return expand(decl.init, before, new Set([...seen, declaration]), bindings); + } + const copy = t.cloneNode(node, false); + if (t.isFunction(copy) && copy.params.every((param) => t.isIdentifier(param))) { + bindings = new Map(bindings); + copy.params = copy.params.map((param) => { + const marker = t.identifier('_bound'); + marker.name = ``; + const binding = declarations.get(param); + if (binding) bindings.set(binding, marker); + return marker; + }); + } + const object = copy as unknown as Record; + for (const key of t.VISITOR_KEYS[node.type] ?? []) { + // Noncomputed property names are selectors, not references to local bindings. + if ( + (key === 'property' && + (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) && + !node.computed) || + (key === 'key' && t.isObjectProperty(node) && !node.computed) + ) + continue; + const value = object[key]; + object[key] = Array.isArray(value) + ? value.map((child) => (t.isNode(child) ? expand(child, before, seen, bindings) : child)) + : t.isNode(value) + ? expand(value, before, seen, bindings) + : value; + } + return copy; + }; + return (node, before) => { + nextBinding = 0; + visited = 0; + return expand(node, before); + }; +} + +function helperIdentity( + fn: t.FunctionDeclaration | t.FunctionExpression | t.ArrowFunctionExpression, + declaration: t.Node, + scope: Scope, +): string | null { + const copy = t.cloneNode(fn, true); + const statement = t.isFunctionDeclaration(copy) ? copy : t.expressionStatement(copy); + const frame = scope.record.chain.findIndex((item) => + [...item.decls.values()].some((decl) => decl.kind === 'node' && decl.node === declaration), + ); + if (frame < 0) return null; + const chain = scope.record.chain.slice(0, frame + 1); + let closed = true; + traverse(t.file(t.program([statement])), { + ReferencedIdentifier(path) { + if (path.scope.getBinding(path.node.name)) return; + const decl = resolveName(path.node.name, chain); + if (decl?.kind === 'import') path.node.name = `<${decl.key}>`; + else closed = false; + }, + }); + return closed ? sha1(generate(copy, { compact: true, comments: false }).code) : null; +} + +function containsRegExp(value: unknown): boolean { + if (value === null || typeof value !== 'object') return false; + if (Array.isArray(value)) return value.some(containsRegExp); + const record = value as Record; + return record.kind === 'regexp' || Object.values(record).some(containsRegExp); +} + +function preserveValues(node: t.Node, scope: Scope, abstractInputs = false): t.Node { + const value = literal(node, scope); + if (value !== undefined) { + const marker = t.identifier('_value'); + marker.name = ``; + return marker; + } + if (t.isIdentifier(node) && !scope.locals.has(node.name)) { + const decl = resolveName(node.name, scope.record.chain); + if (decl?.kind === 'import') { + const marker = t.identifier('_import'); + marker.name = `<${decl.key}>`; + return marker; + } + if (decl?.kind === 'node') { + const fn = t.isVariableDeclarator(decl.node) ? decl.node.init : decl.node; + if ( + t.isFunctionDeclaration(fn) || + t.isFunctionExpression(fn) || + t.isArrowFunctionExpression(fn) + ) { + const marker = t.identifier('_local'); + const identity = helperIdentity(fn, decl.node, scope); + marker.name = + identity === null + ? `` + : ``; + return marker; + } + // Frames do not prove constant values or immutability. Preserve the binding, + // so repeated reads can match without equating names in different scopes. + const marker = t.identifier('_local'); + marker.name = ``; + return marker; + } + } + if (t.isTemplateLiteral(node)) { + const marker = t.identifier('_template'); + marker.name = '