Merge branch 'main' into feat/adonisjs-taint

This commit is contained in:
Gergő Magyar 2026-11-02 08:27:37 +00:00 • committed by GitHub
commit 9ec2d52a93
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
307 changed files with 29436 additions and 1393 deletions

View file

@ -87,6 +87,7 @@ Generates repository documentation from the knowledge graph using an LLM. HTTP p
| `--concurrency <n>` | Parallel LLM calls (default: 3) |
| `--timeout <seconds>` | LLM request timeout in seconds (default: disabled) |
| `--retries <n>` | Max LLM retry attempts per request (default: 3) |
| `--max-completion-tokens <n>` | Max completion tokens per LLM request (default: 16384). `GITNEXUS_MAX_COMPLETION_TOKENS` overrides `maxCompletionTokens` in `~/.gitnexus/config.json`; the flag overrides both. |
| `--lang <lang>` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) |
| `--gist` | Publish wiki as a public GitHub Gist |

View file

@ -0,0 +1,53 @@
name: Download test artifact
description: Download a required artifact from this run with bounded transport retries
inputs:
name:
description: Exact artifact name to download from this workflow run
required: true
path:
description: Destination directory
required: true
runs:
using: composite
steps:
# Restart the whole action after its internal retries are exhausted, so
# each attempt obtains a fresh download URL for the same required artifact.
- name: Download
id: first
continue-on-error: true
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: ${{ inputs.name }}
path: ${{ inputs.path }}
digest-mismatch: error
- name: Wait before retry
if: ${{ !cancelled() && steps.first.outcome == 'failure' }}
shell: bash
run: sleep 10
- name: Retry download
id: second
if: ${{ !cancelled() && steps.first.outcome == 'failure' }}
continue-on-error: true
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: ${{ inputs.name }}
path: ${{ inputs.path }}
digest-mismatch: error
- name: Wait before final attempt
if: ${{ !cancelled() && steps.second.outcome == 'failure' }}
shell: bash
run: sleep 30
# Keep the final failure fatal: shards must never run without the snapshot.
- name: Final download attempt
if: ${{ !cancelled() && steps.second.outcome == 'failure' }}
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: ${{ inputs.name }}
path: ${{ inputs.path }}
digest-mismatch: error

View file

@ -43,14 +43,16 @@ jobs:
const path = require('path');
const runId = context.payload.workflow_run.id;
const allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
// Sharded CI runs can upload more than one page of artifacts.
const allArtifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
owner: context.repo.owner,
repo: context.repo.repo,
run_id: runId,
per_page: 100,
});
async function downloadArtifact(name, dest) {
const match = allArtifacts.data.artifacts.find(a => a.name === name);
const match = allArtifacts.find(a => a.name === name);
if (!match) {
core.warning(`Artifact "${name}" not found`);
return false;
@ -74,6 +76,8 @@ jobs:
- name: Extract artifacts
shell: bash
run: |
# With no artifacts, let the metadata step skip the report cleanly.
mkdir -p "$RUNNER_TEMP/dl"
cd "$RUNNER_TEMP/dl"
# Extract each artifact into its own directory to avoid filename collisions
for z in *.zip; do
@ -160,13 +164,14 @@ jobs:
// Try each run until we find a downloadable test-reports artifact
for (const run of runs.data.workflow_runs) {
const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
owner: context.repo.owner,
repo: context.repo.repo,
run_id: run.id,
per_page: 100,
});
const testReports = artifacts.data.artifacts.find(a => a.name === 'test-reports');
const testReports = artifacts.find(a => a.name === 'test-reports');
if (!testReports) {
core.info(`Run ${run.id}: no test-reports artifact, trying next`);
continue;

View file

@ -140,7 +140,7 @@ jobs:
run: npx tsx scripts/ensure-fts.ts
working-directory: gitnexus
- name: Download frozen timing snapshot
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
uses: ./.github/actions/download-test-artifact
with:
name: shard-timing-input
path: gitnexus/.vitest/timings
@ -201,7 +201,7 @@ jobs:
path: gitnexus/.vitest/blob
merge-multiple: true
- name: Download frozen timing snapshot
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
uses: ./.github/actions/download-test-artifact
with:
name: shard-timing-input
path: gitnexus/.vitest/timings
@ -291,7 +291,7 @@ jobs:
- name: Reuse this run's timing input on a retry
id: previous-input
if: steps.execution.outputs.initialize != 'true'
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
uses: ./.github/actions/download-test-artifact
with:
name: shard-timing-input
path: .vitest/timings
@ -404,7 +404,7 @@ jobs:
run: npx tsx scripts/ensure-fts.ts
working-directory: gitnexus
- name: Download frozen timing snapshot
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
uses: ./.github/actions/download-test-artifact
with:
name: shard-timing-input
path: gitnexus/.vitest/timings
@ -455,7 +455,7 @@ jobs:
with:
lifecycle-scripts: 'false'
- name: Download frozen timing snapshot
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
uses: ./.github/actions/download-test-artifact
with:
name: shard-timing-input
path: gitnexus/.vitest/timings
@ -940,6 +940,19 @@ jobs:
run: node --import tsx bench/trpc-route-extractor/measure.mjs --check
working-directory: gitnexus
- name: Go route-fact correctness and linear scaling (#3750)
if: ${{ !cancelled() }}
run: node --expose-gc --import tsx bench/go-route-facts/measure.js --check --output go-route-facts.json
working-directory: gitnexus
- name: Upload Go route-fact measurements
if: always()
uses: ./.github/actions/upload-test-artifact
with:
name: go-route-facts-benchmark
path: gitnexus/go-route-facts.json
retention-days: 5
- name: Cross-language scope-capture fingerprint + scaling guards
# Runs even after an earlier guard fails (#2895). Every step here was
# fail-fast, so the FIRST failing --check aborted the job and every guard
@ -1352,6 +1365,9 @@ jobs:
GITNEXUS_DEBUG_HEAP: '1'
run: node --import tsx bench/scope-resolution/dense-java.mts 300 20 10 --check 2>&1 | tee scope-resolution-dense-java.log
working-directory: gitnexus
- name: Check Java overload and ingestion scaling
run: node bench/java-overload-resolution/measure.js --check --output java-overload-resolution-benchmark.json
working-directory: gitnexus
- name: Upload scope-resolution measurements
if: always()
uses: ./.github/actions/upload-test-artifact
@ -1361,6 +1377,7 @@ jobs:
gitnexus/scope-resolution-benchmark.json
gitnexus/scope-resolution-call-signature-memory.json
gitnexus/scope-resolution-dense-java.log
gitnexus/java-overload-resolution-benchmark.json
retention-days: 5
# Locked eval suite. setup-uv and uv itself are immutable so CI exercises
@ -1534,10 +1551,8 @@ jobs:
sha="$(uv run --locked --extra dev python -m workflow_bench.release_report task-sha)"
[[ "$sha" =~ ^[0-9a-f]{40}$ ]]
git -C .. worktree add --detach "${RUNNER_TEMP}/tasks-base" "$sha"
cd "${RUNNER_TEMP}/tasks-base"
npm ci --audit=false --fund=false
npm ci --prefix gitnexus --audit=false --fund=false
npm run build --prefix gitnexus
uv run --locked --extra dev python -m workflow_bench.release_build \
--repo "${RUNNER_TEMP}/tasks-base"
- name: Prove process-tree and sandbox containment
env:
CLAUDE_CANARY_BIN: ${{ runner.temp }}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude
@ -1553,6 +1568,7 @@ jobs:
tests/test_mock_provider.py
tests/test_mcp_tool_loading.py
tests/test_release_evaluation_smoke.py
tests/test_release_controls.py
tests/test_evolve.py::test_outer_runner_pid_namespace_kills_setsid_descendant
tests/test_oracle_assets.py::test_hidden_vitest_config_executes_sibling_oracle_against_candidate_checkout
tests/test_oracle_assets.py::test_shipped_oracles_reject_task_base_and_accept_reference_fixes

View file

@ -17,6 +17,10 @@ on:
description: Start, probe, and stop EC2 without paid model calls
type: boolean
default: false
diagnostic_trial:
description: 'Two serial cells, $5 CLI stop each; requires a full candidate SHA, gpt-5.6-sol/xhigh, and no other experiment flags'
type: boolean
default: false
mcp_tool_loading:
description: Add candidate deferred-MCP experiment (12 extra paid cells)
type: boolean
@ -93,6 +97,25 @@ jobs:
cleanup_required: ${{ steps.boot.outputs.cleanup_required }}
stop_deadline_epoch: ${{ steps.boot.outputs.stop_deadline_epoch }}
steps:
- name: Validate diagnostic trial authorization scope
env:
TRIAL: ${{ inputs.diagnostic_trial }}
RUNNER_ONLY: ${{ inputs.runner_only }}
CLI_ARM: ${{ inputs.cli_arm }}
DEFERRED_ARM: ${{ inputs.mcp_tool_loading }}
CANDIDATE_REF: ${{ inputs.candidate_ref }}
MODEL: ${{ inputs.model }}
EFFORT: ${{ inputs.effort }}
run: |
set -euo pipefail
if [[ "$TRIAL" == true ]]; then
if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_RUN_ATTEMPT" != 1 ||
"$RUNNER_ONLY" == true || "$CLI_ARM" == true || "$DEFERRED_ARM" == true ||
"$MODEL" != gpt-5.6-sol || "$EFFORT" != xhigh || ! "$CANDIDATE_REF" =~ ^[0-9a-f]{40}$ ]]; then
echo '::error::Trial requires a fresh manual dispatch, a full candidate SHA, gpt-5.6-sol/xhigh, and no other experiment flags. Paid trials are never retried automatically.'
exit 1
fi
fi
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
@ -234,6 +257,7 @@ jobs:
working-directory: eval
env:
GH_TOKEN: ${{ github.token }}
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
run: |
set -euo pipefail
if [[ -n "$INPUT_PR" ]]; then
@ -244,10 +268,12 @@ jobs:
fi
[[ -z "$INPUT_PR_SHA" && -z "$COMPARISON_RUN_ID" ]] || { echo '::error::PR preview inputs require candidate_pr.'; exit 1; }
# Resolve last stable once; both runtimes use the same trusted tasks.
stable="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq '.tag_name')"
[[ "$stable" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo '::error::No stable release found.'; exit 1; }
stable_sha="$(git rev-parse --verify "$stable^{commit}")"
echo "stable_sha=$stable_sha" >> "$GITHUB_OUTPUT"
if [[ "${DIAGNOSTIC_TRIAL:-false}" != true ]]; then
stable="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq '.tag_name')"
[[ "$stable" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo '::error::No stable release found.'; exit 1; }
stable_sha="$(git rev-parse --verify "$stable^{commit}")"
echo "stable_sha=$stable_sha" >> "$GITHUB_OUTPUT"
fi
ref="$INPUT_REF"
if [[ "$ref" == 'latest-rc' ]]; then
ref="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --jq \
@ -275,7 +301,7 @@ jobs:
persist-credentials: false
fetch-depth: 0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
if: inputs.candidate_pr == ''
if: inputs.candidate_pr == '' && inputs.diagnostic_trial != true
with:
ref: ${{ steps.resolve.outputs.stable_sha }}
path: stable
@ -338,24 +364,41 @@ jobs:
test "$("$cli/node_modules/@anthropic-ai/claude-code-linux-x64/claude" --version)" = '2.1.214 (Claude Code)'
- name: Build pinned runtime and task dependencies
working-directory: eval
env:
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
run: |
set -euo pipefail
for checkout in tasks-base stable candidate; do
if [[ -n "$INPUT_PR" && "$checkout" == stable ]]; then continue; fi
if [[ ( -n "$INPUT_PR" || "${DIAGNOSTIC_TRIAL:-false}" == true ) && "$checkout" == stable ]]; then continue; fi
uv run --locked --extra dev python -m workflow_bench.release_build \
--repo "$GITHUB_WORKSPACE/$checkout"
done
- name: Prepare pinned task set
working-directory: eval
env:
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
run: |
set -euo pipefail
for runtime in stable candidate; do
if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi
if [[ ( -n "$INPUT_PR" || "${DIAGNOSTIC_TRIAL:-false}" == true ) && "$runtime" == stable ]]; then continue; fi
uv run --locked --extra dev python -m workflow_bench.release_report prepare \
--repo "$GITHUB_WORKSPACE/$runtime" --task-repo "$GITHUB_WORKSPACE/tasks-base" \
--out "$RUNNER_TEMP/release-evaluation/$runtime" \
--model "$MODEL" --effort "$EFFORT" --runs 3
done
- name: Prepare the capped diagnostic trial
if: inputs.diagnostic_trial == true
working-directory: eval
env:
RUNTIME_SHA: ${{ steps.resolve.outputs.sha }}
run: |
uv run --locked --extra dev python -m workflow_bench.diagnostic_experiments \
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
--repository "$GITHUB_WORKSPACE/tasks-base" --runtime "$GITHUB_WORKSPACE/candidate" \
--harness-sha "$GITHUB_SHA" --runtime-sha "$RUNTIME_SHA" \
--model "$MODEL" --effort "$EFFORT" --seed 3762 --repeats 1 \
--variants no_graph mcp_eager --task-ids trivial-status-json-alias \
--max-budget-usd 5 --bind-task-dependencies --out "$RUNNER_TEMP/release-evaluation/trial"
- name: Require historical task dependencies before paid PR sessions
if: inputs.candidate_pr != ''
working-directory: eval
@ -375,12 +418,23 @@ jobs:
- name: Materialize both release graphs before paid sessions
timeout-minutes: 120
working-directory: eval
env:
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
run: |
set -euo pipefail
if [[ "${DIAGNOSTIC_TRIAL:-false}" == true ]]; then
uv run --locked --extra dev python -m workflow_bench.release_preflight \
--tasks "$RUNNER_TEMP/release-evaluation/trial/cell-0000.tasks.json" \
--grading-controls \
--gitnexus-root "$GITHUB_WORKSPACE/candidate" \
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude"
exit 0
fi
for runtime in stable candidate; do
if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi
uv run --locked --extra dev python -m workflow_bench.release_preflight \
--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \
--grading-controls \
--gitnexus-root "$GITHUB_WORKSPACE/$runtime" \
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude"
done
@ -406,14 +460,29 @@ jobs:
env:
GITNEXUS_BENCH_OPENAI_API_KEY: ${{ secrets.GITNEXUS_BENCH_OPENAI_API_KEY }}
GITNEXUS_BENCH_ANTHROPIC_API_KEY: ${{ secrets.GITNEXUS_BENCH_ANTHROPIC_API_KEY }}
DIAGNOSTIC_TRIAL: ${{ inputs.diagnostic_trial }}
run: |
set -euo pipefail
if [[ "${DIAGNOSTIC_TRIAL:-false}" == true ]]; then
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
[[ "$GITHUB_RUN_ATTEMPT" == 1 ]] || { echo '::error::Paid trials cannot be rerun, including failed-job retries.'; exit 1; }
(( remaining > 0 )) || { echo '::error::No time remains before the stop and evidence reserve.'; exit 1; }
(( remaining <= 3600 )) || remaining=3600
timeout --signal=TERM --kill-after=60 "${remaining}s" \
uv run --locked --extra dev python -m workflow_bench.diagnostic_trial --execute \
--plan "$RUNNER_TEMP/release-evaluation/trial/plan.json" \
--runtime "$GITHUB_WORKSPACE/candidate" \
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude"
exit 0
fi
arms=(baseline_nomcp baseline)
if [[ -n "$INPUT_PR" ]]; then arms=(baseline); fi
for runtime in stable candidate; do
if [[ -n "$INPUT_PR" && "$runtime" == stable ]]; then continue; fi
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
(( remaining > 0 )) || { echo '::error::No time remains before the stop and evidence reserve.'; exit 1; }
GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/$runtime/provider-usage.jsonl" \
GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-$runtime" \
timeout --signal=TERM --kill-after=60 "${remaining}s" \
uv run --locked --extra dev python -m workflow_bench.runner \
--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \
@ -425,6 +494,8 @@ jobs:
done
# Descriptive only: its own output directory leaves the paired evidence
# that release_report and release_gate read exactly as without the opt-in.
# Trials reject these opt-ins before startup. Preserve the exact gates
# recognized by the historical measurement-protocol validator.
- name: Run the opt-in CLI-only arm on the candidate
if: env.CLI_ARM == 'true'
continue-on-error: true
@ -437,6 +508,8 @@ jobs:
set -euo pipefail
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
(( remaining > 0 )) || { echo '::warning::No time remains for the CLI-only arm.'; exit 1; }
GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/candidate/cli-provider-usage.jsonl" \
GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-cli" \
timeout --signal=TERM --kill-after=60 "${remaining}s" \
uv run --locked --extra dev python -m workflow_bench.runner \
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
@ -456,6 +529,8 @@ jobs:
set -euo pipefail
remaining=$(( ${{ needs.start-runner.outputs.stop_deadline_epoch }} - $(date +%s) - 5400 ))
(( remaining > 0 )) || { echo '::error::No time remains for deferred sessions before the evidence reserve.'; exit 1; }
GITNEXUS_BENCH_PROVIDER_USAGE="$RUNNER_TEMP/release-evaluation/candidate/deferred-provider-usage.jsonl" \
GITNEXUS_BENCH_SWEEP_ID="release-${{ github.run_id }}-${{ github.run_attempt }}-deferred" \
timeout --signal=TERM --kill-after=60 "${remaining}s" \
uv run --locked --extra dev python -m workflow_bench.runner \
--tasks "$RUNNER_TEMP/release-evaluation/candidate/tasks.yaml" \
@ -465,7 +540,7 @@ jobs:
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \
--out "$RUNNER_TEMP/release-evaluation/candidate/deferred"
- name: Summarize complete or incomplete measurements
if: ${{ always() && steps.resolve.outputs.sha != '' }}
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
working-directory: eval
run: |
if [[ -n "$INPUT_PR" ]]; then
@ -491,7 +566,7 @@ jobs:
done
exit "$failed"
- name: Require complete paired evidence and no task regression
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' }}
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }}
working-directory: eval
run: |
uv run --locked --extra dev python -m workflow_bench.release_gate \
@ -501,7 +576,7 @@ jobs:
--stable-sha '${{ steps.resolve.outputs.stable_sha }}' \
--out "$RUNNER_TEMP/release-evaluation/public"
- name: Summarize matched MCP loading experiment
if: ${{ always() && inputs.mcp_tool_loading == true && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' }}
if: ${{ always() && inputs.mcp_tool_loading == true && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }}
working-directory: eval
run: |
uv run --locked --extra dev python -m workflow_bench.mcp_loading_report \
@ -510,7 +585,7 @@ jobs:
--metadata "$RUNNER_TEMP/release-evaluation/candidate/metadata.json" \
--out "$RUNNER_TEMP/release-evaluation/public"
- name: Collect sanitized evidence and publish summary
if: ${{ always() && steps.resolve.outputs.sha != '' }}
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
run: |
set -euo pipefail
public="$RUNNER_TEMP/release-evaluation/public"
@ -529,7 +604,7 @@ jobs:
if [[ -f "$public/$report.md" ]]; then cat "$public/$report.md" >> "$GITHUB_STEP_SUMMARY"; fi
done
- name: Summarize the opt-in CLI-only arm
if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' }}
if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
continue-on-error: true
working-directory: eval
run: |
@ -539,8 +614,83 @@ jobs:
--paired "$candidate/raw/results.jsonl" --cli "$candidate/cli-raw/results.jsonl" \
--out "$RUNNER_TEMP/release-evaluation/public"
cat "$RUNNER_TEMP/release-evaluation/public/cli-arm-comparison.md" >> "$GITHUB_STEP_SUMMARY"
- name: Summarize time, cost, and tool measurements
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
working-directory: eval
env:
DEFERRED_ARM: ${{ inputs.mcp_tool_loading }}
run: |
set -euo pipefail
root="$RUNNER_TEMP/release-evaluation"
public="$root/public"
mkdir -p "$public"
failed=0
for variant in stable candidate cli deferred; do
case "$variant" in
stable)
if [[ -n "$INPUT_PR" ]]; then continue; fi
results="$root/stable/raw/results.jsonl"
ledger="$root/stable/provider-usage.jsonl" ;;
candidate)
results="$root/candidate/raw/results.jsonl"
ledger="$root/candidate/provider-usage.jsonl" ;;
cli)
if [[ "$CLI_ARM" != true ]]; then continue; fi
results="$root/candidate/cli-raw/results.jsonl"
ledger="$root/candidate/cli-provider-usage.jsonl" ;;
deferred)
if [[ "$DEFERRED_ARM" != true || -n "$INPUT_PR" ]]; then continue; fi
results="$root/candidate/deferred/results.jsonl"
ledger="$root/candidate/deferred-provider-usage.jsonl" ;;
esac
if ! uv run --locked --extra dev python -m workflow_bench.cost_diagnostics \
--results "$results" --provider-usage "$ledger" --allow-incomplete \
--out "$public/$variant-diagnostics.json" --markdown-out "$public/$variant-diagnostics.md"; then
printf '\nDetailed measurements for %s could not be validated; inspect the step log.\n' "$variant" >> "$GITHUB_STEP_SUMMARY"
failed=1
continue
fi
{
printf '\n<details><summary>Detailed measurements: %s</summary>\n\n' "$variant"
cat "$public/$variant-diagnostics.md"
printf '\n</details>\n'
} >> "$GITHUB_STEP_SUMMARY"
done
exit "$failed"
- name: Summarize the diagnostic trial
if: ${{ always() && inputs.diagnostic_trial == true && steps.resolve.outputs.sha != '' }}
working-directory: eval
run: |
set -euo pipefail
root="$RUNNER_TEMP/release-evaluation"
mkdir -p "$root/public"
printf "\n## Two-cell diagnostic trial\n\nOne task, one repetition per arm. \$5 CLI budget stop per cell; provider billing can exceed the planned \$10. This trial checks telemetry and is not release-quality evidence.\n" >> "$GITHUB_STEP_SUMMARY"
failed=0
for cell in cell-0000 cell-0001; do
if ! uv run --locked --extra dev python -m workflow_bench.cost_diagnostics \
--results "$root/trial/results/$cell/results.jsonl" \
--provider-usage "$root/trial/results/$cell.provider.jsonl" --allow-incomplete \
--out "$root/public/trial-$cell-diagnostics.json" \
--markdown-out "$root/public/trial-$cell-diagnostics.md"; then
printf '\nMeasurements for %s could not be validated.\n' "$cell" >> "$GITHUB_STEP_SUMMARY"
failed=1
continue
fi
cat "$root/public/trial-$cell-diagnostics.md" >> "$GITHUB_STEP_SUMMARY"
done
exit "$failed"
- name: Upload sanitized diagnostic trial measurements
if: ${{ always() && inputs.diagnostic_trial == true && steps.resolve.outputs.sha != '' }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: diagnostic-trial-${{ steps.resolve.outputs.sha }}-${{ github.run_id }}
path: |
${{ runner.temp }}/release-evaluation/public/trial-*-diagnostics.json
${{ runner.temp }}/release-evaluation/public/trial-*-diagnostics.md
if-no-files-found: error
retention-days: 30
- name: Upload sanitized release evidence
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' }}
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr == '' && inputs.diagnostic_trial != true }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-agent-evaluation-${{ steps.resolve.outputs.sha }}
@ -553,26 +703,32 @@ jobs:
${{ runner.temp }}/release-evaluation/public/release-quality-gate.md
${{ runner.temp }}/release-evaluation/public/mcp-tool-loading.json
${{ runner.temp }}/release-evaluation/public/mcp-tool-loading.md
${{ runner.temp }}/release-evaluation/public/*-diagnostics.json
${{ runner.temp }}/release-evaluation/public/*-diagnostics.md
if-no-files-found: error
retention-days: 30
- name: Upload the opt-in CLI-only arm comparison
if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' }}
if: ${{ always() && env.CLI_ARM == 'true' && steps.resolve.outputs.sha != '' && inputs.diagnostic_trial != true }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-cli-arm-${{ steps.resolve.outputs.sha }}
path: |
${{ runner.temp }}/release-evaluation/public/cli-arm-comparison.json
${{ runner.temp }}/release-evaluation/public/cli-arm-comparison.md
${{ runner.temp }}/release-evaluation/public/cli-diagnostics.json
${{ runner.temp }}/release-evaluation/public/cli-diagnostics.md
if-no-files-found: warn
retention-days: 30
- name: Upload sanitized PR cost comparison
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr != '' }}
if: ${{ always() && steps.resolve.outputs.sha != '' && inputs.candidate_pr != '' && inputs.diagnostic_trial != true }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: pr-cost-comparison-${{ steps.resolve.outputs.sha }}
path: |
${{ runner.temp }}/release-evaluation/public/pr-cost-comparison.json
${{ runner.temp }}/release-evaluation/public/pr-cost-comparison.md
${{ runner.temp }}/release-evaluation/public/*-diagnostics.json
${{ runner.temp }}/release-evaluation/public/*-diagnostics.md
if-no-files-found: error
retention-days: 30

View file

@ -226,6 +226,9 @@ jobs:
ls "$dest"
- name: Plan the run
env:
# Full-suite Layer 2 exceeds Node's default ~4 GiB heap. Budget 8 GiB
# for this step, leaving room on the 16 GB runner for native memory.
NODE_OPTIONS: --max-old-space-size=8192
SEED: ${{ inputs.seed || github.run_id }}
SHARDS: ${{ inputs.shards }}
MUTATION: ${{ steps.scope.outputs.mutation }}
@ -246,6 +249,9 @@ jobs:
${{ runner.temp }}/scenario-map/scenario-run-manifest.json
${{ runner.temp }}/scenario-map/layer0.json
${{ runner.temp }}/scenario-map/layer2.json
${{ runner.temp }}/scenario-map/consolidation.json
${{ runner.temp }}/scenario-map/consolidation.html
${{ runner.temp }}/scenario-map/fixtures.json
${{ runner.temp }}/scenario-map/coverage/coverage.json
if-no-files-found: error
retention-days: 14
@ -261,9 +267,9 @@ jobs:
needs: plan
if: ${{ !cancelled() && needs.plan.outputs.artifact != '' }}
runs-on: ubuntu-latest
# Locally: PDG index about 3 min (peak RSS about 8.4 GB of the runner's 16 GB), the whole
# stage about 5 min.
timeout-minutes: 60
# Full-suite tracing runs candidate files twice, including their runnable siblings.
# The 31k-case inventory exceeded 50 minutes; leave room for both passes and uploads.
timeout-minutes: 100
continue-on-error: true
permissions:
contents: read
@ -283,7 +289,7 @@ jobs:
name: ${{ needs.plan.outputs.artifact }}
path: ${{ runner.temp }}/scenario-map
- name: Run value flow
timeout-minutes: 50
timeout-minutes: 90
env:
GITNEXUS_WORKER_READY_TIMEOUT_MS: '120000'
run: |
@ -660,13 +666,14 @@ jobs:
import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import { shouldUpdateIssue } from './tools/scenario-map/report/issue-body.ts';
import { readReportFields } from './tools/scenario-map/report/json.ts';
import { completenessLine, degradedLine, runLink } from './tools/scenario-map/report/markdown.ts';
const { ISSUE_NUMBER, GITHUB_OUTPUT, RUNNER_TEMP } = process.env;
const reportDir = path.join(RUNNER_TEMP, 'scenario-map-report');
const issueDir = path.join(RUNNER_TEMP, 'scenario-map-issue');
const report = JSON.parse(
readFileSync(path.join(reportDir, 'scenario-map-report.json'), 'utf8'),
const report = await readReportFields(
path.join(reportDir, 'scenario-map-report.json'), ['run', 'completeness'],
);
const existing = ISSUE_NUMBER
? readFileSync(path.join(issueDir, 'existing-body.md'), 'utf8')

View file

@ -432,6 +432,60 @@ Wire format is **v2**: `2|<base>|<step>|<step>…`, one-character version prefix
Receivers the resolver could not type are not silently dropped. Each records a `ResolutionOutcome` (`scope-resolution/resolution-outcome.ts`) carrying the receiver's *shape* (`classifyReceiverShape`: `chain-call` / `chain-field` / `chain-mixed` / `chain-unwrap` / `no-chain` — the bench censuses these) and its *origin* (`in-program` / `external` / `unknown`). `scope-resolution/unresolved-receivers.ts` aggregates them per member name into the index-persisted `unresolvedReceiverMembers` summary, keeping in-program and external counts under separate keys. Only in-program drops make a count short: an external-rooted call (`System.out.println`, `fetch(...)`) has no in-graph node an edge could have reached, so it is reported but does not hedge. `impact` / `context` read that summary and publish `epistemic: 'exact' | 'lower-bound'`, prose `boundaries`, and the machine-readable `causes` split (`EpistemicCauses` in `mcp/local/local-backend.ts`).
Python's provider classifies attribute reads and assignment targets separately; augmented assignments produce both read and write references. Constructor assignments on a proven instance receiver also create `Property` nodes without requiring a value type annotation. Annotation-only targets do not assign a value. These rules follow the [Python attribute-reference](https://docs.python.org/3/reference/expressions.html#attribute-references) and [assignment](https://docs.python.org/3/reference/simple_stmts.html#assignment-statements) semantics.
Property references declined across language boundaries are recorded in `crossLanguageProperties`. Both `context` and upstream `impact` on an affected property report `epistemic: 'lower-bound'`, with the explanation in `boundaries` and `unresolved`, and the definition languages in `anchorLanguages`. This evidence does not fabricate cross-language edges and does not qualify a downstream walk.
### Java overload selection and missing call edges (#3789)
Java resolves an overload family through provider hooks before the shared passes
prune candidates by owner or conversion rank. The provider gathers eligible
members across the receiver's compile-time hierarchy, removes overridden
signatures, and applies the JLS 15.12.2 phases in order: strict invocation, loose
invocation, then variable arity. Within the first applicable phase it compares
parameter types pairwise for specificity. Boxing, primitive widening, arrays,
null, and concrete invariant generic arguments retain their distinct meanings.
Lexically scoped argument facts travel in Java's existing `captureSideChannel`,
so parse workers and durable cache replay make the same decisions.
Static imports use accessible overload families; an explicit import shadows only
the same signature from an on-demand import. Enum constant bodies and implicit
record accessors reuse the provider's existing synthetic identities. Field,
hierarchy, and subtype walks memoize shared ancestors while preserving cycle and
depth bounds.
The provider models a bounded set of qualified JDK type relationships and
`Arrays.asList` / `Collections.singletonList` calls. These factories are poly
expressions: their type arguments depend on the candidate's parameter type.
Source declarations and lexical shadowing take precedence over these summaries.
Unsupported generic inference, unavailable argument types, and incomplete
inherited inventories leave an overload unresolved. A complete original family
containing only one declaration may retain the existing unique-member inference
for opaque arguments; this exception does not apply to a family narrowed to one
candidate. Runtime implementation dispatch follows the selected signature.
It preserves the original receiver's subtype bound and excludes static and private
methods from virtual dispatch.
Captures also retain callable-owned return annotations and structural identities
for `var` initializers. The shared passes keep the selected compile-time method
separate from runtime dispatch targets, then propagate owned return types to
untyped assignments and revisit waiting receiver calls through a dependency
queue. Sequential assignments have no expression-depth cutoff. Unresolved producers
cannot restore an earlier name-based return guess. Generic implementation
signatures substitute a whole formal type parameter only when concrete source
heritage proves its binding; unsupported substitutions remain unknown.
Suppressed overload calls record graph identities for their candidate methods,
declaring owners, and caller. `unresolved-overloads.ts` reconciles decisions per
call site and resolution phase and persists a bounded `unresolvedOverloads`
receipt, including an explicit complete-empty receipt. `impact` and `context`
report matching omitted sites through `causes.overloadResolution` and
`epistemic: 'lower-bound'`; possible callers never become ordinary `CALLS`
edges or inflate known caller counts. Impact checks the nodes whose neighbors
were actually expanded before pagination, using candidate identities upstream
and caller identities downstream. Missing, malformed, or capped receipts retain
an uncertainty boundary. Re-analyze with the current build (`--force` when
repairing metadata) to replace a legacy receipt.
### Optional CFG/PDG emission (`--pdg`, #2081–#2086)
On a `--pdg` run providers build control-flow graphs through `LanguageProvider.cfgVisitor` (Tree-sitter ASTs) or `collectSourceCfgs` (standalone source processing). Every supported language supplies one of these hooks; Vue covers executable scripts. AST visitors run in the parse worker, while source-backed providers run during fresh scope extraction. Both produce plain `ParsedFile.cfgSideChannel` data. Scope-resolution emits the program-dependence layers **inside Phase 4 of `runScopeResolution`, while the disk-backed ParsedFile store is still live**. The store is cleared after that phase, so a later standalone phase cannot consume it.

View file

@ -1,5 +1,23 @@
# Migration Guide
## LadybugDB 0.22.0 storage and extensions
LadybugDB 0.22.0 reads indexes written by 0.21.1, but a write/checkpoint upgrades
their storage format from version 47 to 49. No manual data conversion is needed.
Use the packaged FTS and VECTOR binaries shipped with the new core; older
extension binaries can load successfully and then crash when building an index.
Before upgrading, stop index writers and readers and retain a backup of the
complete index storage directory if rollback is required. Include shared stores
when worktrees or sibling clones share an index. Keep clients sharing a writable
index on compatible engine versions.
After an index is written by 0.22.0, downgrading the dependency alone is not
enough: 0.21.1 refuses storage version 49. Restore the backup, or rebuild with the
older engine using `gitnexus analyze --force`. A rebuild under an engine that
cannot read the upgraded index cannot recover its cached embeddings; use the
backup to preserve them, or regenerate them with `--embeddings`.
## Automatic parser cache identities
Parser caches now use artifact-derived base and PDG identities instead of

View file

@ -43,6 +43,10 @@ query: shared `--http` ran the batch in 7.7 s at a flat 517 MB, versus 30.0 s an
You can also query the graph directly from the terminal — `gitnexus query`, `context`, `impact`, `trace`, `cypher`, `detect-changes`, and `check` mirror the MCP tools of the same names, and `gitnexus doctor` prints runtime platform capabilities.
`gitnexus impact <symbol> --max-tokens 1000` budgets compact JSON at four UTF-8 bytes per token. Single-repo callgraph results retain the nearest dependants first. Counts and risk remain complete; `pagination.trimmedForBudget` describes omitted symbols. Group and PDG results must fit intact. If required content cannot fit, the command returns a JSON error and exits nonzero. Extremely small budgets return JSON `null` with the same failure status. Budgeted output is compact even on a terminal; other direct tool output is compact when piped and indented on a terminal.
Impact defaults to 30 symbols per depth when pagination is omitted. Use `--limit` and `--offset` for explicit pages, or `GITNEXUS_IMPACT_DEFAULT_LIMIT=0` for all symbols. `--summary-only`, group queries and PDG mode bypass this default. Symbols list at most three processes, with `processCount` preserving the full count when shortened. `--max-tokens` is explicit on the CLI; `GITNEXUS_MCP_DEFAULT_MAX_TOKENS` applies only to MCP.
`gitnexus analyze --watch` requires a Git repository. It runs one initial
analysis, then debounces scanner-admitted working-tree changes for 300 ms by
default and applies serialized incremental refreshes. Events arriving during a
@ -224,15 +228,19 @@ sampling limits, and recovery.
Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max-file-size`, `--verbose`). Use the env-var form when you'd otherwise repeat the same flag every run, or when invoking GitNexus from a long-running host (MCP server, eval-server, CI shell) that already manages its own environment. CLI flags take precedence over `.gitnexusrc`, which takes precedence over env vars, which take precedence over built-in defaults.
| Query variable | Default | Effect | When to tune |
| --- | --- | --- | --- |
| Query variable | Default | Effect | When to tune |
| --------------------------- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
| `GITNEXUS_QUERY_MAX_RSS_MB` | max(8192 MiB, 25% of available RAM), capped at 75% of available RAM | Process-memory ceiling while pooled reads run; available RAM is the smaller of physical RAM and a reported runtime/container constraint. Positive MiB values override it; `0` disables the guard. Invalid, overflowing, or sub-byte values use the default. A breach exits the process with code 70. | Set in the CLI/MCP/serve environment when a legitimate read workload needs a different budget. It does not resize the database buffer pool. |
| `GITNEXUS_QUERY_TRACE` | unset | `1` writes query duration, row count, and sampled memory to stderr. Bound parameters are omitted; query text is included. | Diagnose an expensive read or a memory-guard exit. Inspect trace output before sharing it. |
| `GITNEXUS_QUERY_TRACE` | unset | `1` writes query duration, row count, and sampled memory to stderr. Bound parameters are omitted; query text is included. | Diagnose an expensive read or a memory-guard exit. Inspect trace output before sharing it. |
| Analyze configuration | Default | Effect | When to tune |
| --- | --- | --- | --- |
| `GITNEXUS_RC` | unset | Absolute path to a JSON file with the same keys as `.gitnexusrc`. Its keys override repository configuration; CLI flags still win. Missing, invalid, or oversized files fail analysis. | Keep local configuration outside the checkout. Re-indexing hooks and watch processes must inherit it too. Changing or removing router aliases or handler wrappers rebuilds route evidence. |
| Variable | Default | Effect | Tune when… |
| ----------------------------------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `GITNEXUS_TEST_REPORT` | unset | Write a JSON execution receipt from `scripts/run-cross-platform.ts`; accepts a basename ending in `.json`. | CI platform shards use distinct filenames so the completeness check can require every report. |
| `GITNEXUS_SHARD_PROFILE` | `coverage` | Selects the timing profile for Vitest shard balancing and merged timing reports: `coverage`, `windows-latest`, or `macos-latest`. | CI selects its native OS; set the matching profile when reproducing a shard locally. See [Automatic shard balancing](../TESTING.md#automatic-shard-balancing). |
| `GITNEXUS_SHARD_PROFILE` | `coverage` | Selects the timing profile for Vitest shard balancing and merged timing reports: `coverage`, `windows-latest`, or `macos-latest`. | CI selects its native OS; set the matching profile when reproducing a shard locally. See [Automatic shard balancing](../TESTING.md#automatic-shard-balancing). |
| `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers <n>`. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. |
| `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. |
| `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. |
@ -247,7 +255,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max
| `GITNEXUS_PROFILE_DEFERRED_SLOW_MS` | `3000` (verbose) / `5000` | Per-file threshold in ms above which `processCallsFromExtracted` emits a `slow file …` log line. Parsed via `Number()`: accepts integers (`5000`), scientific notation (`2.5e3`), decimals (`.5`), and hex (`0x10`). Non-finite or non-positive values fall back to the default. | Hunting a few outlier files dominating the deferred call-resolution stage; lower to surface more, raise to focus only on the worst. |
| `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. |
| `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size <kb>`. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. |
| `GITNEXUS_INCLUDE_DOT_DIRS` | unset | Comma-separated dot-directory names to index, e.g. `.github,.scallywag`. Changes take effect on the next analyze; status replays the recorded list. Hidden files, credential paths, and symlinks under opted-in paths remain excluded. Ignore rules still apply. | Indexing trusted source under hidden directories. For `.github`, also add `!.github/` to `.gitnexusignore`. Path protection does not scan source text for secrets. |
| `GITNEXUS_INCLUDE_DOT_DIRS` | unset | Comma-separated dot-directory names to index, e.g. `.github,.scallywag`. Changes take effect on the next analyze; status replays the recorded list. Hidden files, credential paths, and symlinks under opted-in paths remain excluded. Ignore rules still apply. | Indexing trusted source under hidden directories. For `.github`, also add `!.github/` to `.gitnexusignore`. Path protection does not scan source text for secrets. |
| `GITNEXUS_MAX_PROCESSES` | dynamic (`max(20, round(symbols/10))`) | Analyze-time process-detection process cap. Equivalent to `--max-processes <n>` / `.gitnexusrc` `maxProcesses`. Explicit values replace the dynamic formula (not a multiplier). `0` is invalid, not unlimited. Changing this re-detects flows on the next analyze without `--force`. Distinct from query-time `IMPACT_MAX_CHUNKS`. | `[processes] … whole flows are MISSING` names `--max-processes` after entry points were never traced or flows were dropped. Tracing does not start the next entry once collected traces already reach `maxProcesses * 2`; a started entry can still emit every trace that entry produces. |
| `GITNEXUS_MAX_PROCESS_BRANCHING` | `4` | Analyze-time per-node branching cap during flow tracing. Equivalent to `--max-process-branching <n>`. Shape-only: raising it shortens fewer traces; it does not restore whole missing flows. | A flow is present but `calleesDropped` is high at debug. |
| `GITNEXUS_MAX_PROCESS_TRACE_DEPTH` | `10` | Analyze-time DFS depth cap during flow tracing. Equivalent to `--max-process-trace-depth <n>`. Shape-only. | A reported flow is shorter than the code path (`tracesDepthCapped` at debug). |
@ -274,6 +282,9 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max
| `GITNEXUS_NO_GITIGNORE` | unset | When set, skips `.gitignore` parsing. `.gitnexusignore` is still honored. | Indexing a repo whose `.gitignore` excludes files you actually want indexed (e.g., generated code committed for cross-repo lookup). |
| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When exactly `1`, skips optional vendored Dart, Proto, Swift, Kotlin and Zig grammar setup. The C grammar and other native dependencies remain required. | Skipping optional language parsers when their prebuilds are unavailable and no source-build toolchain is installed. |
| `GITNEXUS_MCP_READ_ONLY` | unset | Set to `1` to expose only proven single-repository read tools and resources; `0` disables the policy and any other value fails startup. | The MCP server runs in an environment where graph mutation, raw Cypher, and cross-repository group routing must be unavailable. |
| `GITNEXUS_MCP_ENABLE_REFRESH` | unset | Set to `1` to list the `refresh_index` MCP tool on a local `gitnexus mcp` server (stdio or `--http`). `gitnexus serve` never exposes it, read-only mode hides and rejects it, and any value other than `0` or `1` fails startup. | Agents on a local server can see a stale index but have no shell to run `gitnexus analyze`. |
| `GITNEXUS_MCP_REFRESH_MAX_FILES` | `5000` | `refresh_index` refuses a repository whose working tree holds more files than this, counted the way analyze scans it before starting the rebuild. `0` removes the limit. | Raise it when larger repositories should be refreshable over MCP. |
| `GITNEXUS_MCP_REFRESH_TIMEOUT_MS` | `600000` | Maximum time one `refresh_index` call waits for its rebuild before returning an error. The worker is not interrupted and finishes in the background. At most `2147483647`. | A refresh needs longer than ten minutes. |
| `GITNEXUS_MCP_ALLOWED_REPOS` | unset | Comma-separated allowlist of canonical indexed repository names or absolute paths. Invalid, ambiguous, or blank entries fail startup. | One MCP process must expose only a bounded subset of the repositories in the global registry. |
| `GITNEXUS_MCP_DEFAULT_REPO` | unset | Canonical indexed repository name or absolute path used when a tool or resource omits its repository. Must belong to the allowlist when one is set. | Several repositories are available but unqualified MCP calls should resolve deterministically. |
| `GITNEXUS_MCP_DEFAULT_MAX_TOKENS` | unset | Default positive-integer response budget for MCP `query`, `context`, and `impact`, estimated at four UTF-8 bytes per token. Explicit `maxTokens` wins. | Long MCP responses consume too much model context and callers cannot reliably add a per-request budget. |

View file

@ -111,6 +111,17 @@ Set `GITNEXUS_MCP_READ_ONLY=1` before starting the MCP server to expose only the
The default is unchanged when the variable is unset or `0`. Any other value fails server startup rather than silently weakening the policy.
## MCP index refresh
A local server started with `GITNEXUS_MCP_ENABLE_REFRESH=1` also lists `refresh_index`, which rebuilds one registered repository's index from its working tree. It is meant for agents that can see a stale index (`staleness.status` of `behind` or `diverged`) but cannot run `gitnexus analyze` themselves.
- **Index-only.** The rebuild runs in the same forked analyze worker auto-sync uses, with `--skip-agents-md --skip-skills`: no `AGENTS.md`, `CLAUDE.md`, skill, hook, or other repository file is written. It is incremental where it can be and keeps existing embeddings and PDG mode.
- **Local only.** `gitnexus serve` (and so the Docker `/api/mcp` surface) never lists it, whatever its environment. With `GITNEXUS_MCP_READ_ONLY=1` it is hidden from discovery and rejected before dispatch.
- **Bounded.** Before starting, the server counts the repository's working tree the way analyze scans it (same patterns and ignore rules, stopping one past the limit) and refuses it when that count exceeds `GITNEXUS_MCP_REFRESH_MAX_FILES` (default `5000`; `0` for no limit, and no count). The count is taken before size filtering, so it can exceed what analyze then indexes. `dry_run` reports it as `estimatedFiles` (`null` when over the limit or with no limit) next to the last index's `lastIndexedFiles`. After `GITNEXUS_MCP_REFRESH_TIMEOUT_MS` (default ten minutes) the call stops waiting and returns an error. The analyze worker never interrupts an analysis it has started, so it finishes and publishes in the background, holding the index lock until then. Cancelling the request does not stop a started rebuild either. After timeout or cancellation, a new call gets "already running" and `dry_run` reports `inProgress` until the worker exits. Only one refresh per repository runs at a time. The file limit does not bound embedding generation, which `.gitnexusrc` can request.
- **Observable.** `dry_run: true` reports staleness, the indexed file count, whether `.gitnexusrc` makes the rebuild generate embeddings, and whether a refresh would run. A real call waits for the rebuild and returns `indexedAt`, `lastCommit`, `stats`, and `staleness`, with the values before the refresh under `previous`. A rebuild whose graph write collapsed is reported as an error, as `gitnexus analyze` and `serve` report it. Clients that send a progress token receive progress notifications; the MCP TypeScript SDK client gives up on a request after 60 seconds by default unless it resets that timeout on progress.
The tool takes `repo` and `dry_run` only. Like an applied `rename`, it needs an explicit `repo` unless one repository is indexed or an MCP default is configured.
## MCP repository policy
Set `GITNEXUS_MCP_ALLOWED_REPOS` to a comma-separated list of canonical registry names or absolute indexed paths. Entries are trimmed, resolved against the registry, and deduplicated at startup. When exactly one repository is allowed it becomes the implicit default; when several are allowed, callers must select one unless `GITNEXUS_MCP_DEFAULT_REPO` is also set.
@ -135,9 +146,13 @@ data; omission does not mean absence. Remove filters for a broader view. For
impact, existing `summaryOnly`, `limit` and `offset` already provide focused views.
Omitting these filters preserves the existing response.
The `query`, `context`, and `impact` tools accept an optional positive-integer `maxTokens` argument. It bounds the complete formatted MCP response, including hints and error text, using a deterministic four-UTF-8-bytes-per-token estimate. When truncation is required, the response ends with `…` and remains valid UTF-8.
The `query`, `context`, and `impact` tools accept an optional positive-integer `maxTokens` argument. It bounds the complete formatted MCP response, including hints and error text, using a deterministic four-UTF-8-bytes-per-token estimate. For `query` and `context`, truncation ends the response with `…` and preserves valid UTF-8.
Set `GITNEXUS_MCP_DEFAULT_MAX_TOKENS` to apply the same guardrail when callers do not send `maxTokens`. An explicit tool argument takes precedence. Leaving both unset preserves the existing response byte-for-byte; this is a transport guardrail, not semantic pagination or an exact model-specific tokenizer limit.
Set `GITNEXUS_MCP_DEFAULT_MAX_TOKENS` to apply the budget when callers do not send `maxTokens`. An explicit tool argument takes precedence. Leaving both unset disables token budgeting. The estimate is not an exact model-specific tokenizer limit.
For single-repo callgraph results, `impact` removes whole symbols from the deepest levels first. Counts, risk and completeness warnings remain intact, and `pagination.trimmedForBudget.shownByDepth` records the returned counts. If metadata alone cannot fit, it returns a JSON error with MCP `isError: true`; increase `maxTokens` or narrow the request. Budgets too small for an error object return JSON `null` with `isError: true`. Group and PDG results are not paginated or trimmed by this mechanism; an oversized result also returns a budget error.
Without explicit pagination, single-repo callgraph impact returns up to 30 symbols per depth. Client-supplied `offset: 0` and `summaryOnly: false` still use this default. Set `GITNEXUS_IMPACT_DEFAULT_LIMIT=0` for an unpaged result, or set `limit` and `offset` for explicit pages. Each symbol lists at most three processes; `processCount` records the full count when the list is shortened. Backend results used by the web UI and internal callers are unchanged.
## Tool Examples
@ -235,7 +250,7 @@ affected_processes: [LoginFlow, RegistrationFlow, ...]
### Rename (Graph and Semantic Layer)
```js
rename({symbol_name: "validateUser", new_name: "verifyUser", repo: "my-app", dry_run: true})
rename({ symbol_name: 'validateUser', new_name: 'verifyUser', repo: 'my-app', dry_run: true });
// Omitting dry_run also previews. For an ambiguous name, repeat with the symbol_uid from context().
```

View file

@ -20,6 +20,12 @@ gitnexus wiki --force
gitnexus wiki --timeout <seconds> # LLM request timeout in seconds (default: disabled)
gitnexus wiki --retries <n> # Max LLM retry attempts per request (default: 3)
# Raise the completion-token limit (default: 16384)
gitnexus wiki --max-completion-tokens 131072
GITNEXUS_MAX_COMPLETION_TOKENS=131072 gitnexus wiki
# Or set "maxCompletionTokens": 131072 in ~/.gitnexus/config.json
# Precedence: flag, then the environment variable, then the config file.
# Allow a specific LAN/self-hosted HTTP LLM host (HTTPS is preferred for remote endpoints)
gitnexus wiki --base-url http://llama-box.local:8080/v1 --allow-insecure-connection llama-box.local
# Or set a comma-separated host allowlist:
@ -29,6 +35,17 @@ GITNEXUS_ALLOW_INSECURE_CONNECTION=llama-box.local,192.168.1.23
gitnexus wiki --lang <lang> # e.g. english, chinese, spanish, japanese
```
The completion limit must be a positive integer supported by your model. It applies to
OpenAI-compatible HTTP providers; local agent CLI providers manage their own limits.
For reasoning models, the limit includes reasoning tokens as well as the visible answer.
When a provider reports `finish_reason: "length"`, GitNexus rejects the incomplete response
instead of saving it as a finished page. Failed modules are listed, the command exits with
status 1, and a normal rerun retries them even if the repository commit has not changed.
Increase `--max-completion-tokens` within your model's supported limit before retrying.
Truncated grouping or overview responses also fail the run; batched grouping retains its
directory-based fallback. Incomplete runs are not published with `--gist`.
Use `--force` to replace incomplete pages already saved by an older version.
For safety, `http://` LLM base URLs are allowed by default only for loopback hosts (`localhost`, `127.0.0.1`, `::1`). `--allow-insecure-connection` and `GITNEXUS_ALLOW_INSECURE_CONNECTION` accept exact hostnames or IP addresses only; do not include schemes, ports, paths, credentials, or wildcards.
The wiki generator reads the indexed graph structure, groups files into modules via LLM, generates per-module documentation pages, and creates an overview page — all with cross-references to the knowledge graph.

View file

@ -129,13 +129,17 @@ class _FakeMcpServer:
self.listener = socket.socket(socket.AF_UNIX)
self.listener.bind(str(path))
self.listener.listen(1)
self.listener.settimeout(0.1)
self.stopped = threading.Event()
self.thread = threading.Thread(target=self._serve, daemon=True)
self.thread.start()
def _serve(self) -> None:
while True:
while not self.stopped.is_set():
try:
connection, _ = self.listener.accept()
except TimeoutError:
continue
except OSError:
return
with connection, connection.makefile("rwb") as stream:
@ -157,7 +161,20 @@ class _FakeMcpServer:
stream.flush()
def close(self) -> None:
self.listener.close()
self.stopped.set()
try:
# Wait for accept to exit before releasing its fd for another server.
self.thread.join(timeout=5)
finally:
self.listener.close()
assert not self.thread.is_alive(), "fake MCP server did not stop"
@pytest.mark.skipif(not hasattr(socket, "AF_UNIX"), reason="the shim speaks MCP over a Unix socket")
def test_fake_mcp_server_close_stops_idle_accept_thread(tmp_path):
server = _FakeMcpServer(tmp_path / "mcp.sock", [])
server.close()
assert not server.thread.is_alive()
@pytest.mark.skipif(not hasattr(socket, "AF_UNIX"), reason="the shim speaks MCP over a Unix socket")

View file

@ -0,0 +1,526 @@
"""Offline measurements must not invent attribution, completeness or savings."""
import copy
import json
import os
import subprocess
import sys
from collections import Counter
from pathlib import Path
import pytest
import yaml
from tests.test_session_diagnostics import result, use
from workflow_bench.cost_diagnostics import build_cost_report, provider_summary
from workflow_bench.diagnostic_experiments import VARIANTS, prepare_plan
from workflow_bench.session_diagnostics import diagnose_session, public_diagnostics
from workflow_bench.workload_diagnostics import classify_bash
def trace(actions):
events = []
for index, (tool, args, body) in enumerate(actions):
events += [use(tool, tool_id=str(index), args=args), result(body, tool_id=str(index))]
return events
def test_graph_reference_is_not_confused_with_supplied_source_and_paths_stay_private():
events = trace([
("mcp__gitnexus__query", {}, {"definitions": [{"filePath": "src/private.ts"}]}),
("mcp__gitnexus__context", {}, {"symbol": {"filePath": "src/private.ts", "content": "secret source"}}),
("Read", {"file_path": "/workspace/src/private.ts", "offset": 10, "limit": 5}, "source"),
("Read", {"file_path": "/workspace/src/private.ts", "offset": 10, "limit": 5}, "source"),
("Read", {"file_path": "/workspace/other/private.ts"}, "unrelated"),
])
report = diagnose_session(events, None, 10)
work = report["workload"]
assert work["source_reads"] == 3
assert work["repeated_source_reads"] == 1
assert work["reads_after_graph_reference"] == work["reads_after_graph_content_reference"] == 2
assert work["query_context_read_chains"] == 2
assert "private.ts" not in json.dumps(public_diagnostics([report]))
assert "secret source" not in json.dumps(report)
def test_read_before_graph_result_is_not_credited_as_a_followup():
events = [use(), use("Read", tool_id="read", args={"file_path": "src/a.ts"}),
result({"filePath": "src/a.ts"}), result("source", tool_id="read")]
work = diagnose_session(events, None, 10)["workload"]
assert work["reads_after_graph_reference"] == 0
def test_server_reported_time_is_separate_from_host_wait_and_missing_stays_unknown():
events = trace([("mcp__gitnexus__query", {}, {"timing": {"wall": 125}, "epistemic": "lower-bound"}),
("mcp__gitnexus__impact", {}, {"risk": "LOW"})])
report = diagnose_session(events, [0, 2, 3, 5], 6)
safe = public_diagnostics([report])[0]
assert safe["tools"]["mcp__gitnexus__query"]["server_wall_s"] == 0.125
assert safe["tools"]["mcp__gitnexus__query"]["elapsed_s"] == 2
assert safe["tools"]["mcp__gitnexus__query"]["lower_bound"] == 1
assert safe["tools"]["mcp__gitnexus__impact"]["server_wall_s"] is None
@pytest.mark.parametrize("command", ["cd gitnexus && npm test", "uv run --locked --extra dev python -m pytest"])
def test_verification_repetition_respects_edits_and_unknown_shell_mutations(command):
check = ("Bash", {"command": command}, "ok")
events = trace([check, check, ("Edit", {"file_path": "secret"}, "ok"), check,
("Bash", {"command": "python custom_script.py"}, "ok"), check])
work = diagnose_session(events, None, 10)["workload"]
assert work["verification_calls"] == 4
assert work["verification_repeated_without_observed_change"] == 1
assert work["verification_repeated_after_possible_change"] == 2
assert work["edits_after_verification"] == 1
@pytest.mark.parametrize(("command", "category"), [
("uv run python -m pytest -q", "test"), ("npm run test:unit", "test"),
("uv run --locked --extra dev python -m pytest -q", "test"),
("uv run --frozen --no-sync --offline --extra=dev pytest", "test"),
("uv run --project ./eval --group dev --python 3.12 -- pytest", "test"),
("uv run --locked --extra dev ruff check .", "lint"),
("uv run --locked --extra dev ruff format .", "mutation"),
("uv run --unknown pytest", "other"), ("uv run --extra", "other"),
("uv run --extra= pytest", "other"), ("uv run --extra --locked pytest", "other"),
("uv run --locked=true pytest", "other"), ("uv run --locked", "other"),
("uv run -- --locked pytest", "other"),
("npx tsc --noEmit", "typecheck"), ("npx eslint .", "lint"),
("npx eslint . --fix", "mutation"), ("npm run build", "build"),
("cat source.ts", "source"), ("rg 'x' src", "source"),
("echo pytest", "other"), ("npm test && npm run build", "mixed"),
("cd gitnexus && npm test && touch x", "mixed"),
("cat x > y", "mixed"), ("sed -i 's/x/y/' a", "mutation"),
("sed --in-place 's/x/y/' a", "mutation"), ("npx prettier -w .", "mutation"),
("uv run ruff format .", "mutation"), ("uv run ruff format --check .", "lint"),
("echo $(npm test)", "mixed"), ("unterminated '", "other"),
])
def test_shell_classification_is_conservative(command, category):
assert classify_bash(command) == category
def test_usage_missing_fields_are_unknown_and_cache_counts_are_not_double_counted():
first = {"type": "assistant", "message": {"id": "a", "usage": {
"input_tokens": 10, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20, "output_tokens": 4,
}}}
second = {"type": "assistant", "message": {"id": "b", "usage": {"input_tokens": 30}}}
report = diagnose_session([first, second, {"type": "system", "subtype": "compact_boundary"}], None, 5)
work = report["workload"]
assert work["usage_snapshots"] == 2 and work["complete_input_snapshots"] == 1
assert work["max_input_tokens"] == work["first_input_tokens"] == 130
assert work["sum_input_tokens"] is work["last_input_tokens"] is None
assert work["sum_cache_read_tokens"] is work["sum_output_tokens"] is None
assert work["compactions"] == 1
work["prompt"] = "secret"
assert "secret" not in json.dumps(public_diagnostics([report]))
def provider_event():
return {"status": "success", "response_id": "one", "sweep_id": "sweep", "provider": "litellm-normalized",
"native_usage": {"prompt_tokens": 100, "completion_tokens": 20,
"prompt_tokens_details": {"cached_tokens": 60, "cache_write_tokens": 10}},
"request_metrics": {"tools_bytes": 300, "duration_s": 2}}
def test_provider_deduplicates_identical_hooks_and_refuses_conflicting_totals():
event = provider_event()
report = provider_summary([event, copy.deepcopy(event)])
assert report["observed_requests"] == 1 and report["duplicate_callbacks"] == 1
assert report["token_totals"]["total_input_tokens"] == 100
assert report["token_totals"]["ordinary_input_tokens"] == 30
assert report["token_totals"]["reasoning_output_tokens"] is None
assert report["attribution"] == "ledger_only_no_cell_join"
conflicting = copy.deepcopy(event)
conflicting["native_usage"]["prompt_tokens"] = 120
report = provider_summary([event, conflicting])
assert report["conflicting_response_ids"] == 1
assert all(value is None for value in report["token_totals"].values())
def test_missing_identity_failed_usage_and_unknown_providers_remain_visible():
event = provider_event()
del event["response_id"]
report = provider_summary([event, event, {"status": "failure", "provider": "unknown"}])
assert report["requests_without_response_id"] == 3 and not report["identity_complete"]
assert report["failures"] == report["invalid_usage_records"] == 1
assert report["token_totals"]["total_input_tokens"] is None
assert report["request_metrics"]["tools_bytes"]["missing"] == 1
def test_cost_report_keeps_failures_in_spend_and_missing_telemetry_is_not_zero():
rows = [{"arm": "baseline", "task": "test", "resolved": True, "ok": True, "cost_usd": 2},
{"arm": "baseline", "task": "test", "resolved": False, "ok": False, "cost_usd": 3}]
report = build_cost_report(rows)
group = report["groups"][0]
assert group["cost_per_solved"] == 5
assert group["failed_cells"] == 1 and group["diagnostic_cells"] == 0
assert group["metrics"]["duration_s"]["mean"] is None
rows[1]["cost_usd"] = float("nan")
assert build_cost_report(rows)["groups"][0]["cost_per_solved"] is None
def test_different_model_or_prompt_cohorts_and_duplicate_exports_cannot_blend():
row = {"arm": "baseline", "task": "test", "run": 0, "model": "model-a", "cost_usd": 2}
other = row | {"model": "model-b", "cost_usd": 20}
assert len(build_cost_report([row, other])["groups"]) == 2
with pytest.raises(ValueError, match="duplicate cell"):
build_cost_report([row, row])
def test_tool_summaries_preserve_cohort_boundaries_and_budget_treatment():
diagnostic = diagnose_session(trace([("Read", {"file_path": "source.ts"}, "source")]), None, 5)
row = {"arm": "baseline", "task": "test", "run": 0, "max_budget_usd": None,
"session_diagnostics": [diagnostic]}
report = build_cost_report([row, row | {"max_budget_usd": 5}])
assert len(report["groups"]) == 2
assert report["tool_markdown"].count("Diagnostics available for 1/1 cells") == 2
assert "Diagnostics available for 2/2 cells" not in report["tool_markdown"]
def test_proxy_envelope_metrics_preserve_accounting_without_recording_content(tmp_path, monkeypatch):
from types import SimpleNamespace
from workflow_bench.litellm_usage_callback import USAGE_LOG_ENV_VAR, ProviderUsageLogger
log = tmp_path / "provider.jsonl"
monkeypatch.setenv(USAGE_LOG_ENV_VAR, str(log))
kwargs = {"messages": [{"role": "user", "content": "SECRET PROMPT"}],
"optional_params": {"tools": [{"name": "PRIVATE TOOL"}]}, "response_cost": 0.25}
ProviderUsageLogger()._append("success", kwargs, SimpleNamespace(usage={"prompt_tokens": 20}), 10, 12)
payload = log.read_text()
row = json.loads(payload)
assert "SECRET" not in payload and "PRIVATE" not in payload
assert row["request_metrics"]["tools_count"] == 1
assert row["request_metrics"]["messages_count"] == 1
assert row["request_metrics"]["duration_s"] == 2
assert row["request_metrics"]["proxy_estimated_cost_usd"] == 0.25
ProviderUsageLogger()._append("failure", {"optional_params": [], "response_cost": float("nan")},
SimpleNamespace(usage={"prompt_tokens": 20}), None, None)
row = json.loads(log.read_text().splitlines()[-1])
assert row["native_usage"]["prompt_tokens"] == 20
assert row["request_metrics"]["tools_bytes"] is row["request_metrics"]["duration_s"] is None
@pytest.mark.parametrize("fail", [False, True])
def test_cell_phases_include_cleanup_and_partial_failure_without_changing_credit(monkeypatch, tmp_path, fail):
from tests.test_runner_hardening import _cell_context, _stub_cell_dependencies
from workflow_bench import runner
from workflow_bench.session_diagnostics import CELL_PHASES
_stub_cell_dependencies(monkeypatch, tmp_path)
if fail:
def explode(*args, **kwargs):
raise RuntimeError("synthetic failure")
monkeypatch.setattr(runner, "run_arm", explode)
row = runner.run_cell(_cell_context(tmp_path), 2, "workflow")
phases = row["cell_phase_seconds"]
assert set(phases) <= set(CELL_PHASES)
assert "cleanup" in phases and "agent_and_verification" in phases
assert ("artifacts" not in phases) == fail
assert sum(phases.values()) == pytest.approx(row["cell_duration_s"])
assert row["cell_finished_at"] >= row["cell_started_at"]
assert row["resolved"] is (not fail)
def test_release_and_cli_artifacts_publish_safe_diagnostics_and_keep_phase_validation(tmp_path):
from tests.test_cli_shim import _report_row, _write_results
from tests.test_release_report import DIGEST, NOW, SHA, metadata, rows
from workflow_bench import cli_arm_report, release_report
diagnostic = diagnose_session(trace([("Bash", {"command": "gitnexus query secret"}, "ok")]), None, 10)
extra = {"session_diagnostics": [diagnostic], "cell_duration_s": 12,
"cell_phase_seconds": {"clone": 1.0, "secret content": 99, "mcp_startup": float("nan")}}
measurements = rows()
measurements[1].update(extra)
report = release_report.build_report(measurements, metadata(), now=NOW)
release_report.validate_report(report, runtime_sha=SHA, task_set_digest=DIGEST, now=NOW)
assert report["per_run"][1]["cell_phase_seconds"] == {"clone": 1.0, "mcp_startup": None}
assert "secret" not in json.dumps(report)
paired, cli = _write_results(tmp_path, [_report_row("baseline", 0, 1), _report_row("baseline_nomcp", 0, 1)],
[_report_row("baseline_cli", 0, 1, **extra)])
report = cli_arm_report.build_report(paired, cli)
assert "secret" not in json.dumps(report)
assert "Tool diagnostics: baseline_cli" in cli_arm_report.render_markdown(report)
def plan_args():
return dict(repository="/repo", runtime="/runtime", runtime_sha="a" * 40, harness_sha="b" * 40,
model="gpt-5.6-sol", effort="xhigh", repeats=5, seed=3762, variants=list(VARIANTS))
def task_document():
return {"tasks": [
{"id": name, "ref": "c" * 40, "prompt": "Fix this", "class": task_class, "verify": "true",
"oracle": {"command": "true", "files": [{"source": "fixture.txt", "target": "fixture.txt"}]}}
for name, task_class in (("one", "trivial"), ("two", "cross-module"))
]}
def test_plan_is_deterministic_balanced_isolated_and_never_runs_a_model():
original = task_document()
plan = prepare_plan(original, **plan_args())
assert plan == prepare_plan(original, **plan_args())
assert len(plan["cells"]) == 50 and plan["position_balanced"]
assert plan["execution"] == "not_started" and plan["release_gate_eligible"] is False
assert original == task_document()
for task in ("one", "two"):
for variant in VARIANTS:
positions = Counter(c["position"] for c in plan["cells"] if c["task"] == task and c["variant"] == variant)
assert set(positions.values()) == {1} and len(positions) == 5
ledgers = {c["env"]["GITNEXUS_BENCH_PROVIDER_USAGE"] for c in plan["cells"]}
assert len(ledgers) == 50
for cell in plan["cells"]:
assert (cell["prompt_sha256"] != cell["base_prompt_sha256"]) == (cell["variant"] == "mcp_optional_prompt")
assert "--unsafe-no-bwrap" not in cell["argv"]
@pytest.mark.parametrize("override", [{"runtime_sha": "main"}, {"model": "latest"}, {"repeats": 0},
{"variants": ["cli", "cli"]}, {"task_ids": ["missing"]}])
def test_plan_rejects_unpinned_or_ambiguous_inputs(override):
with pytest.raises(ValueError):
prepare_plan(task_document(), **(plan_args() | override))
@pytest.mark.parametrize("document", [None, [], "tasks", 42])
def test_plan_rejects_non_mapping_yaml_documents(document):
with pytest.raises(ValueError, match="task document must contain tasks"):
prepare_plan(document, **plan_args())
@pytest.mark.parametrize("model", [None, 1, "", " ", " auto ", "claude-sonnet-LATEST",
"provider/latest", "provider:auto", "provider@AUTO"])
def test_plan_uses_runner_model_validation(model):
with pytest.raises(ValueError):
prepare_plan(task_document(), **(plan_args() | {"model": model}))
def test_plan_normalizes_model_in_metadata_and_commands():
plan = prepare_plan(task_document(), **(plan_args() | {"model": " gpt-5.6-sol "}))
assert plan["model"] == "gpt-5.6-sol"
for cell in plan["cells"]:
assert cell["argv"][cell["argv"].index("--model") + 1] == "gpt-5.6-sol"
@pytest.mark.parametrize("task_id", ["_task", "-task", "a" * 129])
def test_plan_rejects_task_ids_the_runner_cannot_use(task_id):
document = task_document()
document["tasks"][0]["id"] = task_id
with pytest.raises(ValueError, match="artifact-safe slug"):
prepare_plan(document, **plan_args())
def test_every_prepared_task_satisfies_the_runner_metadata_contract():
from workflow_bench.runner_tasks import select_tasks
document = task_document()
document["tasks"][0]["id"] = "a.task-1"
plan = prepare_plan(document, **plan_args())
for cell in plan["cells"]:
selected, skipped = select_tasks(cell["task_document"]["tasks"], include_expensive=True)
assert selected == cell["task_document"]["tasks"] and skipped == []
document["tasks"][0]["verify"] = ""
with pytest.raises(ValueError, match="verify"):
prepare_plan(document, **plan_args())
@pytest.mark.parametrize("intervening_read", [False, True])
def test_post_graph_input_totals_require_every_relevant_snapshot(intervening_read):
complete = {"type": "assistant", "message": {"id": "complete", "usage": {
"input_tokens": 10, "cache_read_input_tokens": 100, "cache_creation_input_tokens": 20,
}}}
incomplete = {"type": "assistant", "message": {"id": "incomplete", "usage": {"input_tokens": 30}}}
events = [use(), result({}), complete]
if intervening_read:
events += [use("Read", tool_id="read", args={"file_path": "src/a.ts"}), result("source", tool_id="read")]
events.append(incomplete)
work = diagnose_session(events, None, 5)["workload"]
assert work["sum_input_tokens"] is None
assert work["usage_after_graph_snapshots"] == (1 if intervening_read else 2)
assert work["input_tokens_after_graph"] == (130 if intervening_read else None)
# Preserve the actual first/last positions, even when a snapshot is incomplete.
reversed_usage = diagnose_session([incomplete, complete], None, 5)["workload"]
assert reversed_usage["first_input_tokens"] is None and reversed_usage["last_input_tokens"] == 130
complete_usage = diagnose_session([complete], None, 5)["workload"]
assert complete_usage["sum_input_tokens"] == 130
def test_trial_plan_passes_each_cell_budget_without_starting_a_model():
args = plan_args() | {"repeats": 1, "variants": ["no_graph", "mcp_eager"], "task_ids": ["one"], "max_budget_usd": 5.0}
plan = prepare_plan(task_document(), **args)
assert len(plan["cells"]) == 2
assert plan["planned_cli_budget_usd"] == 10.0
assert plan["execution"] == "not_started"
for cell in plan["cells"]:
assert cell["argv"][cell["argv"].index("--max-budget-usd") + 1] == "5.0"
@pytest.mark.parametrize("budget", [0, -1, float("nan"), float("inf"), True, 10 ** 1000])
def test_invalid_budget_fails_before_any_process_starts(monkeypatch, tmp_path, budget):
from workflow_bench import runner_sessions
def must_not_start(*args, **kwargs):
pytest.fail("invalid budget launched a process")
monkeypatch.setattr(runner_sessions, "run_managed", must_not_start)
with pytest.raises(ValueError, match="max-budget-usd"):
runner_sessions.run_claude("task", tmp_path, claude_bin="claude", timeout=1, max_budget_usd=budget)
def test_cli_budget_stop_remains_a_failed_billed_session(monkeypatch, tmp_path):
from tests.test_workflow_bench_sessions import fake_cli_result, report_variant
from workflow_bench import runner_sessions
captured = []
def run(command, **kwargs):
captured.extend(command)
return fake_cli_result(report_variant(subtype="error_max_budget_usd", total_cost_usd=5.1))
monkeypatch.setattr(runner_sessions, "run_managed", run)
record = runner_sessions.run_claude("task", tmp_path, claude_bin="claude", timeout=1, max_budget_usd=5)
assert captured[captured.index("--max-budget-usd") + 1] == "5.0"
assert record["ok"] is False and record["cost_usd"] == 5.1
assert record["max_budget_usd"] == 5
def test_budget_rejects_multisession_arms_before_reading_task_inputs(monkeypatch, capsys):
from workflow_bench import runner
monkeypatch.setattr("sys.argv", ["runner", "--tasks", "nonexistent.yaml", "--model", "gpt-5.6-sol",
"--arms", "workflow", "--max-budget-usd", "5"])
with pytest.raises(SystemExit) as failure:
runner.main()
assert failure.value.code == 2
assert "--max-budget-usd is limited to single-session baseline arms" in capsys.readouterr().err
def test_summary_cli_gathers_safe_measurements_and_provider_coverage(tmp_path, monkeypatch):
from workflow_bench import cost_diagnostics
diagnostic = diagnose_session(trace([
("mcp__gitnexus__query", {"query": "SECRET QUERY"}, {"staleness": {"status": "behind"}}),
("Bash", {"command": "uv run --locked --extra dev pytest"}, "SECRET RESULT"),
]), [0, 2, 3, 5], 8)
rows = [{"task": "one", "arm": "baseline", "run": 0, "cost_usd": 2, "duration_s": 8,
"ok": False, "resolved": False, "session_diagnostics": [diagnostic], "private": "SECRET PATH",
"cell_phase_seconds": {"clone": 1, "agent_and_verification": 9, "PRIVATE PHASE": 123}}]
results, ledger = tmp_path / "results.jsonl", tmp_path / "provider.jsonl"
results.write_text(json.dumps(rows[0]) + "\n")
event = provider_event() | {"messages": "SECRET PROMPT", "response_id": "SECRET ID"}
ledger.write_text(json.dumps(event) + "\n")
out, markdown = tmp_path / "diagnostics.json", tmp_path / "diagnostics.md"
monkeypatch.setattr("sys.argv", ["cost_diagnostics", "--results", str(results), "--provider-usage", str(ledger),
"--out", str(out), "--markdown-out", str(markdown)])
cost_diagnostics.main()
body = markdown.read_text()
assert "SECRET" not in out.read_text() + body and "PRIVATE PHASE" not in body
assert "Solved: 0/1; failed sessions: 1" in body
assert "| clone | 1 / 0 | 1 |" in body
assert "| verification_calls | 1 | 1/1 |" in body
assert "| tools_bytes | 1 / 0 | 300 |" in body
assert "| total_input_tokens | 100 |" in body
assert "Largest, slowest, and problematic tool calls" in body
assert "requests without identity: 0" in body
assert json.loads(out.read_text())["provider"]["ledger_complete"] is True
def test_summary_cli_reports_missing_legacy_telemetry_and_torn_provider_rows(tmp_path, monkeypatch):
from workflow_bench import cost_diagnostics
results, ledger = tmp_path / "results.jsonl", tmp_path / "provider.jsonl"
results.write_text(json.dumps({"task": "old", "arm": "baseline", "cost_usd": 1}) + "\n{torn")
ledger.write_text(json.dumps(provider_event()) + "\n{torn")
out, markdown = tmp_path / "diagnostics.json", tmp_path / "diagnostics.md"
monkeypatch.setattr("sys.argv", ["cost_diagnostics", "--results", str(results), "--provider-usage", str(ledger),
"--allow-incomplete", "--out", str(out), "--markdown-out", str(markdown)])
cost_diagnostics.main()
report = json.loads(out.read_text())
assert len(report["input_problems"]) == 2
assert report["provider"]["observed_requests"] == 1
assert report["provider"]["ledger_complete"] is False
assert all(value is None for value in report["provider"]["token_totals"].values())
body = markdown.read_text()
assert "Partial input" in body and "diagnostics: 0/1" in body
assert "| clone | 0 / 1 | n/a |" in body
assert "| total_input_tokens | n/a |" in body
def test_summary_cli_reports_unavailable_inputs_after_early_failure(tmp_path, monkeypatch):
from workflow_bench import cost_diagnostics
out, markdown = tmp_path / "diagnostics.json", tmp_path / "diagnostics.md"
monkeypatch.setattr("sys.argv", ["cost_diagnostics", "--results", str(tmp_path / "missing"),
"--provider-usage", str(tmp_path / "missing-ledger"), "--allow-incomplete",
"--out", str(out), "--markdown-out", str(markdown)])
cost_diagnostics.main()
assert "Observed cells: 0" in markdown.read_text()
assert "Provider ledger unavailable" in markdown.read_text()
assert json.loads(out.read_text())["provider"] is None
def test_summary_escapes_markdown_and_html_labels():
from workflow_bench.cost_diagnostics import render_markdown
report = build_cost_report([{"arm": "baseline", "task": "</code><img src=x>|bad![x](https://example.test)"}])
markdown = render_markdown(report)
assert "<img" not in markdown and "&#124;" in markdown
assert "![x]" not in markdown
def test_summary_does_not_turn_missing_tool_counters_into_zero():
from workflow_bench.session_diagnostics import SCHEMA, render_diagnostics
body = "\n".join(render_diagnostics([{"arm": "baseline", "session_diagnostics": [
{"schema": SCHEMA, "tools": {"Read": {"calls": 1}}},
]}]))
assert "| Read | 1 | n/a |" in body
assert "Other session wall time (seconds): n/a." in body
def test_summary_treats_unrepresentable_numbers_as_unknown():
report = build_cost_report([{"arm": "baseline", "task": "one", "cost_usd": 10 ** 1000}])
assert report["groups"][0]["metrics"]["cost_usd"]["observed"] == 0
assert report["groups"][0]["metrics"]["cost_usd"]["missing"] == 1
@pytest.mark.skipif(os.name == "nt", reason="release summary executes in bash on the Linux runner")
@pytest.mark.parametrize(("pr", "expected"), [("", {"stable", "candidate", "cli", "deferred"}),
("3773", {"candidate", "cli"})])
def test_actual_workflow_summary_step_publishes_each_enabled_sweep(tmp_path, pr, expected):
repository = Path(__file__).resolve().parents[2]
workflow = yaml.safe_load((repository / ".github/workflows/release-evaluation.yml").read_text())
step = next(step for job in workflow["jobs"].values() for step in job.get("steps", [])
if step.get("name") == "Summarize time, cost, and tool measurements")
# Run the workflow's actual shell and report command with this test's Python;
# dependency provisioning and paid model commands are outside this step.
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
uv = bin_dir / "uv"
uv.write_text('#!/bin/sh\n[ "$1 $2 $3 $4 $5" = "run --locked --extra dev python" ] || exit 90\n'
'shift 5\nexec "$SUMMARY_TEST_PYTHON" "$@"\n')
uv.chmod(0o755)
root = tmp_path / "release-evaluation"
for runtime, directory, ledger in (("stable", "raw", "provider-usage.jsonl"),
("candidate", "raw", "provider-usage.jsonl"),
("candidate", "cli-raw", "cli-provider-usage.jsonl"),
("candidate", "deferred", "deferred-provider-usage.jsonl")):
out = root / runtime / directory
out.mkdir(parents=True)
out.joinpath("results.jsonl").write_text(json.dumps({
"arm": "baseline_cli" if directory == "cli-raw" else "baseline", "task": "one",
"run": 0, "cost_usd": 2, "duration_s": 5, "resolved": True, "ok": True,
}) + "\n")
(root / runtime / ledger).write_text(json.dumps(provider_event() | {"request": "SECRET"}) + "\n")
summary = tmp_path / "summary.md"
completed = subprocess.run(["bash", "-euo", "pipefail", "-c", step["run"]], cwd=repository / "eval",
env={**os.environ, "PATH": f"{bin_dir}:{os.environ['PATH']}",
"SUMMARY_TEST_PYTHON": sys.executable, "RUNNER_TEMP": str(tmp_path),
"GITHUB_STEP_SUMMARY": str(summary), "INPUT_PR": pr,
"CLI_ARM": "true", "DEFERRED_ARM": "true"},
capture_output=True, text=True, timeout=60)
assert completed.returncode == 0, completed.stderr
assert {p.name.removesuffix("-diagnostics.json") for p in (root / "public").glob("*.json")} == expected
body = summary.read_text()
for variant in expected:
assert f"Detailed measurements: {variant}" in body
assert json.loads((root / "public" / f"{variant}-diagnostics.json").read_text())["provider"]["observed_requests"] == 1
assert "SECRET" not in body

View file

@ -0,0 +1,241 @@
"""The paid trial must stay at two bounded calls and publish private-safe results."""
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import time
from types import SimpleNamespace
import pytest
import yaml
from workflow_bench import diagnostic_experiments, diagnostic_trial
from workflow_bench.session_diagnostics import diagnose_session
ROOT = Path(__file__).resolve().parents[2]
SHA = "a" * 40
@pytest.fixture
def trial_plan(tmp_path, monkeypatch):
runtime = tmp_path / "candidate"
runtime.mkdir()
task_repo = tmp_path / "tasks-base"
task_repo.mkdir()
monkeypatch.setattr(diagnostic_experiments, "trial_task_bindings", lambda tasks, expected=None: [{"id": tasks[0]["id"]}])
output = tmp_path / "release-evaluation" / "trial"
monkeypatch.setattr(sys, "argv", [
"prepare", "--tasks", str(ROOT / "eval/workflow_bench/tasks.scenarios.yaml"),
"--repository", str(task_repo), "--runtime", str(runtime), "--runtime-sha", SHA,
"--harness-sha", SHA, "--model", diagnostic_trial.MODEL, "--repeats", "1",
"--variants", "no_graph", "mcp_eager", "--task-ids", diagnostic_trial.TASK,
"--max-budget-usd", "5", "--bind-task-dependencies", "--out", str(output),
])
diagnostic_experiments.main()
monkeypatch.setattr(diagnostic_trial, "_head", lambda repository:
"c4ecf398de637cf779ab2238e69958cfd5d601ce" if repository == task_repo else SHA)
return output / "plan.json", runtime
def fake_runner(calls, *, codes=(0, 0), missing=None):
events = [
{"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "one", "name": "Read",
"input": {"file_path": "PRIVATE"}}]}},
{"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "one", "content": "PRIVATE"}]}},
]
diagnostics = diagnose_session(events, [1, 2], 3)
def run(command, *, cwd, env, check):
calls.append((command, env))
output = Path(command[command.index("--out") + 1])
output.mkdir(parents=True)
row = {
"task": diagnostic_trial.TASK, "arm": command[command.index("--arms") + 1], "run": 0,
"model": diagnostic_trial.MODEL, "cost_usd": 0.1, "resolved": True, "ok": True,
"session_diagnostics": [diagnostics] if missing != "diagnostics" else [],
}
if missing != "results":
(output / "results.jsonl").write_text(json.dumps(row) + "\n")
if missing != "ledger":
Path(env["GITNEXUS_BENCH_PROVIDER_USAGE"]).write_text(json.dumps({
"status": "success", "provider": "openai", "response_id": "private-id",
"native_usage": {"prompt_tokens": 10, "completion_tokens": 2}, "prompt": "PRIVATE",
}) + "\n")
return SimpleNamespace(returncode=codes[len(calls) - 1])
return run
@pytest.mark.parametrize("codes", [(0, 0), (1, 0), (0, 1)])
def test_trial_executes_exactly_two_serial_budgeted_cells_and_retains_failures(trial_plan, monkeypatch, codes):
path, runtime = trial_plan
calls = []
monkeypatch.setattr(diagnostic_trial.subprocess, "run", fake_runner(calls, codes=codes))
assert diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") == int(any(codes))
assert len(calls) == 2
assert [command[command.index("--arms") + 1] for command, _ in calls] == ["baseline", "baseline_nomcp"]
for command, _ in calls:
for flag, value in (("--max-budget-usd", "5.0"), ("--workers", "1"), ("--runs", "1"),
("--timeout", "900"), ("--model", diagnostic_trial.MODEL), ("--claude-bin", "/pinned/claude")):
assert command[command.index(flag) + 1] == value
assert command[:3] == [sys.executable, "-m", "workflow_bench.runner"]
assert len({env["GITNEXUS_BENCH_PROVIDER_USAGE"] for _, env in calls}) == 2
receipt = json.loads((path.parent / "execution.json").read_text())
assert [cell["exit_code"] for cell in receipt["cells"]] == list(codes)
assert all(cell["measurements_present"] for cell in receipt["cells"])
assert receipt["status"] == ("failed" if any(codes) else "completed")
with pytest.raises(ValueError, match="repeat paid work"):
diagnostic_trial.execute_trial(path, runtime, "/pinned/claude")
assert len(calls) == 2
@pytest.mark.parametrize("field,value", [("model", "other"), ("workers", 2), ("repeats", 3),
("max_budget_usd_per_cell", 10), ("planned_cli_budget_usd", 20),
("release_gate_eligible", True), ("execution", "started"), ("seed", 1)])
def test_policy_changes_fail_before_any_paid_process(trial_plan, monkeypatch, field, value):
path, runtime = trial_plan
plan = json.loads(path.read_text())
plan[field] = value
path.write_text(json.dumps(plan))
calls = []
monkeypatch.setattr(diagnostic_trial.subprocess, "run", fake_runner(calls))
with pytest.raises(ValueError):
diagnostic_trial.execute_trial(path, runtime, "/pinned/claude")
assert calls == []
@pytest.mark.parametrize("change", ["third_cell", "extra_argument", "manifest_hash", "extra_task", "ledger", "revision"])
def test_cell_tampering_fails_before_any_paid_process(trial_plan, monkeypatch, change):
path, runtime = trial_plan
plan = json.loads(path.read_text())
cell = plan["cells"][0]
manifest = path.parent / "cell-0000.tasks.json"
if change == "third_cell":
plan["cells"].append(cell.copy())
elif change == "extra_argument":
cell["argv"] += ["--runs", "10"]
elif change == "manifest_hash":
manifest.write_text(manifest.read_text() + " ")
elif change == "extra_task":
document = json.loads(manifest.read_text())
document["tasks"].append(document["tasks"][0].copy())
manifest.write_text(json.dumps(document))
cell["task_manifest_sha256"] = hashlib.sha256(manifest.read_bytes()).hexdigest()
elif change == "ledger":
cell["env"]["GITNEXUS_BENCH_PROVIDER_USAGE"] = "/tmp/reused-ledger"
else:
plan["runtime_sha"] = "b" * 40
path.write_text(json.dumps(plan))
calls = []
monkeypatch.setattr(diagnostic_trial.subprocess, "run", fake_runner(calls))
with pytest.raises(ValueError):
diagnostic_trial.execute_trial(path, runtime, "/pinned/claude")
assert calls == []
def test_interrupted_trial_cannot_be_reexecuted_without_a_new_plan(trial_plan, monkeypatch):
path, runtime = trial_plan
calls = []
def crash(*args, **kwargs):
calls.append(args)
raise OSError("synthetic launch failure")
monkeypatch.setattr(diagnostic_trial.subprocess, "run", crash)
with pytest.raises(OSError):
diagnostic_trial.execute_trial(path, runtime, "/pinned/claude")
with pytest.raises(FileExistsError):
diagnostic_trial.execute_trial(path, runtime, "/pinned/claude")
assert len(calls) == 1
@pytest.mark.parametrize("missing", ["results", "ledger", "diagnostics"])
def test_missing_measurements_cannot_make_the_trial_green(trial_plan, monkeypatch, missing):
path, runtime = trial_plan
calls = []
monkeypatch.setattr(diagnostic_trial.subprocess, "run", fake_runner(calls, missing=missing))
assert diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") == 1
assert len(calls) == 2
def workflow():
return yaml.safe_load((ROOT / ".github/workflows/release-evaluation.yml").read_text())
@pytest.mark.parametrize("override", [{}, {"TRIAL": "false", "GITHUB_RUN_ATTEMPT": "2"},
{"GITHUB_RUN_ATTEMPT": "2"}, {"GITHUB_EVENT_NAME": "schedule"},
{"RUNNER_ONLY": "true"}, {"CLI_ARM": "true"}, {"DEFERRED_ARM": "true"},
{"MODEL": "other"}, {"EFFORT": "max"}, {"CANDIDATE_REF": "latest-rc"}])
def test_real_startup_guard_rejects_expanded_or_repeated_paid_trials(override):
step = workflow()["jobs"]["start-runner"]["steps"][0]
env = {**os.environ, "TRIAL": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_RUN_ATTEMPT": "1",
"RUNNER_ONLY": "false", "CLI_ARM": "false", "DEFERRED_ARM": "false", "CANDIDATE_REF": SHA,
"MODEL": diagnostic_trial.MODEL, "EFFORT": "xhigh", **override}
result = subprocess.run(["bash", "-c", step["run"]], env=env, capture_output=True)
assert (result.returncode == 0) == (not override or override.get("TRIAL") == "false")
@pytest.mark.parametrize("attempt", ["1", "2"])
def test_real_paid_step_routes_trial_to_bounded_executor_without_full_sweep(tmp_path, attempt):
steps = workflow()["jobs"]["evaluate"]["steps"]
script = next(step["run"] for step in steps if step.get("name") == "Run repeated paired agent sessions")
script = re.sub(r"\$\{\{.*?\}\}", str(int(time.time()) + 7200), script)
binary = tmp_path / "uv"
binary.write_text(f"#!{sys.executable}\nimport json,os,sys\nwith open(os.environ['CAPTURE'],'a') as f: f.write(json.dumps(sys.argv[1:])+'\\n')\n")
binary.chmod(0o755)
capture = tmp_path / "calls.jsonl"
result = subprocess.run(["bash", "-c", script], capture_output=True, env={
**os.environ, "DIAGNOSTIC_TRIAL": "true", "CAPTURE": str(capture), "PATH": f"{tmp_path}:{os.defpath}",
"RUNNER_TEMP": str(tmp_path), "GITHUB_WORKSPACE": str(tmp_path), "GITHUB_RUN_ATTEMPT": attempt,
})
if attempt != "1":
assert result.returncode != 0 and not capture.exists()
return
assert result.returncode == 0
calls = [json.loads(line) for line in capture.read_text().splitlines()]
assert len(calls) == 1
assert "workflow_bench.diagnostic_trial" in calls[0] and "--execute" in calls[0]
assert "workflow_bench.runner" not in calls[0]
def test_trial_is_manual_only_and_cannot_publish_release_evidence():
document = workflow()
assert document[True]["workflow_dispatch"]["inputs"]["diagnostic_trial"]["default"] is False
assert "diagnostic_trial" not in document[True]["workflow_call"]["inputs"]
steps = document["jobs"]["evaluate"]["steps"]
excluded = {"Summarize complete or incomplete measurements", "Require complete paired evidence and no task regression",
"Collect sanitized evidence and publish summary", "Summarize time, cost, and tool measurements",
"Upload sanitized release evidence", "Upload sanitized PR cost comparison"}
for step in steps:
if step.get("name") in excluded:
assert "inputs.diagnostic_trial != true" in step["if"]
upload = next(step for step in steps if step.get("name") == "Upload sanitized diagnostic trial measurements")
assert "always()" in upload["if"] and "inputs.diagnostic_trial == true" in upload["if"]
assert upload["with"]["name"].startswith("diagnostic-trial-")
assert all("/public/trial-*-diagnostics." in line for line in upload["with"]["path"].splitlines())
assert "refs/heads/main" in document["jobs"]["evaluate"]["if"]
assert document["jobs"]["evaluate"]["environment"] == "gitnexus-evolution"
def test_real_trial_summary_publishes_both_cells_without_private_payloads(trial_plan, tmp_path, monkeypatch):
path, runtime = trial_plan
calls = []
with monkeypatch.context() as scoped:
scoped.setattr(diagnostic_trial.subprocess, "run", fake_runner(calls))
assert diagnostic_trial.execute_trial(path, runtime, "/pinned/claude") == 0
binary = tmp_path / "uv"
binary.write_text(f"#!{sys.executable}\nimport os,sys\na=sys.argv[1:]; os.execv(sys.executable,[sys.executable,*a[a.index('python')+1:]])\n")
binary.chmod(0o755)
summary = tmp_path / "summary.md"
step = next(step for step in workflow()["jobs"]["evaluate"]["steps"] if step.get("name") == "Summarize the diagnostic trial")
subprocess.run(["bash", "-c", step["run"]], check=True, cwd=ROOT / "eval", capture_output=True, env={
**os.environ, "PATH": f"{tmp_path}:{os.defpath}", "RUNNER_TEMP": str(tmp_path), "GITHUB_STEP_SUMMARY": str(summary),
})
body = summary.read_text()
assert "Two-cell diagnostic trial" in body and "planned $10" in body
assert body.count("Observed cells: 1") == 2
assert "PRIVATE" not in body and "private-id" not in body
assert len(list((tmp_path / "release-evaluation/public").glob("trial-*-diagnostics.json"))) == 2

View file

@ -322,6 +322,12 @@ def test_shared_lifecycle_jobs_stay_identical_across_both_ec2_workflows():
renamed = {"watch-evaluate-pickup": "watch-evolve-pickup", "evaluate": "evolve"}
for job in ("start-runner", "check-runner", "runner-ready", "stop-runner"):
expected = {key: value for key, value in evaluation[job].items() if key != "name"}
if job == "start-runner":
guard, *lifecycle = expected["steps"]
assert guard["name"] == "Validate diagnostic trial authorization scope"
assert "uses" not in guard and "secrets." not in json.dumps(guard)
# Input rejection precedes EC2 startup; compare every lifecycle step.
expected["steps"] = lifecycle
if job != "stop-runner":
preview = "(github.event_name == 'pull_request_target' && inputs.candidate_pr != '') ||\n "
assert expected["if"].count(preview) == 1

View file

@ -170,7 +170,12 @@ def test_a_request_through_the_real_gateway_records_native_usage(tmp_path, monke
) as gateway:
request = urllib.request.Request(
gateway.base_url + "/v1/messages",
data=json.dumps({"model": "gpt-4.1", "max_tokens": 32, "messages": [{"role": "user", "content": "ping"}]}).encode(),
data=json.dumps({
"model": "gpt-4.1", "max_tokens": 32,
"messages": [{"role": "user", "content": "PRIVATE REQUEST"}],
"tools": [{"name": "inspect_file", "description": "PRIVATE TOOL DESCRIPTION",
"input_schema": {"type": "object", "properties": {"path": {"type": "string"}}}}],
}).encode(),
headers={"Content-Type": "application/json", "x-api-key": gateway.auth_token, "anthropic-version": "2023-06-01"},
)
with urllib.request.urlopen(request, timeout=60):
@ -189,6 +194,11 @@ def test_a_request_through_the_real_gateway_records_native_usage(tmp_path, monke
assert native["prompt_tokens_details"]["cache_write_tokens"] == 1_000
assert event["provider"] == LITELLM_NORMALIZED
assert event["call_type"] == "anthropic_messages", "the observed call type, not a Responses one"
metrics = event["request_metrics"]
assert metrics["messages_count"] == metrics["tools_count"] == 1
assert metrics["messages_bytes"] > 0 and metrics["tools_bytes"] > 0
assert metrics["duration_s"] >= 0
assert "PRIVATE" not in usage_log.read_text(), "numeric envelope evidence must not retain request content"
usage = normalize_usage(event["provider"], native)
assert usage.total_input_tokens == 10_000

View file

@ -73,6 +73,23 @@ def test_pr_comparison_rejects_readable_runtime_cells(evidence):
assert not report["complete"] and not report["improved"]
def test_pr_summary_preserves_safe_diagnostics_in_complete_and_partial_reports(evidence):
from tests.test_session_diagnostics import result, use
from workflow_bench.session_diagnostics import diagnose_session
diagnostic = diagnose_session([use(args={"query": "SECRET"}), result("SECRET")], [1, 2], 3)
evidence[1][0]["session_diagnostics"] = [diagnostic]
evidence[1][0]["cell_phase_seconds"] = {"clone": 2, "SECRET": 9}
report = compare(evidence)
assert report["per_run"][0]["cell_phase_seconds"] == {"clone": 2}
assert "SECRET" not in json.dumps(report)
assert "Tool diagnostics: baseline" in pr_evaluation.render_markdown(report)
evidence[1].pop()
partial = compare(evidence)
assert not partial["complete"]
assert "Tool diagnostics: baseline" in pr_evaluation.render_markdown(partial)
def test_comparison_measures_only_new_mcp_cells_and_preserves_historical_cost(evidence):
before = deepcopy(evidence)
report = compare(evidence)

View file

@ -0,0 +1,71 @@
"""Reference grading must work before a paid release or telemetry session starts."""
from contextlib import nullcontext
import os
from pathlib import Path
from types import SimpleNamespace
import pytest
import yaml
from workflow_bench import release_controls
from workflow_bench.proposer_sandbox import SandboxError, preflight_bubblewrap
from workflow_bench.task_assets import TaskAssetCache
@pytest.mark.parametrize("failure", [None, "negative_passes", "startup", "permission", "patch", "typecheck", "reference"])
def test_controls_require_behavioral_failure_then_a_typed_reference(monkeypatch, tmp_path, failure):
task = {"id": "trivial-status-json-alias"}
clone = tmp_path / "clone"
clone.mkdir()
mounts, calls = [], []
sandbox = SimpleNamespace(clone=clone, environment=lambda: {},
command_prefix_for=lambda **kw: mounts.append(kw) or ["sandbox"])
monkeypatch.setattr(release_controls, "make_worktree", lambda *_: clone)
monkeypatch.setattr(release_controls, "sanitize_clone_for_hidden_oracles", lambda *_: None)
monkeypatch.setattr(release_controls, "stage_task_assets", lambda *_a, **_kw: [])
monkeypatch.setattr(release_controls, "capture_task_oracle", lambda *_: object())
monkeypatch.setattr(release_controls, "prepare_sandbox", lambda **_: nullcontext(sandbox))
def oracle(*_args, diagnostics=None):
calls.append("negative" if diagnostics is not None else "reference")
if diagnostics is not None:
diagnostics.update(exit_code=1, assertion_failure=True, startup_failure=failure == "startup",
permission_failure=failure == "permission")
return failure == "negative_passes", "hidden output withheld"
return failure != "reference", "hidden output withheld"
def verify(command, *_args, **_kwargs):
step = "patch" if command.startswith("git apply") else "typecheck"
calls.append(step)
return failure != step, "PRIVATE OUTPUT"
monkeypatch.setattr(release_controls, "_run_hidden_oracle", oracle)
monkeypatch.setattr(release_controls, "run_verify", verify)
if failure:
with pytest.raises(SandboxError) as error:
release_controls.check_task_controls(task, repo=tmp_path, sha="a" * 40, parent=tmp_path,
assets=object(), claude_bin=Path("claude"), bwrap_bin=Path("bwrap"))
assert "PRIVATE" not in str(error.value)
else:
release_controls.check_task_controls(task, repo=tmp_path, sha="a" * 40, parent=tmp_path,
assets=object(), claude_bin=Path("claude"), bwrap_bin=Path("bwrap"))
assert calls == ["negative", "patch", "typecheck", "reference"]
assert mounts[-1] == {"read_only_workspace": True, "unshare_network": True}
if failure in ("negative_passes", "startup", "permission"):
assert calls == ["negative"]
@pytest.mark.skipif(os.environ.get("GITNEXUS_REQUIRE_BWRAP_CANARY") != "1",
reason="requires the provisioned Linux containment runner")
def test_real_release_reference_controls_use_pinned_dependencies_and_read_only_grading(tmp_path):
# No provider, proxy, or agent process. The pinned CLI executable is only a
# sandbox mount; grading invokes the historical TypeScript/Vitest toolchain.
repo = Path(os.environ["GITNEXUS_ORACLE_TASK_DEPENDENCIES"])
claude = Path(os.environ["CLAUDE_CANARY_BIN"])
tasks = yaml.safe_load((Path(__file__).parents[1] / "workflow_bench/tasks.scenarios.yaml").read_text())["tasks"]
task = next(t for t in tasks if t["id"] == "trivial-status-json-alias")
with TaskAssetCache(tmp_path / "assets") as cache:
snapshot = cache.prepare(task, repo=repo, resolved_sha=task["ref"])
release_controls.check_task_controls(task, repo=repo, sha=task["ref"], parent=tmp_path,
assets=snapshot, claude_bin=claude, bwrap_bin=preflight_bubblewrap())

View file

@ -39,6 +39,7 @@ def prepared(monkeypatch, tmp_path):
monkeypatch.setattr(release_preflight, "preflight_bubblewrap", lambda: Path("/fake/bwrap"))
monkeypatch.setattr(release_preflight, "require_claude_sandbox_helpers", lambda: None)
monkeypatch.setattr(release_preflight, "trusted_gitnexus_runtime_mounts", lambda **_: ())
monkeypatch.setattr(release_preflight, "check_task_controls", lambda *_a, **_kw: None)
monkeypatch.setattr(task_assets, "_try_reflink", lambda *_: False)
# Only graph construction/containment are substituted. Binding resolution,
@ -101,3 +102,27 @@ def test_preflight_rejects_task_assets_that_replace_the_graph(prepared, tmp_path
release_preflight.preflight_release(
[{**task, "sandbox_copy": [".gitnexus/meta.json"]}], gitnexus_root=tmp_path, claude_bin=Path("claude"),
)
def test_grading_control_failure_stops_preflight_with_captured_task_dependencies(prepared, monkeypatch, tmp_path):
task, _, _ = prepared
dependency = Path(task["repo"]) / "node_modules" / "fixture"
dependency.mkdir(parents=True)
(dependency / "index.js").write_bytes(b"module.exports = 'pinned dependency';\n")
task["sandbox_dependencies"] = [{"source": "node_modules", "target": "node_modules"}]
calls = []
def reject(task_arg, **kwargs):
calls.append(kwargs)
assets = kwargs["assets"]
assert task_arg == task
assert assets.dependency_declarations == (("node_modules", "node_modules"),)
assert len(assets.dependencies) == 1
captured = assets.root / assets.dependencies[0].snapshot_path / "fixture/index.js"
assert captured.read_bytes() == b"module.exports = 'pinned dependency';\n"
raise SandboxError("reference grading control failed")
monkeypatch.setattr(release_preflight, "check_task_controls", reject)
with pytest.raises(SandboxError, match="reference grading control failed"):
release_preflight.preflight_release([task], gitnexus_root=tmp_path, claude_bin=Path("claude"), grading_controls=True)
assert len(calls) == 1 and calls[0]["sha"] == task["ref"]

View file

@ -0,0 +1,65 @@
"""Partial retrieval judgments must not turn unjudged code into false positives."""
import copy
import json
from pathlib import Path
import pytest
from workflow_bench.retrieval_diagnostics import score_retrieval
CASES = {"source_sha": "a" * 40, "query_options": {"limit": 5, "max_symbols": 10, "include_content": False}, "cases": [
{"id": "case", "query": "find handler", "anchors": [{"file": "src/a.ts", "name": "handle"}]},
{"id": "missing", "query": "another query", "anchors": [{"file": "src/b.ts"}]},
]}
def recordings():
return {"source_sha": "a" * 40, "runtime_sha": "b" * 40, "index_complete": True,
"index_staleness": "current", "sanitized_graph": True,
"query_options": CASES["query_options"],
"cases": [{"id": "case", "query": "find handler", "response": {
"process_symbols": [{"filePath": "src/unjudged.ts", "name": "other"}],
"definitions": [{"filePath": "src/a.ts", "name": "handle"}],
}}]}
def test_partial_relevance_and_missing_cases_do_not_become_precision_or_zero():
report = score_retrieval(CASES, recordings())
assert report["cases"][0]["anchor_recall"] == 1
assert report["cases"][0]["first_anchor_position"] == 2
assert report["cases"][1]["anchor_recall"] is None
assert not report["complete"]
assert "precision" not in report["cases"][0]
@pytest.mark.parametrize("change", [{"source_sha": "c" * 40}, {"index_staleness": "behind"},
{"index_complete": False}, {"sanitized_graph": False},
{"sanitized_graph": "false"}, {"runtime_sha": "main"}, {"query_options": {"limit": 1}}])
def test_unmatched_or_incomplete_index_is_not_scored(change):
with pytest.raises(ValueError):
score_retrieval(CASES, recordings() | change)
def test_changed_query_duplicate_case_and_missing_symbol_are_distinct():
recorded = recordings()
recorded["cases"][0]["query"] = "different question"
with pytest.raises(ValueError, match="query differs"):
score_retrieval(CASES, recorded)
recorded = recordings()
recorded["cases"].append(copy.deepcopy(recorded["cases"][0]))
with pytest.raises(ValueError, match="duplicate"):
score_retrieval(CASES, recorded)
recorded = recordings()
recorded["cases"][0]["response"] = {"process_symbols": [], "definitions": []}
assert score_retrieval(CASES, recorded)["cases"][0]["anchor_recall"] == 0
def test_committed_cases_cover_each_current_scenario_with_pinned_source():
import yaml
root = Path(__file__).resolve().parents[1] / "workflow_bench"
cases = json.loads((root / "retrieval_cases.json").read_text())
tasks = yaml.safe_load((root / "tasks.scenarios.yaml").read_text())["tasks"]
assert {c["id"] for c in cases["cases"]} == {t["id"] for t in tasks}
assert {t["ref"] for t in tasks} == {cases["source_sha"]}

View file

@ -214,7 +214,7 @@ def _expectation(**overrides: Any) -> ComparatorReuseExpectation:
binding = TaskReuseBinding(
task_base_sha=SHA,
task_prompt_digest=runner.hashlib.sha256(b"review it").hexdigest(),
task_prompt_digest=runner.task_prompt_digest({"prompt": "review it"}),
oracle_digest="oracle-content",
oracle_command_digest="oracle-command",
oracle_manifest_digest="oracle-manifest",
@ -258,7 +258,7 @@ def test_a_changed_binding_rejects_the_same_emitted_row(
binding = TaskReuseBinding(
task_base_sha=SHA,
task_prompt_digest=runner.hashlib.sha256(b"review it").hexdigest(),
task_prompt_digest=runner.task_prompt_digest({"prompt": "review it"}),
oracle_digest="oracle-content",
oracle_command_digest="oracle-command",
oracle_manifest_digest="oracle-manifest",
@ -266,3 +266,14 @@ def test_a_changed_binding_rejects_the_same_emitted_row(
sandbox_dependency_manifest_digest="DIFFERENT-dependencies",
)
assert row_is_reusable_comparator(row, _expectation(tasks={TASK_ID: binding})) is False
def test_old_prompt_identity_cannot_reuse_a_row_without_the_environment_instruction(emitted_row):
legacy = {**emitted_row, "task_prompt_digest": runner.hashlib.sha256(b"review it").hexdigest()}
assert not row_is_reusable_comparator(legacy, _expectation())
def test_changed_environment_instruction_invalidates_a_previously_matching_row(emitted_row, monkeypatch):
assert row_is_reusable_comparator(emitted_row, _expectation())
monkeypatch.setattr(runner, "MCP_REPOSITORY_PROMPT", runner.MCP_REPOSITORY_PROMPT + " Changed instruction.")
assert not row_is_reusable_comparator(emitted_row, _expectation())

View file

@ -0,0 +1,302 @@
"""Diagnostics must explain overhead without changing credit or exposing content."""
import json
from collections import Counter
import pytest
from tests.test_release_report import DIGEST, NOW, SHA, metadata, rows
from tests.test_workflow_bench_sessions import VALID_REPORT, fake_cli_result, session_record
from workflow_bench import runner_sessions, session_diagnostics as diagnostics
from workflow_bench.release_report import build_report, render_markdown, validate_report
def use(tool="mcp__gitnexus__query", *, tool_id="one", args=None):
return {"type": "assistant", "message": {"content": [
{"type": "tool_use", "id": tool_id, "name": tool, "input": args or {"query": "private source"}},
]}}
def result(body, *, tool_id="one", error=False):
content = body if isinstance(body, str) else [{"type": "text", "text": json.dumps(body) + "\n\n---\nNext"}]
return {"type": "user", "message": {"content": [
{"type": "tool_result", "tool_use_id": tool_id, "is_error": error, "content": content},
]}}
@pytest.mark.parametrize("message", [None, {}, {"id": "message"}, {"content": None}, [], "invalid"])
def test_top_level_content_preserves_tool_results_and_source_followups(message):
events = [use(), result({"definitions": []}), use("Read", tool_id="read"), result("source", tool_id="read")]
for event in events:
event["content"] = event["message"]["content"]
event["message"] = message
report = diagnostics.diagnose_session(events, [1, 2, 3, 4], 5)
assert report["timing_complete"] is True
assert report["tool_wait_s"] == 2
query = report["tools"]["mcp__gitnexus__query"]
assert query["calls"] == query["empty"] == query["source_followups"] == 1
assert query["unfinished"] == 0
assert report["tools"]["Read"]["calls"] == 1
@pytest.mark.parametrize("nested", [[], "invalid"])
def test_present_non_null_nested_content_prevents_top_level_fallback(nested):
events = [use(), result("done")]
for event in events:
event["content"] = event["message"]["content"]
event["message"]["content"] = nested
report = diagnostics.diagnose_session(events, [1, 2], 3)
assert report["calls"] == []
assert report["tools"] == {}
def test_nested_tool_content_wins_over_conflicting_top_level_blocks():
events = [use(), result("done")]
events[0]["content"] = use("Read", tool_id="shadow")["message"]["content"]
events[1]["content"] = result("shadow", tool_id="shadow")["message"]["content"]
report = diagnostics.diagnose_session(events, [1, 2], 3)
assert list(report["tools"]) == ["mcp__gitnexus__query"]
assert report["tools"]["mcp__gitnexus__query"]["calls"] == 1
assert report["timing_complete"] is True
assert report["tool_wait_s"] == 1
@pytest.mark.parametrize(("wall", "expected"), [
(0, 0), (500, 0.5), (123.5, 0.1235),
(None, None), (True, None), ("1", None), (-1, None),
(float("nan"), None), (float("inf"), None), (10**400, None),
])
def test_server_timing_values_do_not_fail_or_reclassify_completed_tool_calls(wall, expected):
events = [use(), result({"timing": {"wall": wall}})]
report = diagnostics.diagnose_session(events, [1, 2], 3)
tool = report["tools"]["mcp__gitnexus__query"]
assert tool["calls"] == 1
assert tool["errors"] == tool["unfinished"] == 0
assert tool["server_wall_s"] == expected
assert tool["server_timed_calls"] == int(expected is not None)
assert report["timing_complete"] is True
assert report["tool_wait_s"] == 1
def test_event_clock_handles_chunk_boundaries_blank_lines_and_unterminated_event(monkeypatch):
ticks = iter([10, 11, 13, 15])
monkeypatch.setattr(diagnostics.time, "monotonic", lambda: next(ticks))
clock = diagnostics.EventClock()
clock.observe(b'\n {"message":"')
clock.observe('é"}\r\n\n'.encode())
clock.observe(b'{"last":true}')
assert clock.finish(2) == [3, 5]
assert clock.finish(3) is None
def test_event_clock_bounds_memory_and_never_claims_complete_overflow(monkeypatch):
monkeypatch.setattr(diagnostics, "MAX_EVENTS", 2)
clock = diagnostics.EventClock()
clock.observe(b'{}\n{}\n{}\n')
assert len(clock.offsets) == 2
assert clock.finish(3) is None
def test_overlapping_tools_are_not_double_counted_as_wall_time():
events = [use(), use("Bash", tool_id="two"), result({"status": "found"}), result("done", tool_id="two")]
report = diagnostics.diagnose_session(events, [1, 2, 5, 7], 10)
assert report["timing_complete"] is True
assert report["tool_wait_s"] == 6
assert report["mcp_wait_s"] == 4
assert report["other_wall_s"] == 4
assert [call["duration_s"] for call in report["calls"]] == [4, 5]
assert report["tools"]["Bash"]["elapsed_s"] == 5
@pytest.mark.parametrize("offsets", [None, [], [1], [3, 2], [-1, 2], [1, 20], [True, 2], [1, float("nan")]])
def test_unavailable_or_invalid_timing_is_unknown_not_zero(offsets):
report = diagnostics.diagnose_session([use(), result("done")], offsets, 10)
assert report["timing_complete"] is False
assert report["tool_wait_s"] is report["mcp_wait_s"] is report["other_wall_s"] is None
assert report["calls"][0]["duration_s"] is None
@pytest.mark.parametrize(("body", "signal"), [
({"error": "Symbol 'privateName' not found"}, "not_found"),
({"status": "ambiguous", "candidates": [{"name": "secret"}]}, "ambiguous"),
({"risk": "UNKNOWN"}, "unknown_risk"),
({"processes": [], "process_symbols": [], "definitions": []}, "empty"),
("Error: result (108,197 characters) exceeds maximum allowed tokens", "output_limit_exceeded"),
])
def test_resolution_problems_and_followup_source_reads_are_counted(body, signal):
events = [use(), result(body), use("Read", tool_id="read"), result("source", tool_id="read")]
report = diagnostics.diagnose_session(events, [1, 2, 3, 4], 5)
tally = report["tools"]["mcp__gitnexus__query"]
assert tally[signal] == 1
assert tally["source_followups"] == 1
assert report["calls"][0]["next_tools"] == ["Read"]
assert "privateName" not in json.dumps(report)
def test_transport_errors_are_distinct_from_successful_resolution():
bad = diagnostics.diagnose_session([use(), result("failure", error=True)], [1, 2], 3)
good = diagnostics.diagnose_session([use(), result({"status": "found"})], [1, 2], 3)
assert bad["tools"]["mcp__gitnexus__query"]["errors"] == 1
assert good["tools"]["mcp__gitnexus__query"]["errors"] == 0
def test_plain_text_mcp_errors_are_not_counted_as_success():
report = diagnostics.diagnose_session([use(), result("Error: database unavailable")], [1, 2], 3)
assert report["tools"]["mcp__gitnexus__query"]["errors"] == 1
def test_non_ascii_payload_byte_counts_handle_escaped_lone_surrogates():
report = diagnostics.diagnose_session([use(args={"query": "é\ud800"}), result("é\ud800")], [1, 2], 3)
assert report["calls"][0]["result_bytes"] == len('"é\\ud800"'.encode())
assert report["timing_complete"] is True
def test_identical_arguments_are_counted_without_publishing_arguments_or_hashes():
events = [use(args={"token": "super-secret"}), result("done"),
use(tool_id="two", args={"token": "super-secret"}), result("done", tool_id="two")]
report = diagnostics.diagnose_session(events, [1, 2, 3, 4], 5)
assert report["calls"][1]["repeat_of"] == 0
assert report["tools"]["mcp__gitnexus__query"]["repeated"] == 1
assert "super-secret" not in json.dumps(report)
assert "signature" not in json.dumps(report)
def test_unmatched_results_reused_ids_and_call_limit_make_timing_incomplete(monkeypatch):
for events in ([use()], [result("orphan")], [use(), result("ok"), use(), result("ok")]):
assert diagnostics.diagnose_session(events, list(range(len(events))), 10)["timing_complete"] is False
monkeypatch.setattr(diagnostics, "MAX_CALLS", 1)
report = diagnostics.diagnose_session([use(), use(tool_id="two")], [1, 2], 5)
assert report["truncated"] is True and len(report["calls"]) == 1
assert report["tool_wait_s"] is None
@pytest.mark.parametrize(("command", "tool"), [
("gitnexus impact parseFile --direction upstream", "cli__gitnexus__impact"),
("/usr/local/bin/gitnexus query 'some concept'", "cli__gitnexus__query"),
("gitnexus query 'x;y'", "cli__gitnexus__query"),
('gitnexus query "x;y"', "cli__gitnexus__query"),
(r"gitnexus query x\;y", "cli__gitnexus__query"),
("gitnexus query '$(literal) | & > < `literal`'", "cli__gitnexus__query"),
('gitnexus query "$(cat secret)"', "Bash"),
('gitnexus query "`cat secret`"', "Bash"),
("gitnexus query x > result.txt", "Bash"),
("gitnexus query x < input.txt", "Bash"),
("gitnexus query x; touch changed", "Bash"),
("gitnexus query x | cat", "Bash"),
("gitnexus query x\nnpm test", "Bash"),
("gitnexus impact x && npm test", "Bash"),
("echo 'gitnexus query x'", "Bash"),
("gitnexus query $(cat secret)", "Bash"),
("gitnexus query 'unterminated", "Bash"),
])
def test_cli_attribution_only_covers_standalone_shim_commands(command, tool):
report = diagnostics.diagnose_session([use("Bash", args={"command": command}), result("ok")], [1, 2], 3)
assert report["calls"][0]["tool"] == tool
assert command not in json.dumps(report)
@pytest.mark.parametrize("mask", range(1, 8))
@pytest.mark.parametrize("has_hit", [False, True])
def test_empty_query_signals_cover_every_selected_response_section(mask, has_hit):
sections = [name for bit, name in enumerate(("processes", "process_symbols", "definitions")) if mask & (1 << bit)]
body = dict.fromkeys(sections, [])
if has_hit:
body[sections[0]] = [{"name": "found"}]
events = [use(), result(body), use("Read", tool_id="read", args={"file_path": "src/a.ts"}),
result("source", tool_id="read")]
report = diagnostics.diagnose_session(events, None, 5)
tool = report["tools"]["mcp__gitnexus__query"]
assert tool["empty"] == int(not has_hit)
assert tool["source_followups"] == int(not has_hit)
@pytest.mark.parametrize("body", [{}, {"definitions": None}, {"timing": {"wall": 1}}])
def test_absent_or_invalid_query_sections_are_not_evidence_of_an_empty_result(body):
report = diagnostics.diagnose_session([use(), result(body)], None, 5)
assert report["tools"]["mcp__gitnexus__query"]["empty"] == 0
def test_usage_snapshots_are_deduplicated_and_no_cost_is_fabricated():
event = use()
event["message"].update(id="message", usage={"input_tokens": 10, "output_tokens": 2, "secret": "hidden"})
later = {"type": "assistant", "message": {"id": "message", "content": [], "usage": {"output_tokens": 5}}}
report = diagnostics.diagnose_session([event, result("ok"), later], [1, 2, 3], 4)
assert report["assistant_usage"] == [{"event_index": 2, "output_tokens": 5}]
assert "cost_usd" not in json.dumps(report)
assert "hidden" not in json.dumps(report)
def test_public_diagnostics_is_idempotent_and_cannot_publish_arbitrary_content():
raw = diagnostics.diagnose_session([use(), result({"status": "found", "content": "secret source"})], [1, 2], 3)
raw.update(prompt="secret prompt", api_key="secret key")
raw["tools"]["private-tool-name"] = {"calls": 5}
raw["calls"][0]["password"] = "secret password"
safe = diagnostics.public_diagnostics([raw])
assert diagnostics.public_diagnostics(safe) == safe
assert safe[0]["largest_results"][0]["index"] == 0
assert safe[0]["slowest_calls"][0]["duration_s"] == 1
assert "secret" not in json.dumps(safe)
assert "private-tool-name" not in json.dumps(safe)
assert "calls" not in safe[0]
def test_missing_status_and_stale_or_truncated_results_are_visible_in_public_diagnostics():
events = [use(), result({"status": "not_found", "risk": "UNKNOWN", "truncated": True,
"staleness": {"status": "behind"}}), use("Read", tool_id="read")]
report = diagnostics.diagnose_session(events, None, 3)
tally = report["tools"]["mcp__gitnexus__query"]
assert tally["stale"] == tally["truncated_results"] == tally["source_followups"] == 1
safe = diagnostics.public_diagnostics([report])
assert safe[0]["problem_calls"][0]["not_found"] is True
assert diagnostics.public_diagnostics(safe) == safe
def test_release_report_exports_diagnostics_without_changing_gate_or_accepting_injected_fields():
measurements = rows()
raw = diagnostics.diagnose_session([use(), result({"error": "private symbol not found"})], [1, 2], 3)
measurements[1]["session_diagnostics"] = [raw]
report = build_report(measurements, metadata(), now=NOW)
assert report["complete"] is True
assert report["paired"]["mean_cost_change_pct"] == 25
validate_report(report, runtime_sha=SHA, task_set_digest=DIGEST, now=NOW)
text = render_markdown(report)
assert "Diagnostics available for 1/3 cells" in text
assert "not proof of a workaround" in text
assert "private symbol" not in json.dumps(report)
report["per_run"][1]["session_diagnostics"][0]["secret"] = "forged"
with pytest.raises(ValueError, match="outside the published report schema"):
validate_report(report, runtime_sha=SHA, task_set_digest=DIGEST, now=NOW)
def test_runner_collects_timings_without_progress_or_changing_session_cost(monkeypatch, tmp_path):
events = [use(), result("ok"), json.loads(VALID_REPORT)]
raw = b"".join((json.dumps(event) + "\n").encode() for event in events)
def run(*args, **kwargs):
for line in raw.splitlines(keepends=True):
kwargs["stdout_observer"](line)
return fake_cli_result(raw.decode())
monkeypatch.setattr(runner_sessions, "run_managed", run)
record = runner_sessions.run_claude("task", tmp_path, claude_bin="claude", timeout=5)
assert record["ok"] is True and record["cost_usd"] == 0.1
assert record["session_diagnostics"][0]["timing_complete"] is True
assert len(record["session_diagnostics"][0]["calls"]) == 1
combined = runner_sessions.sum_sessions([record, record])
assert len(combined["session_diagnostics"]) == 2
assert combined["cost_usd"] == 0.2
assert runner_sessions.sum_sessions([session_record()])["session_diagnostics"] == []
def test_diagnostics_tool_whitelist_matches_the_harness_surface():
assert {f"mcp__gitnexus__{name}" for name in diagnostics.COMMANDS} == set(runner_sessions.GITNEXUS_READ_ONLY_TOOLS)
def test_observational_followups_can_count_one_action_for_multiple_problem_results():
first = use()
second_block = use(tool_id="two")["message"]["content"][0]
first["message"]["content"].append(second_block)
events = [first, result({"risk": "UNKNOWN"}), result({"risk": "UNKNOWN"}, tool_id="two"), use("Read", tool_id="read")]
report = diagnostics.diagnose_session(events, None, 5)
assert report["tools"]["mcp__gitnexus__query"]["source_followups"] == 2
assert Counter(call["tool"] for call in report["calls"])["Read"] == 1

View file

@ -0,0 +1,101 @@
"""Prepared trials must reject dependency drift before launching paid cells."""
import json
from pathlib import Path
import subprocess
import sys
from typing import get_args
import pytest
import yaml
from tests import test_diagnostic_trial
from workflow_bench import diagnostic_experiments, diagnostic_trial
from workflow_bench.process_control import ProcessState
from workflow_bench.verification_diagnostics import public_verification
@pytest.fixture
def bound_trial(tmp_path, monkeypatch):
repo, runtime = tmp_path / "tasks-base", tmp_path / "runtime"
repo.mkdir()
runtime.mkdir()
(repo / ".gitignore").write_text("node_modules/\n")
(repo / "package-lock.json").write_text('{"lockfileVersion": 3}\n')
for args in (
["init", "-q"], ["add", "."],
["-c", "user.name=Test", "-c", "user.email=test@example.test", "commit", "-qm", "fixture"],
):
subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True)
sha = subprocess.check_output(["git", "-C", str(repo), "rev-parse", "HEAD"], text=True).strip()
dependency = repo / "node_modules" / "fixture.js"
dependency.parent.mkdir()
dependency.write_text("module.exports = 'pinned';\n")
corpus = yaml.safe_load((Path(__file__).parents[1] / "workflow_bench/tasks.scenarios.yaml").read_text())
task = next(task for task in corpus["tasks"] if task["id"] == diagnostic_trial.TASK)
task.update(ref=sha, sandbox_dependencies=[{"source": "node_modules", "target": "node_modules"}])
manifest = tmp_path / "tasks.yaml"
manifest.write_text(yaml.safe_dump({"tasks": [task]}))
output = tmp_path / "trial"
monkeypatch.setattr(sys, "argv", [
"prepare", "--tasks", str(manifest), "--repository", str(repo), "--runtime", str(runtime),
"--runtime-sha", test_diagnostic_trial.SHA, "--harness-sha", test_diagnostic_trial.SHA,
"--model", diagnostic_trial.MODEL, "--repeats", "1", "--variants", "no_graph", "mcp_eager",
"--max-budget-usd", "5", "--bind-task-dependencies", "--out", str(output),
])
diagnostic_experiments.main()
monkeypatch.setattr(diagnostic_trial, "_head", lambda path: sha if path == repo else test_diagnostic_trial.SHA)
return output / "plan.json", runtime, repo, dependency
def test_prepared_bindings_reach_both_runner_processes(bound_trial, monkeypatch):
path, runtime, _, _ = bound_trial
plan = json.loads(path.read_text())
calls = []
monkeypatch.setattr(diagnostic_trial.subprocess, "run", test_diagnostic_trial.fake_runner(calls))
assert diagnostic_trial.execute_trial(path, runtime, "claude") == 0
assert len(calls) == 2
for (command, _), cell in zip(calls, plan["cells"], strict=True):
bound = json.loads(command[command.index("--task-bindings-json") + 1])
assert bound == cell["task_bindings"]
assert bound[0]["sandbox_dependency_content_digest"]
@pytest.mark.parametrize("change", ["tracked", "staged", "modified_dependency", "new_dependency", "deleted_dependency", "missing_binding"])
def test_trial_rejects_checkout_or_dependency_drift_before_execution(bound_trial, monkeypatch, change):
path, runtime, repo, dependency = bound_trial
if change in ("tracked", "staged"):
(repo / "package-lock.json").write_text('{"lockfileVersion": 2}\n')
if change == "staged":
subprocess.run(["git", "-C", str(repo), "add", "package-lock.json"], check=True)
elif change == "modified_dependency":
dependency.write_text("module.exports = 'changed';\n")
elif change == "new_dependency":
(dependency.parent / "extra.js").write_text("unexpected dependency\n")
elif change == "deleted_dependency":
dependency.unlink()
else:
plan = json.loads(path.read_text())
del plan["cells"][0]["task_bindings"]
path.write_text(json.dumps(plan))
monkeypatch.setattr(diagnostic_trial.subprocess, "run", lambda *_a, **_kw: pytest.fail("paid executor reached"))
with pytest.raises(ValueError, match="clean pinned|definition drifted|pin task dependency"):
diagnostic_trial.execute_trial(path, runtime, "claude")
assert not (path.parent / "execution.json").exists()
def test_planning_rejects_dirty_dependency_source_before_pinning(bound_trial):
path, _, repo, _ = bound_trial
(repo / "package-lock.json").write_text("dirty\n")
tasks = json.loads((path.parent / "cell-0000.tasks.json").read_text())["tasks"]
with pytest.raises(ValueError, match="clean pinned"):
diagnostic_experiments.trial_task_bindings(tasks)
@pytest.mark.parametrize("state", get_args(ProcessState))
def test_public_verification_preserves_every_terminal_process_state(state):
assert public_verification({"state": state})["state"] == state
def test_public_verification_withholds_unknown_process_state():
assert public_verification({"state": "private unrecognized output"})["state"] == "unknown"

View file

@ -0,0 +1,190 @@
"""Regressions from the first paid two-cell telemetry trial. No model calls."""
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import pytest
from tests import test_diagnostic_trial
from tests.test_workflow_bench_sessions import _probe_cell_provisioning
from workflow_bench import cost_diagnostics, diagnostic_experiments, diagnostic_trial, runner
from workflow_bench.process_control import ManagedProcessResult
from workflow_bench.runner_artifacts import VerificationResult
from workflow_bench.runner_sessions import allowed_agent_tools
from workflow_bench.session_diagnostics import diagnose_session, public_cell_diagnostics
from workflow_bench.verification_diagnostics import public_verification, verification_metrics
trial_plan = test_diagnostic_trial.trial_plan
def test_real_trial_planner_command_keeps_task_dependencies_separate_from_runtime(tmp_path, monkeypatch):
step = next(s for s in test_diagnostic_trial.workflow()["jobs"]["evaluate"]["steps"] if s.get("name") == "Prepare the capped diagnostic trial")
capture = tmp_path / "argv.json"
uv = tmp_path / "uv"
uv.write_text(f"#!{sys.executable}\nimport json,os,sys\nopen(os.environ['CAPTURE'],'w').write(json.dumps(sys.argv[1:]))\n")
uv.chmod(0o755)
subprocess.run(["bash", "-c", step["run"]], check=True, env={
**os.environ, "PATH": f"{tmp_path}:{os.defpath}", "CAPTURE": str(capture),
"RUNNER_TEMP": str(tmp_path), "GITHUB_WORKSPACE": str(tmp_path),
"GITHUB_SHA": "a" * 40, "RUNTIME_SHA": "b" * 40, "MODEL": diagnostic_trial.MODEL, "EFFORT": "xhigh",
})
argv = json.loads(capture.read_text())
assert "--bind-task-dependencies" in argv
monkeypatch.setattr(diagnostic_experiments, "trial_task_bindings", lambda tasks: [{"id": tasks[0]["id"]}])
manifest = Path(argv[argv.index("--tasks") + 1])
manifest.parent.mkdir(parents=True)
source = Path(__file__).parents[1] / "workflow_bench/tasks.scenarios.yaml"
manifest.write_bytes(source.read_bytes())
monkeypatch.setattr(sys, "argv", ["prepare", *argv[argv.index("workflow_bench.diagnostic_experiments") + 1:]])
diagnostic_experiments.main()
plan_path = tmp_path / "release-evaluation/trial/plan.json"
plan = json.loads(plan_path.read_text())
for cell in plan["cells"]:
task = json.loads((plan_path.parent / f"{cell['cell_id']}.tasks.json").read_text())["tasks"][0]
assert task["repo"] == str(tmp_path / "tasks-base")
assert task["repo"] == plan["task_repository"]
assert task["ref"] == "c4ecf398de637cf779ab2238e69958cfd5d601ce"
assert cell["argv"][cell["argv"].index("--gitnexus-root") + 1] == str(tmp_path / "candidate")
def test_old_candidate_dependency_override_fails_before_paid_executor(trial_plan, monkeypatch):
path, runtime = trial_plan
plan = json.loads(path.read_text())
plan["task_repository"] = str(runtime)
for cell in plan["cells"]:
manifest = path.parent / f"{cell['cell_id']}.tasks.json"
doc = json.loads(manifest.read_text())
doc["tasks"][0]["repo"] = str(runtime)
manifest.write_text(json.dumps(doc))
cell["task_manifest_sha256"] = hashlib.sha256(manifest.read_bytes()).hexdigest()
path.write_text(json.dumps(plan))
monkeypatch.setattr(diagnostic_trial.subprocess, "run", lambda *_a, **_k: pytest.fail("paid executor reached"))
with pytest.raises(ValueError, match="pinned task checkout"):
diagnostic_trial.execute_trial(path, runtime, "claude")
assert not (path.parent / "execution.json").exists()
def test_placeholder_cli_usage_does_not_claim_zero_context():
fields = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens", "output_tokens")
work = diagnose_session([{"type": "assistant", "message": {
"id": "placeholder", "usage": dict.fromkeys(fields, 0), "content": [{"type": "text", "text": "done"}],
}}], [1], 2)["workload"]
assert work["usage_snapshots"] == 1 and work["complete_input_snapshots"] == 0
for field in ("first_input_tokens", "max_input_tokens", "sum_input_tokens", "sum_output_tokens"):
assert work[field] is None
def test_provider_request_context_uses_native_cache_semantics_and_withholds_conflicts():
event = {"response_id": "r1", "provider": "litellm-normalized", "native_usage": {
"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 80}, "completion_tokens": 10,
}}
report = cost_diagnostics.provider_summary([event, event])
assert report["request_tokens"]["total_input_tokens"]["max"] == 100
assert report["request_tokens"]["cache_read_input_tokens"]["max"] == 80
assert report["request_tokens"]["total_input_tokens"]["observed"] == 1
conflicted = cost_diagnostics.provider_summary([event, {**event, "status": "failure"}])
assert conflicted["request_tokens"]["total_input_tokens"]["max"] is None
def test_grading_evidence_survives_publication_without_hidden_output():
result = VerificationResult("private command", ManagedProcessResult(
state="exited", returncode=1, stdout_tail="PRIVATE hidden source", stderr_tail="EROFS", duration_s=2.5,
), "PRIVATE hidden source\nAssertionError: private test\nEROFS")
evidence = verification_metrics(result, 2.5)
public = public_cell_diagnostics({"verification_diagnostics": {"hidden": evidence | {"output": result.output}}})
assert "PRIVATE" not in json.dumps(public)
assert public["verification_diagnostics"]["hidden"]["exit_code"] == 1
assert public["verification_diagnostics"]["hidden"]["assertion_failure"] is True
assert public_verification({"state": "PRIVATE", "exit_code": True, "duration_s": float("nan")})["exit_code"] is None
def test_failed_cells_keep_task_provenance_and_budget(monkeypatch, tmp_path):
ctx, *_ = _probe_cell_provisioning(monkeypatch, tmp_path)
ctx.args.max_budget_usd = 5
ctx.args.measurement_revisions = {"runtime_sha": "b" * 40, "harness_sha": "c" * 40}
row = runner.run_cell(ctx, 0, "baseline_nomcp")
assert row["error_kind"] == "infra-error"
assert row["runtime_sha"] == "b" * 40 and row["harness_sha"] == "c" * 40
assert row["max_budget_usd"] == 5 and row["task_base_sha"] == ctx.task_sha
assert row["task_prompt_digest"] and row["model"] == ctx.args.model
def test_mcp_read_tools_are_permitted_and_measured_without_mutating_permissions():
tools = allowed_agent_tools(implementation=True)
assert {"mcp__gitnexus__grep", "mcp__gitnexus__read_file"} <= set(tools)
assert "mcp__gitnexus__rename" not in tools and "mcp__gitnexus__cypher" not in tools
events = [{"type": "assistant", "message": {"content": [
{"type": "tool_use", "id": "grep", "name": "mcp__gitnexus__grep", "input": {}},
]}}]
assert diagnose_session(events, [0], 1)["workload"]["graph_calls"] == 1
def test_cost_summary_displays_setup_grading_and_provider_tokens(tmp_path, monkeypatch):
results = tmp_path / "results.jsonl"
results.write_text(json.dumps({"arm": "baseline", "task": "fixture", "verification_diagnostics": {
"hidden": {"passed": False, "state": "exited", "exit_code": 1, "duration_s": 3, "typecheck_failure": True},
}}) + "\n")
timing = {"complete": True, "workers": 1, "total_s": 600, "before_first_cell_s": 500,
"dependency_setup_s": 100, "foreground_graph_setup_s": 400, "private": "PRIVATE"}
(tmp_path / "sweep-timing.json").write_text(json.dumps(timing))
out, markdown = tmp_path / "public.json", tmp_path / "public.md"
monkeypatch.setattr(sys, "argv", ["diagnostics", "--results", str(results), "--out", str(out), "--markdown-out", str(markdown)])
cost_diagnostics.main()
assert json.loads(out.read_text())["sweep_timing"]["before_first_cell_s"] == 500
assert "PRIVATE" not in out.read_text()
text = markdown.read_text()
assert "before_first_cell_s | 500" in text and "typecheck_failure" in text
def test_release_preflight_requires_grading_controls_in_both_workflow_paths():
step = next(s for s in test_diagnostic_trial.workflow()["jobs"]["evaluate"]["steps"] if s.get("name") == "Materialize both release graphs before paid sessions")
assert step["run"].count("workflow_bench.release_preflight") == step["run"].count("--grading-controls") == 2
def test_public_release_rows_keep_comparable_provenance_and_budget():
from tests.test_release_report import metadata, rows
from workflow_bench.release_report import build_report
source = rows()
for row in source:
row.update(runtime_sha="a" * 40, harness_sha="c" * 40, task_prompt_digest="d" * 64,
max_budget_usd=5.0, mcp_tool_loading="eager" if row["arm"] == "baseline" else "none")
exported = build_report(source, metadata())["per_run"]
report = cost_diagnostics.build_cost_report(exported)
assert all(group["provenance_complete"] for group in report["groups"])
assert all(group["cohort"]["max_budget_usd"] == "5.0" for group in report["groups"])
assert all(group["cohort"]["runtime_digest"] == "" for group in report["groups"])
def test_mcp_session_prompt_names_the_isolated_repository(monkeypatch, tmp_path):
from tests.test_workflow_bench_sessions import VALID_REPORT, bench_args, fake_cli_result, fake_sandbox
from workflow_bench import runner_sessions
prompts = []
def execute(command, **kwargs):
prompts.append(kwargs["stdin_data"].decode())
return fake_cli_result(VALID_REPORT)
monkeypatch.setattr(runner_sessions, "run_managed", execute)
monkeypatch.setattr(runner, "run_verify", lambda *_a, **_k: (True, "ok"))
for arm in ("baseline", "baseline_nomcp"):
runner.run_arm(arm, {"prompt": "Repair the task", "verify": "true"}, tmp_path,
bench_args(), sandbox=fake_sandbox(tmp_path))
assert "benchmark-target" in prompts[0]
assert "benchmark-target" not in prompts[1]
def test_interrupted_timing_does_not_discard_cell_evidence(tmp_path, monkeypatch):
results, out = tmp_path / "results.jsonl", tmp_path / "public.json"
results.write_text('{"arm":"baseline_nomcp","task":"fixture"}\n')
(tmp_path / "sweep-timing.json").write_text('{"complete":')
monkeypatch.setattr(sys, "argv", ["diagnostics", "--results", str(results), "--out", str(out), "--allow-incomplete"])
cost_diagnostics.main()
report = json.loads(out.read_text())
assert report["cells"] == 1 and report["sweep_timing"] is None
assert "Sweep timing file is unreadable." in report["input_problems"]

View file

@ -255,6 +255,9 @@ def test_eval_ci_uses_locked_uv_and_blocking_native_containment_jobs():
"npm ci",
"npm run build",
]
task_build = containment_steps["Build the pinned task toolchain that grades release evaluation"]["run"]
assert "python -m workflow_bench.release_build" in task_build
assert '--repo "${RUNNER_TEMP}/tasks-base"' in task_build
selected_containment_tests = containment_steps["Prove process-tree and sandbox containment"]["run"].split()
assert selected_containment_tests == [
"uv",
@ -281,6 +284,7 @@ def test_eval_ci_uses_locked_uv_and_blocking_native_containment_jobs():
# Paired prepare/session/oracle/report composition uses that same native
# CLI and containment; it must execute here rather than skip everywhere.
"tests/test_release_evaluation_smoke.py",
"tests/test_release_controls.py",
# These also need runtime dependencies absent from the locked pytest job.
"tests/test_evolve.py::test_outer_runner_pid_namespace_kills_setsid_descendant",
"tests/test_oracle_assets.py::test_hidden_vitest_config_executes_sibling_oracle_against_candidate_checkout",
@ -298,7 +302,8 @@ def test_eval_ci_uses_locked_uv_and_blocking_native_containment_jobs():
for path in (repo_root / "eval" / "tests").glob("test_*.py")
if bwrap_canary_marker.search(path.read_text())
)
assert bwrap_canary_files == ["test_cli_shim.py", "test_proposer_sandbox.py", "test_workflow_bench_sessions.py"]
assert bwrap_canary_files == ["test_cli_shim.py", "test_proposer_sandbox.py", "test_release_controls.py",
"test_workflow_bench_sessions.py"]
assert all(f"tests/{name}" in selected_containment_tests for name in bwrap_canary_files)
assert "eval-containment-windows:" in workflow
@ -411,6 +416,62 @@ def test_test_artifact_retries_cannot_hide_missing_evidence(ci_test_jobs):
)
def test_test_artifact_download_retries_require_a_verified_artifact():
repo_root = Path(__file__).resolve().parents[2]
action = yaml.safe_load((repo_root / ".github/actions/download-test-artifact/action.yml").read_text())
assert action["runs"]["using"] == "composite"
assert action["inputs"]["name"]["required"] is True
assert action["inputs"]["path"]["required"] is True
first, wait, second, final_wait, final = action["runs"]["steps"]
assert "if" not in first
assert first["id"] == "first" and second["id"] == "second"
assert first["continue-on-error"] is True
assert second["continue-on-error"] is True
assert not final.get("continue-on-error", False)
# Use raw outcomes: continue-on-error turns a failed attempt's conclusion
# into success. Stop retrying after recovery or cancellation.
for pause, attempt, previous, delay in (
(wait, second, "first", 10),
(final_wait, final, "second", 30),
):
assert pause["if"] == attempt["if"] == "${{ !cancelled() && steps." + previous + ".outcome == 'failure' }}"
assert pause["shell"] == "bash"
assert pause["run"] == f"sleep {delay}"
assert len({step["uses"] for step in (first, second, final)}) == 1
for step in (first, second, final):
assert re.fullmatch(r"actions/download-artifact@[a-f0-9]{40}", step["uses"])
# Every attempt must fetch and verify the same artifact in this run;
# no pattern, alternate run, or relaxed digest check may substitute it.
assert step["with"] == {
"name": "${{ inputs.name }}",
"path": "${{ inputs.path }}",
"digest-mismatch": "error",
}
@pytest.mark.parametrize("job_name", ["tests", "coverage-merge", "shard-plan", "cross-platform", "native-timings"])
def test_frozen_timing_consumers_retry_required_downloads(ci_test_jobs, job_name):
job = ci_test_jobs[job_name]
downloads = [
step for step in job["steps"]
if step.get("with", {}).get("name") == "shard-timing-input"
and step.get("uses") != "./.github/actions/upload-test-artifact"
]
assert len(downloads) == 1
download = downloads[0]
assert download["uses"] == "./.github/actions/download-test-artifact"
assert not download.get("continue-on-error", False)
assert not job.get("continue-on-error", False)
expected_path = ".vitest/timings" if job_name == "shard-plan" else "gitnexus/.vitest/timings"
assert download["with"] == {"name": "shard-timing-input", "path": expected_path}
assert any(
step.get("uses", "").startswith("actions/checkout@")
for step in job["steps"][:job["steps"].index(download)]
)
def test_shard_plan_retry_cannot_replace_frozen_timing_input(ci_test_jobs):
steps = ci_test_jobs["shard-plan"]["steps"]
restore = next(step for step in steps if step.get("id") == "previous-input")

View file

@ -930,6 +930,254 @@ measurements unchanged and compare only runs with matching task contracts.
## MCP response cost
### Session diagnostics
Every new headless session records `session_diagnostics` in `results.jsonl`.
The parent timestamps complete stdout event lines without changing the agent
prompt, tool responses, sandbox, or release criteria. No additional model calls
are made. Existing results without diagnostics remain valid and have unknown
timing, rather than measured zero time.
The release evaluation's GitHub Actions summary includes a **Detailed
measurements** section for each measured runtime or optional arm. It shows cost
and wall-time distributions, setup/cleanup phases, CLI token counters, tool
outcomes and outliers, context/cache signals, repeated verification, and provider
request sizes and usage. Matching `*-diagnostics.json` and `*-diagnostics.md`
files are included in the sanitized evaluation artifacts. PR comparison
summaries retain the candidate's safe diagnostics too.
Each sweep writes its own provider ledger. Only aggregate numeric provider
measurements reach the summary and artifacts; raw request identities and usage
ledgers remain on the runner. Missing or torn inputs are marked as partial,
and provider token totals are withheld when ledger records are unreadable.
Concurrent requests are not assigned to individual cells. Historical runs
without this instrumentation show missing telemetry; regenerating a report
cannot recover tool timings that were never recorded.
Each session contains:
- `calls`: tool name, event indexes, input/result JSON bytes, receipt offsets,
elapsed time, repeated-argument references, result status, staleness,
truncation, and the next assistant action's tool names. MCP errors, missing
symbols, ambiguity, empty query results, UNKNOWN risk, and output-limit
failures are counted separately. Standalone `gitnexus` shim commands are
identified; compound shell commands remain `Bash` because their elapsed time
includes other work.
- `assistant_usage`: the last available usage snapshot for each assistant
message, keyed by event index. Missing fields stay missing. These are
main-session CLI counters, not a complete provider or subagent ledger.
- `workload`: repeated reads, reads of files referenced by graph results,
query/context/read sequences, graph calls before the first observed edit,
verification commands and repeats, edits after verification, compactions,
and input-context growth. Shell commands are classified conservatively;
compound commands remain mixed. Unknown shell actions invalidate the
assumption that the checkout stayed unchanged between verification calls.
These are action counts, including failed edit attempts, not proven waste.
- `tools`: per-tool counts, elapsed sums, response bytes, and source-read
follow-ups after problematic results. One source action can follow multiple
problem results, so follow-ups are associations, not unique extra reads.
- `tool_wait_s`: the union of observed call intervals; parallel tools count
once. `mcp_wait_s` is a subset. `other_wall_s` includes model requests,
startup, transport, retries and scheduling; it is not a measurement of model
reasoning time. Missing results, unmatched IDs, clock gaps or diagnostics
limits make complete wait totals unavailable.
- `server_wall_s`: the sum of tool-reported `timing.wall` milliseconds, converted
to seconds, with a separate coverage count. Missing server timing stays
unknown. This does not equal the host's tool wait or model request time.
The normal release JSON and Markdown include numeric per-tool summaries. The
JSON also keeps each session's five largest results, five slowest calls, and
five problematic calls with the largest results,
with event indexes for locating the detail in local `results.jsonl`. Raw tool
arguments, result text, source, prompts, IDs and argument hashes are excluded
from these diagnostics. Call and assistant-usage details are bounded at 2,048
entries per session; `truncated` exposes that limit. Receipt timing is bounded
at 100,000 event lines. Diagnostics do not make incomplete task evidence valid.
Use the counters to investigate explanations separately: large successful
responses suggest payload overhead; missing/ambiguous targets followed by
source reads suggest a possible workaround; repeated calls suggest a possible
navigation loop. These are signals, not proof of poor resolution or causation.
An empty result is not proof that a symbol is absent. Repeated arguments can be
justified after edits, and smaller output can require more follow-up reads.
Graph precision/recall requires expected-symbol labels or source verification.
The reported session cost is the money measurement; diagnostics never invent
per-tool dollar costs from bytes or elapsed time.
### Cost investigations and controlled experiments
No paid rerun is needed to prepare experiments or summarize existing evidence.
The commands below only read local files and write diagnostic artifacts.
Run them from `eval/` with the project's locked environment.
For the first trial, prepare two cells for one task, with a $5 CLI-reported
budget stop per cell. This checks telemetry delivery before a larger comparison:
```bash
uv run --locked python -m workflow_bench.diagnostic_experiments \
--tasks workflow_bench/tasks.scenarios.yaml \
--repository /path/to/tasks-base --runtime /path/to/built/GitNexus \
--harness-sha FULL_HARNESS_SHA --runtime-sha FULL_RUNTIME_SHA \
--model gpt-5.6-sol --effort xhigh --seed 3762 --repeats 1 \
--variants no_graph mcp_eager --task-ids trivial-status-json-alias \
--max-budget-usd 5 --bind-task-dependencies --out /tmp/gitnexus-diagnostics-trial
```
`--repository` must be the checkout built at the task's pinned commit, including
its dependency trees. `--runtime` is the separately built GitNexus candidate.
`--bind-task-dependencies` requires a clean tracked checkout at the task ref and
records the built dependency contents. The trial executor rejects missing bindings,
dirty tracked files, a different HEAD, or dependency drift. Each runner receives
the saved bindings and checks its captured dependencies again before model calls.
The release workflow also runs offline negative and reference grading controls
with those dependency snapshots and the runner's read-only verifier sandbox.
The unmodified source must fail a behavioral assertion; the reference repair
must pass the hidden oracle and TypeScript. A setup crash is not a negative
control. These checks make no model requests.
The recorded task prompt digest includes the conditional MCP environment
instruction. Comparator reuse rejects older raw-prompt hashes and changes to
that instruction, while keeping the two arms under the same task identity.
The runner forwards `--max-budget-usd` to Claude Code and accepts it only for
single-session baseline arms. The default remains uncapped. Budget-stopped
sessions stay failed and retain reported spend; they cannot establish savings
or successful task completion. The $10 planned total uses CLI accounting, not
a provider invoice cap: an in-flight request or different provider accounting
can exceed it. Preparing the plan does not authorize or execute the trial.
After authorizing those two paid cells, dispatch the trusted main workflow with
`diagnostic_trial=true` and the full reviewed candidate SHA:
```bash
gh workflow run release-evaluation.yml --ref main \
-f diagnostic_trial=true -f candidate_ref=FULL_CANDIDATE_SHA \
-f model=gpt-5.6-sol -f effort=xhigh
```
This mode runs `trivial-status-json-alias` once per arm, serially, with a $5
CLI-reported budget stop and a 900-second session timeout. It rejects additional
CLI/deferred arms, other models/efforts, runner-only mode, and repeat attempts.
The full stable/candidate sweep is skipped. The existing EC2 lock, containment
preflight, stop deadline, evidence reserve, and verified shutdown still apply.
The Actions summary and `diagnostic-trial-*` artifact contain sanitized
measurements for both cells, including failures. Raw provider ledgers remain on
the runner. Trial artifacts cannot approve a release. Missing result rows,
session diagnostics, or provider ledgers fail the trial instead of producing a
green run without measurements. Review the recorded spend before authorizing a
new dispatch after failure; the executor never retries or resumes a used plan.
| Possible cause | Measurement | What still needs an experiment |
| --- | --- | --- |
| Missing, ambiguous or incomplete resolution | Tool outcomes, UNKNOWN risk, lower-bound/stale/truncated responses, next source actions | Verify the intended target exists at the indexed source revision |
| Successful but irrelevant retrieval | Fixed source anchors in `retrieval_cases.json`, anchor recall and first anchor position | Capture the same queries against the sanitized task graph; unjudged results are not false positives |
| Too little detail or repeated investigation | Query/context/read chains, repeated arguments/reads, reads after graph references with and without source | A follow-up read may be necessary; compare matched tasks before calling it waste |
| Large responses and context growth | Result bytes, complete input usage snapshots, first/last/peak context, cache reads/writes, compactions | Compare response-size treatments at equal quality; bytes are not tokens |
| Mandatory graph workflow | Graph calls before edits; eager MCP versus optional-use prompt arm | Prompt ablation measures behavior; it does not remove repository instruction bytes |
| Tool catalog and transport | Eager/deferred/CLI arms; proxy tools/messages bytes and counts | Request sizes are available only when the proxy callback receives those fields |
| Verification and scope growth | Test/typecheck/lint/build counts, repeated checks, edits after verification, diff insertions/deletions/files | Larger diffs and repeated tests are not inherently unnecessary |
| Task fit and quality ceiling | Per-task cost distribution, solved/failed/missing counts, cost per solved task | Add harder independently validated tasks before claiming a quality improvement at 100% success |
| Order, concurrency, warm caches | Seeded rotated order, one worker and separate ledger per cell | Run matched worker-count/cold-start studies separately; fresh processes do not guarantee cold provider/filesystem caches |
| Retries, provider translation and accounting | CLI retry events; deduplicated success/failure callbacks, native normalized usage, reasoning tokens, proxy durations and estimates | Reconcile with provider billing; do not join concurrent requests to cells by timestamp |
| Setup, indexing and infrastructure | Sweep wall time, time before the first cell, dependency preparation, foreground graph setup/waits, cell phases, authored/hidden grading durations | Background graph work can overlap cells; CPU/RSS, workflow provisioning, proxy startup and infrastructure prices need separate evidence; do not fold them into model dollars |
Summarize a runner result file (including failed cells) and optionally its proxy
ledger. Historical release reports with `per_run` rows are also accepted:
```bash
uv run --locked python -m workflow_bench.cost_diagnostics \
--results /path/to/results.jsonl \
--provider-usage /path/to/provider_usage.jsonl \
--out /tmp/cost-diagnostics.json --markdown-out /tmp/cost-diagnostics.md
```
Unknown fields remain null. Distinct model/effort/source/prompt/runtime/loading
cohorts are kept separate. Duplicate exported cell identities are rejected.
Runtime and harness commit SHAs, task prompt digest, and the per-cell CLI budget
survive public export. The legacy evolution `runtime_digest` is optional when
the required commit provenance is present. All-zero CLI usage snapshots are
treated as unavailable; provider-normalized per-request token distributions
give a separate view of context size without guessing from aggregate usage.
When `sweep-timing.json` exists beside the results, the summary includes its
setup measurements. They overlap and must not be added together. Grading
receipts expose duration, exit code, process state, and fixed failure markers;
they never expose hidden-test output. Failure markers are clues, not diagnoses.
Provider response-ID conflicts withhold token totals; missing IDs prevent a
deduplicated total. Failed requests remain in the accounting. Proxy cost is an
estimate, not a provider invoice. The report is descriptive and does not change
release gates. It cannot infer cells omitted from its input: compare every
expected cell in the experiment plan before computing a treatment effect.
`cell_duration_s` includes setup and cleanup; `duration_s` retains its existing
session meaning. Phase spans partition the cell, including a failed active
phase, and unvisited phases stay missing. Shared snapshot construction happens
before cells and is not included. Agent and harness verification share one
phase; it must not be presented as isolated LLM reasoning time. Raw rows retain
UTC start/finish timestamps for order analysis; public release JSON retains
only numeric durations and whitelisted phase names.
Prepare a pilot focused on the overhead floor and the expensive retry task:
```bash
uv run --locked python -m workflow_bench.diagnostic_experiments \
--tasks workflow_bench/tasks.scenarios.yaml \
--repository /path/to/GitNexus --runtime /path/to/built/GitNexus \
--harness-sha FULL_HARNESS_SHA --runtime-sha FULL_RUNTIME_SHA \
--model gpt-5.6-sol --effort xhigh --seed 3762 --repeats 5 \
--task-ids trivial-status-json-alias cross-module-parse-retry \
--out /tmp/gitnexus-cost-experiment
```
This creates `plan.json` and one task manifest per cell. It has **no execution
mode** and never starts an agent. Five arms × two tasks × five repeats prepares
50 cells; this is a design, not spend authorization. Use `--variants` to prepare
a narrower comparison. Position balance requires a repeat count divisible by
the number of arms. The plan records argv arrays, environment paths, task class,
block/position, source pins and prompt/manifest digests. It refuses to overwrite
an existing directory. Pin the final instrumentation commit before execution;
an uncommitted implementation is not reproduced by its parent SHA alone.
After separately authorizing spend, run cells serially in the listed sequence,
from the recorded `cwd`, with the recorded environment and a fresh exclusive
proxy ledger. Each subprocess runs one task/arm/repetition. Its raw `run` is
zero; retain the plan's cell ID and block when assembling comparisons. Do not
mix the optional-prompt arm with ordinary release evidence or reuse its results
for a required-guidance arm. Retain failures and missing cells in the denominator.
Record machine/container, dependency bindings, cache policy, start/finish times
and setup/oracle timing alongside the existing runner output. Reconcile the
ledger sum with CLI-reported session spend, rather than estimating dollars per
tool from duration or payload size.
The four source-verified retrieval cases cover the same pinned scenario source.
Capture each fixed query from the **sanitized** task index used by the runner,
with the same query limits and content setting. Save parsed response objects as:
```json
{
"source_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce",
"runtime_sha": "FULL_RUNTIME_SHA",
"sanitized_graph": true,
"index_complete": true,
"index_staleness": "current",
"query_options": {"limit": 5, "max_symbols": 10, "include_content": false},
"cases": [{"id": "CASE_ID", "query": "EXACT_FIXTURE_QUERY", "response": {}}]
}
```
Verify those provenance fields against the index receipt; the offline scorer
checks declarations, not the database itself. Keep capture metadata and raw
responses private, since they contain repository source. Score without an LLM:
```bash
uv run --locked python -m workflow_bench.retrieval_diagnostics \
--recordings /path/to/retrieval-recordings.json --out /tmp/retrieval-scores.json
```
Missing cases remain missing, errors are not successful empty queries, and stale
or mismatched source recordings are rejected. These are partial anchor labels;
there is no precision score or claim that every other result is irrelevant.
Anchor position follows flattened response order, not a global search rank.
With `include_content: true`, queries share a 16,000-character preview allowance
equally across unique symbols on the returned page. The database returns only
those prefixes, after ranking and pagination. `content_truncated: true` marks

View file

@ -15,6 +15,8 @@ from collections import Counter
from pathlib import Path
from typing import Any
from .session_diagnostics import public_cell_diagnostics, render_diagnostics
ARMS = ("baseline_nomcp", "baseline", "baseline_cli")
EXPOSURE = {"baseline_nomcp": "none", "baseline": "stdio-only", "baseline_cli": "stdio-only"}
METRICS = ("cost_usd", "duration_s", "num_turns", "cache_read_input_tokens", "output_tokens")
@ -141,6 +143,10 @@ def build_report(paired: Path, cli: Path) -> dict[str, Any]:
"pairs": len(matched),
"expected_pairs": len(pairs),
}
report["diagnostic_rows"] = [
{**{field: row.get(field) for field in ("task", "run", "arm")}, **public_cell_diagnostics(row)}
for row, _ in rows
]
return report
@ -180,6 +186,7 @@ def render_markdown(report: dict[str, Any]) -> str:
if report["problems"]:
lines += ["", "**Problems:**", *[f"- {problem}" for problem in report["problems"]]]
lines += ["", "Small pinned task set; descriptive evidence only, not statistical significance.", ""]
lines += render_diagnostics(report.get("diagnostic_rows", []))
return "\n".join(lines)

View file

@ -0,0 +1,335 @@
"""Offline cost evidence report; no API calls, pricing guesses, or release gates."""
from __future__ import annotations
import argparse
import html
import json
import math
import statistics
from dataclasses import asdict
from pathlib import Path
from typing import Any
from .provider_usage import NormalizedUsage, UsageSemanticsError, normalize_usage
from .session_diagnostics import CELL_PHASES, public_diagnostics, render_diagnostics
from .verification_diagnostics import public_verification
NUMERIC = ("cost_usd", "duration_s", "cell_duration_s", "num_turns", "diff_insertions", "diff_deletions", "diff_files",
"input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "output_tokens")
COHORT = ("model", "effort", "task_base_sha", "task_prompt_digest", "runtime_sha", "harness_sha", "runtime_digest", "mcp_tool_loading",
"sandbox_dependency_content_digest", "oracle_digest", "max_budget_usd")
REQUEST_METRICS = ("messages_bytes", "tools_bytes", "system_bytes", "messages_count", "tools_count",
"duration_s", "proxy_estimated_cost_usd")
def _number(value: Any) -> bool:
if type(value) not in (int, float) or value < 0:
return False
try:
return math.isfinite(value)
except OverflowError:
return False
def distribution(values: list[Any]) -> dict[str, Any]:
measured = [v for v in values if _number(v)]
return {
"observed": len(measured), "missing": len(values) - len(measured),
"sum_observed": sum(measured) if measured else None,
"mean": statistics.mean(measured) if measured else None,
"median": statistics.median(measured) if measured else None,
"min": min(measured) if measured else None, "max": max(measured) if measured else None,
"stdev": statistics.stdev(measured) if len(measured) > 1 else None,
}
def provider_summary(events: list[dict[str, Any]]) -> dict[str, Any]:
"""Deduplicate identical callbacks, withhold totals on identity conflicts.
Never join to cells by wall-clock proximity. Concurrent sweeps require a
session ID mapping; serial experiments can use one exclusive ledger/cell.
"""
seen: dict[tuple, dict] = {}
unique, duplicates, conflicts, unidentified = [], 0, set(), 0
for event in events:
response_id = event.get("response_id")
if not isinstance(response_id, str) or not response_id:
unique.append(event)
unidentified += 1
continue
key = (event.get("sweep_id"), event.get("provider"), response_id)
if key in seen:
if seen[key] == event:
duplicates += 1
else:
conflicts.add(key)
continue
seen[key] = event
unique.append(event)
normalized = []
invalid = 0
for event in unique:
try:
normalized.append(asdict(normalize_usage(event.get("provider"), event.get("native_usage"))))
except (UsageSemanticsError, TypeError, ValueError):
invalid += 1
normalized.append({})
token_fields = NormalizedUsage.__dataclass_fields__
token_totals = {
field: sum(row[field] for row in normalized)
if normalized and not conflicts and not unidentified and all(_number(row.get(field)) for row in normalized) else None
for field in token_fields
}
metrics = [e.get("request_metrics") if not conflicts and isinstance(e.get("request_metrics"), dict) else {} for e in unique]
return {
"records": len(events), "observed_requests": len(unique), "duplicate_callbacks": duplicates,
"conflicting_response_ids": len(conflicts), "requests_without_response_id": unidentified,
"invalid_usage_records": invalid,
"successes": sum(e.get("status") == "success" for e in unique),
"failures": sum(e.get("status") == "failure" for e in unique),
"requests_with_session_id": sum(isinstance(e.get("session_id"), str) and bool(e["session_id"]) for e in unique),
"unique_actual_models": len({e["actual_model"] for e in unique if isinstance(e.get("actual_model"), str)}),
"token_totals": token_totals,
"request_tokens": {field: distribution([
row.get(field) if not conflicts and not unidentified else None for row in normalized
]) for field in token_fields},
"request_metrics": {field: distribution([m.get(field) for m in metrics]) for field in REQUEST_METRICS},
"identity_complete": bool(unique) and not conflicts and unidentified == 0,
"attribution": "ledger_only_no_cell_join",
"limitations": [
"No response ID means callbacks cannot be deduplicated; observed totals may overcount.",
"Proxy cost is an estimate, not an invoice. Failures without usage may still incur cost.",
"Callback duration includes proxy/provider work; it is not isolated model thinking time.",
"Request payload bytes are serialized sizes, not token counts.",
],
}
def build_cost_report(rows: list[dict[str, Any]], provider_events: list[dict[str, Any]] | None = None) -> dict[str, Any]:
groups: dict[tuple, list[dict]] = {}
seen_cells = set()
for row in rows:
arm, task = row.get("arm"), row.get("task")
if not isinstance(arm, str) or not isinstance(task, str):
raise ValueError("each cell needs arm and task labels")
identity_fields = {field: row.get(field) if isinstance(row.get(field), str) else "" for field in COHORT}
if arm == "baseline_cli" and "mcp_tool_loading" in row and row["mcp_tool_loading"] is None:
identity_fields["mcp_tool_loading"] = "not_applicable"
if "max_budget_usd" in row:
budget = row["max_budget_usd"]
identity_fields["max_budget_usd"] = "uncapped" if budget is None else str(float(budget)) if _number(budget) and budget > 0 else ""
cohort = tuple(identity_fields[field] for field in COHORT)
identity = (arm, task, row.get("run"), cohort)
if type(row.get("run")) is int:
if identity in seen_cells:
raise ValueError("duplicate cell identity; do not concatenate repeated exports")
seen_cells.add(identity)
groups.setdefault((arm, task, cohort), []).append(row)
summaries = []
tool_sections = []
for (arm, task, cohort), cells in sorted(groups.items()):
costs = [row.get("cost_usd") for row in cells]
solved = sum(row.get("resolved") is True for row in cells)
summaries.append({
"diagnostic_group": len(summaries) + 1,
"arm": arm, "task": task, "cells": len(cells), "solved": solved,
"cohort": dict(zip(COHORT, cohort, strict=True)),
# runtime_digest is a legacy evolution launcher field. Release
# measurements bind the actual runtime and harness commits instead.
"provenance_complete": all(value for field, value in zip(COHORT, cohort, strict=True)
if field != "runtime_digest"),
"verification": {
phase: [public_verification(row.get("verification_diagnostics", {}).get(phase))
for row in cells if isinstance(row.get("verification_diagnostics"), dict)]
for phase in ("authored", "hidden")
},
"quality_missing": sum(type(row.get("resolved")) is not bool for row in cells),
"failed_cells": sum(row.get("ok") is False for row in cells),
"diagnostic_cells": sum(bool(public_diagnostics(row.get("session_diagnostics"))) for row in cells),
"metrics": {field: distribution([row.get(field) for row in cells]) for field in NUMERIC},
"cell_phases": {phase: distribution([
row["cell_phase_seconds"].get(phase) if isinstance(row.get("cell_phase_seconds"), dict) else None
for row in cells
]) for phase in CELL_PHASES},
"cost_per_solved": sum(costs) / solved if solved and all(_number(c) for c in costs) else None,
})
details = render_diagnostics(cells)
if details:
tool_sections += [f"### Diagnostic group {len(summaries)}", *details]
return {
"schema": "gitnexus.cost-diagnostics/v1", "cells": len(rows), "groups": summaries,
"provider": provider_summary(provider_events) if provider_events is not None else None,
"tool_markdown": "\n".join(tool_sections),
"limitations": [
"Descriptive observations only; no causal attribution, significance claim or release verdict.",
"Groups include failed cells and cost per solved includes their reported spend.",
"Missing cells require comparison to the experiment manifest; this report cannot infer omitted rows.",
"Legacy rows without diagnostics cannot reveal retrieval relevance, instruction overhead or tool timings.",
"Main-loop session telemetry excludes subagent internals; session cost may include subagents.",
"Task labels, model/effort, runtime/source pins and prompt digests must match the intended treatment.",
],
}
def render_markdown(report: dict[str, Any]) -> str:
"""Render measured distributions and coverage, without guessing missing spend."""
def number(value: Any) -> str:
return f"{value:,.4f}".rstrip("0").rstrip(".") if _number(value) else "n/a"
def label(value: Any) -> str:
return html.escape(str(value)).replace("\n", " ").translate(
{ord(char): f"&#{ord(char)};" for char in "\\`*_[]()|!"}
)
def table(metrics: dict[str, dict]) -> list[str]:
lines = ["| Measurement | Observed / missing | Observed total | Mean | Median | Min | Max | Std. dev. |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
for name, values in metrics.items():
lines.append(f"| {name} | {values['observed']} / {values['missing']} | " + " | ".join(
number(values[field]) for field in ("sum_observed", "mean", "median", "min", "max", "stdev")
) + " |")
return lines
lines = ["## Time, cost, and tool measurements", "",
f"Observed cells: {report['cells']}. Missing measurements are shown as `n/a`, not zero.",
"Cell token counters come from the CLI. Provider tokens below use provider-specific accounting; do not add the two together.", ""]
if report.get("input_problems"):
lines += ["**Partial input: totals cover only the readable records.**", "",
*[f"- {problem}" for problem in report["input_problems"]], ""]
for group in report["groups"]:
lines += [f"### Group {group['diagnostic_group']}: <code>{label(group['arm'])}</code> / <code>{label(group['task'])}</code>", "",
(f"Solved: {group['solved']}/{group['cells']}; failed sessions: {group['failed_cells']}; "
f"quality unknown: {group['quality_missing']}; diagnostics: {group['diagnostic_cells']}/{group['cells']}."),
f"Cost per solved cell (USD, including failed-session spend): {number(group['cost_per_solved'])}.",
f"Complete cohort provenance: {'yes' if group['provenance_complete'] else 'no'}.", "",
"<details><summary>Measurements and cohort</summary>", "",
"| Cohort field | Value |", "| --- | --- |",
*[f"| {field} | <code>{label(value) if value else 'unknown'}</code> |"
for field, value in group["cohort"].items()], "",
*table(group["metrics"]), "", "Cell phases (seconds; agent_and_verification includes grading):", "",
*table(group["cell_phases"]), "", "</details>", ""]
lines += ["Grading checks (failure markers are observations, not proven root causes):", "",
"| Check | Passed | Exit code | State | Seconds | Observed failure markers |",
"| --- | --- | --- | --- | ---: | --- |"]
for phase, checks in group["verification"].items():
for check in checks:
markers = ", ".join(field for field in ("assertion_failure", "startup_failure", "permission_failure", "typecheck_failure")
if check.get(field) is True)
lines.append(f"| {phase} | {check.get('passed', 'unknown')} | {check.get('exit_code')} | "
f"{check.get('state', 'unknown')} | {number(check.get('duration_s'))} | {markers or 'none observed'} |")
lines += [""]
timing = report.get("sweep_timing")
if timing:
lines += ["### Runner setup and wall time", "",
f"Complete sweep: {'yes' if timing['complete'] else 'no'}; workers: {number(timing['workers'])}.",
("Times overlap: before-first-cell includes dependency and foreground graph setup. "
"Graph time includes foreground waits; background indexing can overlap cells. "
"Total excludes CLI/proxy startup and report rendering."), "",
"| Measurement | Seconds |", "| --- | ---: |",
*[f"| {field} | {number(timing[field])} |" for field in (
"total_s", "before_first_cell_s", "dependency_setup_s", "foreground_graph_setup_s",
)], ""]
provider = report["provider"]
lines += ["### Provider requests", ""]
if provider is None:
lines += ["Provider ledger unavailable. Request counts, provider tokens, and proxy cost are unknown.", ""]
else:
lines += [(f"Observed requests: {provider['observed_requests']}; successes: {provider['successes']}; "
f"failures: {provider['failures']}; duplicate callbacks removed: {provider['duplicate_callbacks']}."),
(f"Conflicting response identities: {provider['conflicting_response_ids']}; "
f"requests without identity: {provider['requests_without_response_id']}; "
f"invalid usage records: {provider['invalid_usage_records']}."),
"Attribution is to this sweep's ledger. Requests are not assigned to individual concurrent cells.", "",
"| Provider token field | Observed total |", "| --- | ---: |",
*[f"| {field} | {number(value)} |" for field, value in provider["token_totals"].items()], "",
*table(provider["request_metrics"]), "",
"Provider-reported tokens per request (includes cached input; independent of CLI context snapshots):", "",
*table(provider["request_tokens"]), "",
*provider["limitations"], ""]
lines += [report["tool_markdown"], "", *report["limitations"], ""]
return "\n".join(lines)
def _read_rows(path: Path, *, allow_incomplete: bool, name: str, problems: list[str]) -> list[dict]:
if not path.is_file() and allow_incomplete:
problems.append(f"{name} file is missing.")
return []
raw = path.read_text()
try:
doc = json.loads(raw)
except json.JSONDecodeError:
rows = []
for line in raw.splitlines():
if not line.strip():
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError:
if not allow_incomplete:
raise
rows.append(None)
else:
rows = doc.get("per_run", [doc]) if isinstance(doc, dict) else doc
if not isinstance(rows, list):
raise ValueError(f"expected {name} rows")
invalid = sum(not isinstance(row, dict) for row in rows)
if invalid:
if not allow_incomplete:
raise ValueError(f"expected {name} rows")
problems.append(f"{name}: {invalid} unreadable record(s).")
return [row for row in rows if isinstance(row, dict)]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--results", type=Path, required=True, help="runner results.jsonl or release report JSON")
parser.add_argument("--provider-usage", type=Path)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--markdown-out", type=Path)
parser.add_argument("--allow-incomplete", action="store_true", help="report missing/torn inputs after an interrupted run")
args = parser.parse_args()
problems: list[str] = []
rows = _read_rows(args.results, allow_incomplete=args.allow_incomplete, name="Results", problems=problems)
provider = None
provider_problems: list[str] = []
if args.provider_usage:
provider = _read_rows(args.provider_usage, allow_incomplete=args.allow_incomplete,
name="Provider ledger", problems=provider_problems)
if provider_problems and not provider:
provider = None
problems += provider_problems
report = build_cost_report(rows, provider)
timing_path = args.results.parent / "sweep-timing.json"
report["sweep_timing"] = None
if timing_path.is_file():
try:
timing = json.loads(timing_path.read_text())
except (OSError, ValueError):
if not args.allow_incomplete:
raise
timing = None
problems.append("Sweep timing file is unreadable.")
if isinstance(timing, dict):
report["sweep_timing"] = {
"complete": timing.get("complete") is True,
**{field: timing.get(field) if _number(timing.get(field)) else None for field in (
"total_s", "before_first_cell_s", "dependency_setup_s", "foreground_graph_setup_s", "workers",
)},
}
if report["provider"] is not None:
report["provider"]["ledger_complete"] = not provider_problems
if provider_problems:
report["provider"]["identity_complete"] = False
report["provider"]["token_totals"] = dict.fromkeys(report["provider"]["token_totals"])
report["input_problems"] = problems
with args.out.open("x") as handle:
json.dump(report, handle, indent=2, allow_nan=False)
handle.write("\n")
if args.markdown_out:
with args.markdown_out.open("x") as handle:
handle.write(render_markdown(report))
if __name__ == "__main__":
main()

View file

@ -0,0 +1,187 @@
"""Prepare counterbalanced experiment inputs. This module NEVER runs an agent."""
from __future__ import annotations
import argparse
import copy
import hashlib
import json
import random
import re
from pathlib import Path
from typing import Any
import yaml
from .process_control import run_checked
from .runner_tasks import normalized_model_identifier, resolve_task_bindings, select_tasks
from .session_budget import session_budget
VARIANTS = {
"no_graph": ("baseline_nomcp", "eager"),
"mcp_eager": ("baseline", "eager"),
"mcp_deferred": ("baseline", "deferred"),
"cli": ("baseline_cli", "eager"),
"mcp_optional_prompt": ("baseline", "eager"),
}
OPTIONAL_PROMPT = """
Experiment-specific instruction: GitNexus is available, but using it is optional
for this task. This overrides repository instructions that require graph-first
navigation, impact analysis before every edit, or detect_changes before commit.
Use the tools that help solve the task. Keep all other repository requirements,
tests, and safety constraints. Do not install or bootstrap additional tools.
"""
def prepare_plan(
document: dict[str, Any], *, repository: str, runtime: str, runtime_sha: str,
harness_sha: str, model: str, effort: str, repeats: int, seed: int,
variants: list[str], task_ids: list[str] | None = None, max_budget_usd: float | None = None,
) -> dict[str, Any]:
"""A seeded Latin-square order within each task; no execution side effects.
One runner process per cell makes its provider ledger attributable without
guessing session IDs. Run indices in runner output are always zero; the
experiment's block/sequence fields carry repetition identity instead.
"""
max_budget_usd = session_budget(max_budget_usd)
if not variants or len(set(variants)) != len(variants) or any(v not in VARIANTS for v in variants):
raise ValueError("variants must be unique known experiment variants")
if not 1 <= repeats <= 100:
raise ValueError("repeats must be between 1 and 100")
if not all(re.fullmatch(r"[0-9a-f]{40}", sha) for sha in (runtime_sha, harness_sha)):
raise ValueError("runtime and harness require full commit SHAs")
if not isinstance(model, str):
raise ValueError("use a named model, not auto/latest")
model = normalized_model_identifier(model)
if effort not in ("low", "medium", "high", "xhigh", "max"):
raise ValueError("invalid effort")
if not isinstance(document, dict):
raise ValueError("task document must contain tasks")
tasks = document.get("tasks")
if not isinstance(tasks, list) or not tasks or not all(isinstance(t, dict) for t in tasks):
raise ValueError("task document must contain tasks")
tasks, _ = select_tasks([{**task, "repo": repository} for task in tasks], include_expensive=True)
ids = [t["id"] for t in tasks]
if task_ids is not None and (not task_ids or not set(task_ids) <= set(ids)):
raise ValueError("unknown or empty task selection")
tasks = [t for t in tasks if task_ids is None or t["id"] in task_ids]
if any(not re.fullmatch(r"[0-9a-f]{40}", str(t.get("ref", ""))) or not isinstance(t.get("prompt"), str) for t in tasks):
raise ValueError("each task needs a pinned ref and prompt")
rng = random.Random(seed)
order = variants.copy()
rng.shuffle(order)
rng.shuffle(tasks)
cells = []
for block in range(repeats):
for task_index, task in enumerate(tasks):
rotation = (block + task_index) % len(order)
for position, variant in enumerate(order[rotation:] + order[:rotation]):
arm, loading = VARIANTS[variant]
prepared = copy.deepcopy(task)
prepared["repo"] = repository
if variant == "mcp_optional_prompt":
prepared["prompt"] += OPTIONAL_PROMPT
sequence = len(cells)
cell_id = f"cell-{sequence:04d}"
cells.append({
"cell_id": cell_id, "sequence": sequence, "block": block,
"position": position, "task": task["id"], "task_class": task.get("class"),
"variant": variant, "arm": arm, "tool_loading": loading,
"base_prompt_sha256": hashlib.sha256(task["prompt"].encode()).hexdigest(),
"prompt_sha256": hashlib.sha256(prepared["prompt"].encode()).hexdigest(),
"task_document": {"tasks": [prepared]},
"argv": ["uv", "run", "--locked", "python", "-m", "workflow_bench.runner",
"--tasks", f"{cell_id}.tasks.json", "--runs", "1", "--workers", "1",
"--arms", arm, "--mcp-tool-loading", loading,
"--gitnexus-root", runtime, "--model", model, "--effort", effort,
"--include-expensive", *(["--max-budget-usd", str(max_budget_usd)] if max_budget_usd is not None else []),
"--out", f"results/{cell_id}"],
"env": {"GITNEXUS_BENCH_SWEEP_ID": cell_id,
"GITNEXUS_BENCH_PROVIDER_USAGE": f"results/{cell_id}.provider.jsonl"},
})
return {
"schema": "gitnexus.diagnostic-experiments/v1", "execution": "not_started",
"harness_sha": harness_sha, "runtime_sha": runtime_sha, "seed": seed,
"task_repository": repository,
"model": model, "effort": effort, "repeats": repeats, "workers": 1,
"max_budget_usd_per_cell": max_budget_usd,
"planned_cli_budget_usd": len(cells) * max_budget_usd if max_budget_usd is not None else None,
"position_balanced": repeats % len(variants) == 0,
"release_gate_eligible": False, "cells": cells,
"limitations": [
"No model-spend authorization; this plan does not execute commands.",
"Verify checkout SHAs, instrumentation diff, dependencies and oracle bindings before execution.",
"Run serially in listed sequence, with an exclusive proxy/ledger per cell; never reuse ledgers.",
"Optional-prompt treatment changes tool-use policy, not repository guidance bytes or tool schemas.",
"This plan controls concurrency at one; a separate matched worker-count study is needed for contention.",
"Fresh processes do not guarantee cold filesystem/provider caches; record machine and cache policy.",
"CLI budget stops use reported model cost; in-flight requests and provider billing can exceed them.",
],
}
def trial_task_bindings(tasks: list[dict[str, Any]], expected: list[dict[str, Any]] | None = None) -> list[dict[str, Any]]:
"""Pin built dependencies only after checking their tracked source checkout."""
for task in tasks:
repo = str(Path(task["repo"]).resolve())
head = run_checked(["git", "-C", repo, "rev-parse", "HEAD"], timeout=60).stdout_tail.strip()
dirty = run_checked(
["git", "-C", repo, "status", "--porcelain", "--untracked-files=no"], timeout=60,
).stdout_tail.strip()
if head != task["ref"] or dirty:
raise ValueError("trial dependencies require a clean pinned task checkout")
return resolve_task_bindings(tasks, expected)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--tasks", type=Path, required=True)
parser.add_argument("--repository", type=Path, required=True)
parser.add_argument("--runtime", type=Path, required=True)
parser.add_argument("--runtime-sha", required=True)
parser.add_argument("--harness-sha", required=True)
parser.add_argument("--model", required=True)
parser.add_argument("--effort", default="xhigh")
parser.add_argument("--repeats", type=int, default=5)
parser.add_argument("--seed", type=int, default=3762)
parser.add_argument("--max-budget-usd", type=float, help="CLI-reported budget stop per cell, not an invoice cap")
parser.add_argument("--variants", nargs="+", choices=VARIANTS, default=list(VARIANTS))
parser.add_argument("--task-ids", nargs="+")
parser.add_argument("--bind-task-dependencies", action="store_true",
help="pin built task dependencies for execution; requires a clean pinned checkout")
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
plan = prepare_plan(
yaml.safe_load(args.tasks.read_text()), repository=str(args.repository.resolve()),
runtime=str(args.runtime.resolve()), runtime_sha=args.runtime_sha, harness_sha=args.harness_sha,
model=args.model, effort=args.effort, repeats=args.repeats, seed=args.seed,
variants=args.variants, task_ids=args.task_ids, max_budget_usd=args.max_budget_usd,
)
if args.bind_task_dependencies:
bindings = {}
for cell in plan["cells"]:
tasks = cell["task_document"]["tasks"]
key = json.dumps(tasks, sort_keys=True)
if key not in bindings:
bindings[key] = trial_task_bindings(tasks)
cell["task_bindings"] = bindings[key]
# Refuse overwrite: a previous plan/ledger may already be evidence.
args.out.mkdir(parents=True, exist_ok=False)
root = args.out.resolve()
(root / "results").mkdir()
for cell in plan["cells"]:
manifest = root / f"{cell['cell_id']}.tasks.json"
manifest.write_text(json.dumps(cell.pop("task_document"), indent=2) + "\n")
cell["task_manifest_sha256"] = hashlib.sha256(manifest.read_bytes()).hexdigest()
cell["argv"][cell["argv"].index("--tasks") + 1] = str(manifest)
cell["argv"][-1] = str(root / "results" / cell["cell_id"])
cell["env"]["GITNEXUS_BENCH_PROVIDER_USAGE"] = str(root / "results" / f"{cell['cell_id']}.provider.jsonl")
cell["cwd"] = str(Path(__file__).resolve().parents[1])
(root / "plan.json").write_text(json.dumps(plan, indent=2) + "\n")
print(f"Prepared {len(plan['cells'])} cells at {root}; no agents started.")
if __name__ == "__main__":
main()

View file

@ -0,0 +1,133 @@
"""Execute the fixed, explicitly authorized two-cell telemetry trial once."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
from typing import Any
from . import diagnostic_experiments
MODEL = "gpt-5.6-sol"
TASK = "trivial-status-json-alias"
BUDGET_USD = 5.0
HARNESS_ROOT = Path(__file__).resolve().parents[2]
def _head(repository: Path) -> str:
return subprocess.run(
["git", "-C", str(repository), "rev-parse", "HEAD"],
check=True, capture_output=True, text=True, timeout=60,
).stdout.strip()
def _validated_cells(plan_path: Path, runtime: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
plan = json.loads(plan_path.read_text())
policy = {
"schema": "gitnexus.diagnostic-experiments/v1", "execution": "not_started",
"model": MODEL, "effort": "xhigh", "repeats": 1, "workers": 1, "seed": 3762,
"max_budget_usd_per_cell": BUDGET_USD, "planned_cli_budget_usd": 10.0,
"release_gate_eligible": False,
}
if not isinstance(plan, dict) or any(plan.get(key) != value for key, value in policy.items()):
raise ValueError("trial plan does not match the authorized two-cell policy")
if plan.get("harness_sha") != _head(HARNESS_ROOT) or plan.get("runtime_sha") != _head(runtime):
raise ValueError("trial checkout does not match its pinned revision")
cells = plan.get("cells")
if not isinstance(cells, list) or len(cells) != 2 or not all(isinstance(c, dict) for c in cells):
raise ValueError("trial must contain exactly two cells")
if {cell.get("variant") for cell in cells} != {"no_graph", "mcp_eager"}:
raise ValueError("trial requires one no-graph and one eager-MCP cell")
root = plan_path.parent
for index, cell in enumerate(cells):
cell_id = f"cell-{index:04d}"
arm = "baseline" if cell["variant"] == "mcp_eager" else "baseline_nomcp"
manifest = root / f"{cell_id}.tasks.json"
output = root / "results" / cell_id
ledger = root / "results" / f"{cell_id}.provider.jsonl"
expected = [
"uv", "run", "--locked", "python", "-m", "workflow_bench.runner",
"--tasks", str(manifest), "--runs", "1", "--workers", "1", "--arms", arm,
"--mcp-tool-loading", "eager", "--gitnexus-root", str(runtime),
"--model", MODEL, "--effort", "xhigh", "--include-expensive",
"--max-budget-usd", str(BUDGET_USD), "--out", str(output),
]
if (cell.get("cell_id") != cell_id or cell.get("task") != TASK or cell.get("argv") != expected
or cell.get("cwd") != str(HARNESS_ROOT / "eval") or cell.get("env") != {
"GITNEXUS_BENCH_SWEEP_ID": cell_id, "GITNEXUS_BENCH_PROVIDER_USAGE": str(ledger),
}):
raise ValueError("trial cell has unexpected arguments or paths")
if manifest.is_symlink() or hashlib.sha256(manifest.read_bytes()).hexdigest() != cell.get("task_manifest_sha256"):
raise ValueError("trial task manifest changed after preparation")
document = json.loads(manifest.read_text())
tasks = document.get("tasks") if isinstance(document, dict) else None
if not isinstance(tasks, list) or len(tasks) != 1 or not isinstance(tasks[0], dict) or tasks[0].get("id") != TASK:
raise ValueError("trial manifest must contain only the authorized task")
task_repo = tasks[0].get("repo")
if (not isinstance(task_repo, str) or task_repo != plan.get("task_repository")
or _head(Path(task_repo)) != tasks[0].get("ref")):
raise ValueError("task source and dependencies must come from the pinned task checkout")
if output.exists() or output.is_symlink() or ledger.exists() or ledger.is_symlink():
raise ValueError("trial results already exist; refusing to repeat paid work")
bindings = cell.get("task_bindings")
if not isinstance(bindings, list) or len(bindings) != 1 or not isinstance(bindings[0], dict):
raise ValueError("trial plan must pin task dependency bindings before execution")
diagnostic_experiments.trial_task_bindings(tasks, bindings)
return plan, cells
def execute_trial(plan_path: Path, runtime: Path, claude_bin: str) -> int:
plan_path, runtime = plan_path.resolve(), runtime.resolve()
plan, cells = _validated_cells(plan_path, runtime)
receipt = plan_path.parent / "execution.json"
state: dict[str, Any] = {
"schema": "gitnexus.diagnostic-trial-execution/v1", "status": "running",
"harness_sha": plan["harness_sha"], "runtime_sha": plan["runtime_sha"],
"planned_cli_budget_usd": 10.0, "budget_is_invoice_cap": False, "cells": [],
}
# A crash can happen after a paid request but before its result is saved.
# Never resume/retry this plan automatically, even when no result exists.
with receipt.open("x") as stream:
stream.write(json.dumps(state, indent=2) + "\n")
failed = False
for cell in cells:
command = [sys.executable, *cell["argv"][4:], "--timeout", "900", "--claude-bin", claude_bin,
"--task-bindings-json", json.dumps(cell["task_bindings"])]
item: dict[str, Any] = {"cell_id": cell["cell_id"], "variant": cell["variant"], "status": "started"}
state["cells"].append(item)
receipt.write_text(json.dumps(state, indent=2) + "\n")
completed = subprocess.run(command, cwd=cell["cwd"], env={**os.environ, **cell["env"]}, check=False)
output = plan_path.parent / "results" / cell["cell_id"] / "results.jsonl"
ledger = Path(cell["env"]["GITNEXUS_BENCH_PROVIDER_USAGE"])
try:
rows = [json.loads(line) for line in output.read_text().splitlines() if line.strip()]
measured = (len(rows) == 1 and isinstance(rows[0], dict) and rows[0].get("task") == TASK
and rows[0].get("arm") == cell["arm"] and bool(rows[0].get("session_diagnostics"))
and ledger.is_file() and ledger.stat().st_size > 0)
except (OSError, ValueError):
measured = False
item.update(status="finished", exit_code=completed.returncode, measurements_present=measured)
failed |= completed.returncode != 0 or not measured
receipt.write_text(json.dumps(state, indent=2) + "\n")
state["status"] = "failed" if failed else "completed"
receipt.write_text(json.dumps(state, indent=2) + "\n")
return int(failed)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--execute", action="store_true", required=True, help="Authorize these two paid cells")
parser.add_argument("--plan", type=Path, required=True)
parser.add_argument("--runtime", type=Path, required=True)
parser.add_argument("--claude-bin", required=True)
args = parser.parse_args()
raise SystemExit(execute_trial(args.plan, args.runtime, args.claude_bin))
if __name__ == "__main__":
main()

View file

@ -19,6 +19,7 @@ accounting because the log write failed would be the worse outcome.
from __future__ import annotations
import json
import math
import os
import threading
from typing import Any
@ -67,6 +68,36 @@ def _plain(value: Any) -> Any:
return None
def _request_metrics(kwargs: dict, start_time: Any, end_time: Any) -> dict:
"""Numeric envelope evidence only; never log request content or headers."""
optional = kwargs.get("optional_params")
optional = optional if isinstance(optional, dict) else {}
metrics = {}
for field in ("messages", "tools", "system"):
value = kwargs.get(field, optional.get(field))
metrics[f"{field}_bytes"] = None
if isinstance(value, (str, dict, list)):
try:
metrics[f"{field}_bytes"] = len(json.dumps(
value, ensure_ascii=False, separators=(",", ":"),
).encode(errors="backslashreplace"))
except (TypeError, ValueError, RecursionError):
# Keep size unknown when serialization fails; usage accounting
# must still be recorded even without this optional metric.
pass
if field != "system":
metrics[f"{field}_count"] = len(value) if isinstance(value, list) else None
try:
elapsed = end_time - start_time
elapsed = elapsed.total_seconds() if hasattr(elapsed, "total_seconds") else elapsed
except (TypeError, ValueError):
elapsed = None
metrics["duration_s"] = elapsed if type(elapsed) in (int, float) and math.isfinite(elapsed) and elapsed >= 0 else None
cost = kwargs.get("response_cost")
metrics["proxy_estimated_cost_usd"] = cost if type(cost) in (int, float) and math.isfinite(cost) and cost >= 0 else None
return metrics
class ProviderUsageLogger(CustomLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: # noqa: ANN001
self._append("success", kwargs, response_obj, start_time, end_time)
@ -123,6 +154,7 @@ class ProviderUsageLogger(CustomLogger):
"completed_at": str(end_time),
# Verbatim. Not flattened, not renamed, not summed.
"native_usage": _plain(getattr(response_obj, "usage", None)),
"request_metrics": _request_metrics(kwargs, start_time, end_time),
}
line = json.dumps(event, default=str) + "\n"
with _LOCK, open(path, "a", encoding="utf-8") as handle:

View file

@ -10,6 +10,7 @@ from pathlib import Path
from typing import Any
from .release_report import META_FIELDS, build_report
from .session_diagnostics import render_diagnostics
COUNTERS = ("num_turns", "input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "output_tokens")
BINDINGS = ("task_prompt_digest", "sandbox_dependency_content_digest")
@ -105,6 +106,10 @@ def render_markdown(report: dict[str, Any]) -> str:
"Hooks disabled; prompts, MCP backend and outputs unchanged. Descriptive evidence only; no release approval.",
"",
]
for mode in ("none", "eager", "deferred"):
details = render_diagnostics([row for row in report.get("per_run", []) if row.get("mcp_tool_loading") == mode])
if details:
lines += [f"### Diagnostics: {mode}", *details, ""]
if not report["complete"]:
return "\n".join([*lines, "**Incomplete or mismatched evidence; no savings estimate.**", "",
*[f"- {p}" for p in report["problems"]], ""])

View file

@ -18,6 +18,8 @@ from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from .session_diagnostics import public_cell_diagnostics, render_diagnostics
SHA = re.compile(r"[0-9a-f]{40}")
REPO = re.compile(r"[\w.-]+/[\w.-]+")
COST_LABEL = "run-cost-evaluation"
@ -349,6 +351,7 @@ def compare_candidate(
and _number(row.get("duration_s"))
)
safe = {field: row[field] for field in ROW_FIELDS if field in row}
safe.update(public_cell_diagnostics(row))
try:
json.dumps(safe, allow_nan=False)
except (ValueError, TypeError, OverflowError):
@ -414,7 +417,8 @@ def render_markdown(report: dict[str, Any]) -> str:
"",
]
if not report["complete"]:
return "\n".join([*lines, "**Incomplete evidence; no savings conclusion.**", "", *report["problems"], ""])
return "\n".join([*lines, "**Incomplete evidence; no savings conclusion.**", "", *report["problems"],
*render_diagnostics(report.get("per_run", [])), ""])
lines += [
"| MCP sessions | Solved | Total model cost | Mean model cost |",
"| --- | --- | --- | --- |",
@ -446,6 +450,7 @@ def render_markdown(report: dict[str, Any]) -> str:
"",
"Only the PR MCP cells are newly measured; no-MCP and stable sessions are not rerun.",
"Historical comparisons are descriptive and do not establish statistical significance or isolate this PR from other runtime changes.",
*render_diagnostics(report.get("per_run", [])),
"",
]
)

View file

@ -0,0 +1,61 @@
"""Offline grading controls using the runner's dependency and sandbox boundaries."""
from __future__ import annotations
import argparse
from pathlib import Path
from typing import Any
from .oracle_assets import ORACLE_ROOT, capture_task_oracle, sanitize_clone_for_hidden_oracles
from .proposer_sandbox import ReadOnlyMount, SandboxError, prepare_sandbox
from .runner import _run_hidden_oracle, _verification_outcome
from .runner_artifacts import make_worktree, run_verify
from .task_assets import stage_task_assets
REFERENCES = {
"trivial-status-json-alias": "trivial-status-json-alias",
"inv-feature-c-include-provenance": "inv-feature-c-include-provenance-flag",
"inv-feature-list-repos-filter": "inv-feature-list-repos-filter",
"cross-module-parse-retry": "cross-module-parse-retry",
}
def check_task_controls(task: dict[str, Any], *, repo: Path, sha: str, parent: Path,
assets: Any, claude_bin: Path, bwrap_bin: Path) -> None:
"""Reject broken setup before spending: base fails assertions, reference passes.
Reference patches and oracle bytes never enter a paid cell. The probe uses
the same immutable dependency snapshot and read-only verifier as the runner.
"""
reference = REFERENCES.get(task["id"])
if reference is None:
raise ValueError(f"no offline grading control for release task {task['id']}")
clone = make_worktree(repo, sha, parent)
sanitize_clone_for_hidden_oracles(clone)
mounts = stage_task_assets(task, repo=repo, clone=clone, snapshot=assets)
snapshot = capture_task_oracle(task)
args = argparse.Namespace(timeout=180)
with prepare_sandbox(clone=clone, claude_bin=claude_bin, bwrap_bin=bwrap_bin,
read_only_mounts=mounts, preflight=False, gitnexus_available=False) as sandbox:
negative: dict[str, Any] = {}
passed, _ = _run_hidden_oracle(snapshot, clone, args, sandbox, diagnostics=negative)
if (passed or negative.get("exit_code") != 1 or not negative.get("assertion_failure")
or negative.get("startup_failure") or negative.get("permission_failure")):
raise SandboxError("negative grading control did not fail on a behavioral assertion")
patch_mount = ReadOnlyMount(ORACLE_ROOT / "reference" / f"{reference}.patch", "/tmp/reference.patch")
applied = run_verify(
"git apply /tmp/reference.patch", clone, 60,
command_prefix=sandbox.command_prefix_for(unshare_network=True, extra_read_only_mounts=(patch_mount,)),
env=sandbox.environment(), require_pid_namespace=True,
)
if not _verification_outcome(applied)[0]:
raise SandboxError("reference patch could not be applied in grading control")
typed = run_verify(
"cd gitnexus && ./node_modules/.bin/tsc --noEmit", clone, 180,
command_prefix=sandbox.command_prefix_for(read_only_workspace=True, unshare_network=True),
env=sandbox.environment(), require_pid_namespace=True,
)
if not _verification_outcome(typed)[0]:
raise SandboxError("reference grading control failed typecheck")
if not _run_hidden_oracle(snapshot, clone, args, sandbox)[0]:
raise SandboxError("reference grading control failed the hidden oracle")

View file

@ -13,11 +13,13 @@ from .process_control import ManagedProcessError, cancellation_scope
from .proposer_sandbox import SandboxError, preflight_bubblewrap, require_claude_sandbox_helpers
from .runner_tasks import resolve_task_bindings, select_tasks
from .runtime_mounts import trusted_gitnexus_runtime_mounts
from .release_controls import check_task_controls
from .sanitized_graph import prepare_sanitized_graph, validate_no_prebuilt_graph_assets
from .task_assets import TaskAssetCache, stage_task_assets
def preflight_release(tasks: list[dict[str, Any]], *, gitnexus_root: Path, claude_bin: Path) -> None:
def preflight_release(tasks: list[dict[str, Any]], *, gitnexus_root: Path, claude_bin: Path,
grading_controls: bool = False) -> None:
"""Exercise the real snapshot/copy path on the same temp filesystem as the runner.
No provider or session code is involved. Graphs are rebuilt by the paid
@ -49,6 +51,10 @@ def preflight_release(tasks: list[dict[str, Any]], *, gitnexus_root: Path, claud
clone = Path(probe)
graph.materialize(clone, sanitized_head=graph.sanitized_head)
stage_task_assets(task, repo=repo, clone=clone, snapshot=assets)
if grading_controls:
check_task_controls(task, repo=repo, sha=sha, parent=Path(directory), assets=assets,
claude_bin=claude_bin, bwrap_bin=bwrap_bin)
print(f"Preflight {task['id']}: negative, reference, and typecheck controls passed", flush=True)
print(f"Preflight {task['id']}: materialization passed", flush=True)
@ -57,6 +63,7 @@ def main() -> None:
parser.add_argument("--tasks", required=True, type=Path)
parser.add_argument("--gitnexus-root", required=True, type=Path)
parser.add_argument("--claude-bin", required=True, type=Path)
parser.add_argument("--grading-controls", action="store_true", help="require offline controls for the release corpus")
args = parser.parse_args()
try:
document = yaml.safe_load(args.tasks.read_text())
@ -64,7 +71,8 @@ def main() -> None:
raise ValueError("task file must contain a tasks list")
tasks, _ = select_tasks(document["tasks"], include_expensive=True)
with cancellation_scope(handle_signals=True):
preflight_release(tasks, gitnexus_root=args.gitnexus_root, claude_bin=args.claude_bin)
preflight_release(tasks, gitnexus_root=args.gitnexus_root, claude_bin=args.claude_bin,
grading_controls=args.grading_controls)
except (ManagedProcessError, OSError, SandboxError, RuntimeError, ValueError, yaml.YAMLError) as exc:
parser.error(str(exc))

View file

@ -17,6 +17,7 @@ import yaml
from .oracle_assets import capture_task_oracles
from .runner_tasks import normalized_model_identifier, select_tasks
from .session_diagnostics import public_cell_diagnostics, render_diagnostics
from .task_assets import DEPENDENCY_CONTENT_BINDING_FIELD
SCHEMA = "gitnexus.release-evaluation/v1"
@ -30,6 +31,12 @@ ROW_FIELDS = (
"model",
"effort",
"task_base_sha",
"task_prompt_digest",
"runtime_sha",
"harness_sha",
"runtime_digest",
"max_budget_usd",
"mcp_tool_loading",
"oracle_digest",
"sandbox_backend",
"runtime_exposure",
@ -110,7 +117,9 @@ def build_report(rows: list[Any], metadata: dict[str, Any], *, now: datetime | N
problems.append(f"row {index}: invalid measurement")
continue
# Invalid rows still need a public, JSON-serializable failure receipt.
safe_rows.append({key: _json_safe(row[key]) for key in ROW_FIELDS if key in row})
safe_row = {key: _json_safe(row[key]) for key in ROW_FIELDS if key in row}
safe_row.update(public_cell_diagnostics(row))
safe_rows.append(safe_row)
task, run, arm = row.get("task"), row.get("run"), row.get("arm")
if not isinstance(task, str) or type(run) is not int or not isinstance(arm, str):
problems.append(f"row {index}: invalid cell identity")
@ -205,6 +214,7 @@ def render_markdown(report: dict[str, Any]) -> str:
"This small pinned task set is release evidence; it does not establish general gains or statistical significance.",
"",
]
lines += render_diagnostics(report.get("per_run", []))
return "\n".join(lines)

View file

@ -0,0 +1,37 @@
{
"source_sha": "c4ecf398de637cf779ab2238e69958cfd5d601ce",
"judgment": "partial source-verified anchors; no precision labels",
"query_options": { "limit": 5, "max_symbols": 10, "include_content": false },
"cases": [
{
"id": "trivial-status-json-alias",
"query": "status CLI command JSON output option registration",
"anchors": [{ "file": "gitnexus/src/cli/index.ts" }]
},
{
"id": "inv-feature-c-include-provenance",
"query": "C include capture quoted and system header provenance",
"anchors": [
{
"file": "gitnexus/src/core/ingestion/languages/c/import-decomposer.ts",
"name": "splitCInclude"
}
]
},
{
"id": "inv-feature-list-repos-filter",
"query": "list_repos MCP repository listings filter by name",
"anchors": [{ "file": "gitnexus/src/mcp/local/local-backend.ts", "name": "listRepos" }]
},
{
"id": "cross-module-parse-retry",
"query": "parse worker transient failure error exit retry quarantine dispatch",
"anchors": [
{
"file": "gitnexus/src/core/ingestion/workers/worker-pool.ts",
"name": "handleWorkerDeath"
}
]
}
]
}

View file

@ -0,0 +1,92 @@
"""Score fixed source anchors against recorded retrievals, without an LLM judge.
These are partial relevance judgments: unjudged results are not false positives.
The score is anchor coverage, not end-to-end task success or search precision.
"""
from __future__ import annotations
import argparse
import json
import re
from pathlib import Path
from typing import Any
def score_retrieval(cases: dict[str, Any], recordings: dict[str, Any]) -> dict[str, Any]:
if recordings.get("source_sha") != cases.get("source_sha") or not cases.get("source_sha"):
raise ValueError("retrieval recordings must match the fixture source SHA")
if recordings.get("index_complete") is not True or recordings.get("index_staleness") != "current":
raise ValueError("retrieval recordings require a complete current index receipt")
if recordings.get("sanitized_graph") is not True or not re.fullmatch(r"[0-9a-f]{40}", str(recordings.get("runtime_sha", ""))):
raise ValueError("record the sanitized graph and runtime identity")
if recordings.get("query_options") != cases.get("query_options") or not cases.get("query_options"):
raise ValueError("record the fixed query options from the fixture")
rows = recordings.get("cases", [])
if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows):
raise ValueError("recorded cases must be a list")
by_id = {row.get("id"): row for row in rows}
if len(by_id) != len(rows):
raise ValueError("duplicate retrieval case ID")
expected_ids = {case["id"] for case in cases["cases"]}
if not set(by_id) <= expected_ids:
raise ValueError("unknown retrieval case ID")
scores = []
for case in cases["cases"]:
row = by_id.get(case["id"])
if row is None:
scores.append({"id": case["id"], "status": "missing", "anchor_recall": None})
continue
if row.get("query") != case["query"]:
raise ValueError(f"query differs for case {case['id']}")
body = row.get("response")
if not isinstance(body, dict):
raise ValueError("record parsed query response objects")
symbols = [*body.get("process_symbols", []), *body.get("definitions", [])]
anchors = []
for symbol in symbols:
if isinstance(symbol, dict) and isinstance(symbol.get("filePath"), str):
anchor = (symbol["filePath"], symbol.get("name"))
if anchor not in anchors:
anchors.append(anchor)
matched = []
for expected in case["anchors"]:
positions = [i + 1 for i, (path, name) in enumerate(anchors)
if path == expected["file"] and ("name" not in expected or name == expected["name"])]
matched.append(min(positions) if positions else None)
failed = bool(body.get("error"))
scores.append({
"id": case["id"], "status": "error" if failed else "observed",
"expected_anchors": len(matched), "found_anchors": sum(p is not None for p in matched),
"anchor_recall": sum(p is not None for p in matched) / len(matched) if matched and not failed else None,
"first_anchor_position": min((p for p in matched if p is not None), default=None),
"returned_anchors": len(anchors), "response_bytes": len(json.dumps(body, ensure_ascii=True).encode()),
"truncated": body.get("truncated") is True or body.get("partial") is True,
})
return {
"schema": "gitnexus.retrieval-diagnostics/v1", "source_sha": cases["source_sha"],
"runtime_sha": recordings["runtime_sha"], "cases": scores,
"complete": all(row["status"] == "observed" and not row.get("truncated") for row in scores),
"limitations": [
"Source anchors are partial judgments; no precision or false-positive rate is inferred.",
"Position is flattened process_symbols then definitions order, not a calibrated relevance rank.",
"Coverage is conditional on the recorded query limits and index layers; preserve the capture receipt.",
"The scorer checks declared provenance; the capture must verify it against the sanitized index.",
],
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--cases", type=Path, default=Path(__file__).with_name("retrieval_cases.json"))
parser.add_argument("--recordings", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
report = score_retrieval(json.loads(args.cases.read_text()), json.loads(args.recordings.read_text()))
with args.out.open("x") as handle:
json.dump(report, handle, indent=2, allow_nan=False)
handle.write("\n")
if __name__ == "__main__":
main()

View file

@ -56,6 +56,8 @@ from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from .session_budget import session_budget
import yaml
from .mcp_transport import connector_config, isolated_mcp
@ -170,6 +172,7 @@ from .runner_sessions import (
sandbox_mcp_config,
sum_sessions,
)
from .verification_diagnostics import verification_metrics
from .runner_tasks import (
normalized_model_identifier,
resolve_task_bindings,
@ -331,6 +334,8 @@ def _run_hidden_oracle(
worktree: Path,
args: argparse.Namespace,
sandbox: SandboxSession,
*,
diagnostics: dict[str, Any] | None = None,
) -> tuple[bool, str]:
"""Stage a captured oracle after the model exits, execute it, then erase it."""
@ -353,22 +358,24 @@ def _run_hidden_oracle(
# credited patch is captured.
oracle_mount = f"{SANDBOX_WORKSPACE}/{mount_name}"
oracle_env[ORACLE_ENV_VAR] = str(stage_root) if host_unsafe else oracle_mount
passed, _output = _verification_outcome(
run_verify(
snapshot.command,
sandbox.clone,
args.timeout,
command_prefix=sandbox.command_prefix_for(
read_only_workspace=True,
unshare_network=True,
extra_read_only_mounts=()
if host_unsafe
else (ReadOnlyMount(source=stage_root, target=oracle_mount),),
),
env=oracle_env,
require_pid_namespace=getattr(sandbox, "require_pid_namespace", True),
)
started = time.monotonic()
result = run_verify(
snapshot.command,
sandbox.clone,
args.timeout,
command_prefix=sandbox.command_prefix_for(
read_only_workspace=True,
unshare_network=True,
extra_read_only_mounts=()
if host_unsafe
else (ReadOnlyMount(source=stage_root, target=oracle_mount),),
),
env=oracle_env,
require_pid_namespace=getattr(sandbox, "require_pid_namespace", True),
)
if diagnostics is not None:
diagnostics.update(verification_metrics(result, time.monotonic() - started))
passed, _output = _verification_outcome(result)
# Candidate code executes in this process. Never persist its stdout
# or stderr: it can read the mounted hidden test bytes and print them.
return passed, "hidden oracle passed" if passed else "hidden oracle failed"
@ -449,6 +456,13 @@ def _unchanged(value: Any) -> Any:
return value
MCP_REPOSITORY_PROMPT = (
"\n\nBenchmark environment: the isolated GitNexus repository is registered as "
"benchmark-target. Use that repo name, or omit repo to use the default. "
"Repository aliases in checked-in guidance refer to the original checkout."
)
def run_arm(
arm: str,
task: dict[str, Any],
@ -465,6 +479,8 @@ def run_arm(
mcp_config_json: str | None = None,
) -> dict[str, Any]:
sessions: list[dict[str, Any]] = []
if arm != "baseline_nomcp":
task = {**task, "prompt": task["prompt"] + MCP_REPOSITORY_PROMPT}
environment_builder = getattr(sandbox, "environment", build_sandbox_environment)
host_text = getattr(sandbox, "host_text", _unchanged)
host_path = getattr(sandbox, "host_path", lambda value: str(value))
@ -492,6 +508,7 @@ def run_arm(
"progress_label": progress_label,
"claude_bin": sandbox.claude_bin,
"timeout": args.timeout,
"max_budget_usd": getattr(args, "max_budget_usd", None),
"model": args.model,
"effort": args.effort,
"env": env,
@ -706,20 +723,23 @@ def run_arm(
verify_mounts = (
ReadOnlyMount(source=review_artifact.parent, target=SANDBOX_REVIEW_OUTPUT),
)
authored_tests_passed, authored_test_output = _verification_outcome(
run_verify(
task["verify"],
worktree,
args.timeout,
command_prefix=sandbox.command_prefix_for(
read_only_workspace=True,
unshare_network=True,
extra_read_only_mounts=verify_mounts,
),
env=verify_env,
require_pid_namespace=getattr(sandbox, "require_pid_namespace", True),
)
verification_started = time.monotonic()
authored_result = run_verify(
task["verify"],
worktree,
args.timeout,
command_prefix=sandbox.command_prefix_for(
read_only_workspace=True,
unshare_network=True,
extra_read_only_mounts=verify_mounts,
),
env=verify_env,
require_pid_namespace=getattr(sandbox, "require_pid_namespace", True),
)
record["verification_diagnostics"] = {
"authored": verification_metrics(authored_result, time.monotonic() - verification_started),
}
authored_tests_passed, authored_test_output = _verification_outcome(authored_result)
review_score: dict[str, Any] | None = None
if arm in ("review", "ce_review"):
try:
@ -748,12 +768,15 @@ def run_arm(
)
oracle_output = "hidden review labels matched" if oracle_passed else "hidden review labels not fully matched"
else:
hidden_diagnostics: dict[str, Any] = {}
oracle_passed, oracle_output = _run_hidden_oracle(
oracle_snapshot,
worktree,
args,
sandbox,
diagnostics=hidden_diagnostics,
)
record["verification_diagnostics"]["hidden"] = hidden_diagnostics
record["authored_tests_passed"] = authored_tests_passed
record["authored_test_output"] = authored_test_output
record["oracle_passed"] = oracle_passed
@ -833,7 +856,10 @@ def task_prompt_digest(task: Mapping[str, Any]) -> str:
stop rows matching, or match rows that should not.
"""
return hashlib.sha256(str(task["prompt"]).encode()).hexdigest()
# Bind the conditional harness instruction as well as the task text. Both
# arms retain the same experiment identity; old raw-prompt hashes expire.
payload = {"task": str(task["prompt"]), "mcp_environment": MCP_REPOSITORY_PROMPT}
return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
# A sustained upstream outage shows up as a run of session/infra/cleanup
@ -1212,6 +1238,18 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
expected kinds is deliberately left to propagate — a harness bug must not be
recorded as an ordinary infra-error and averaged into the evidence.
"""
cell_started_at = datetime.now(UTC).isoformat()
cell_started = time.monotonic()
phase_started = cell_started
active_phase = "validation"
phase_seconds: dict[str, float] = {}
def enter_phase(name: str) -> None:
nonlocal active_phase, phase_started
now = time.monotonic()
phase_seconds[active_phase] = phase_seconds.get(active_phase, 0.0) + now - phase_started
active_phase, phase_started = name, now
args = ctx.args
task = ctx.task
worktree: Path | None = None
@ -1235,6 +1273,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
raise RuntimeError("sanitized graph snapshot is unavailable")
if ctx.asset_snapshot is None:
raise RuntimeError("task asset snapshot is unavailable")
enter_phase("clone")
if ctx.clone_template is not None:
if not ctx.sanitized_head:
raise RuntimeError("clone template is missing its sanitized HEAD")
@ -1243,9 +1282,11 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
else:
worktree = make_worktree(ctx.repo, ctx.task_sha, ctx.trees_dir)
sanitized_head = sanitize_clone_for_hidden_oracles(worktree)
enter_phase("graph_materialize")
if gitnexus_available:
assert ctx.graph_snapshot is not None
ctx.graph_snapshot.materialize(worktree, sanitized_head=sanitized_head)
enter_phase("dependencies")
dependency_mounts = stage_task_assets(
task,
repo=ctx.repo,
@ -1258,12 +1299,14 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
else []
)
ce_mounts = ce_plugin_mounts_for_arm(execution_arm, ctx.ce_plugin_snapshot)
enter_phase("mcp_startup")
if isolated_baseline:
endpoint = resources.enter_context(isolated_mcp(
clone=worktree, mounts=[*dependency_mounts, *gitnexus_mounts], bwrap_bin=ctx.bwrap_bin,
))
gitnexus_mounts = [endpoint]
runtime_exposure = "stdio-only"
enter_phase("sandbox_setup")
with prepare_sandbox(
clone=worktree,
claude_bin=args.claude_bin,
@ -1346,6 +1389,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
before_work_digest = (
implementation_diff_digest(sandbox, orig_sha) if execution_arm in IMPLEMENTATION_ARMS else ""
)
enter_phase("agent_and_verification")
record = run_arm(
execution_arm,
task,
@ -1360,6 +1404,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
oracle_snapshot=ctx.oracle_snapshot,
mcp_config_json=connector_config() if isolated_baseline else None,
)
enter_phase("artifacts")
if execution_arm in ("review", "ce_review"):
review_source = review_output_path(sandbox, REVIEW_OUTPUT)
if review_source.is_file() and not review_source.is_symlink():
@ -1433,6 +1478,7 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
detail = redact_text(str(exc), credential_secrets(args))
print(f"[{task['id']}][{arm}][run {run_idx}] infra-error: {detail}")
finally:
enter_phase("cleanup")
try:
resources.close()
except (SandboxError, OSError, RuntimeError) as exc:
@ -1453,13 +1499,24 @@ def run_cell(ctx: TaskCellContext, run_idx: int, arm: str) -> dict[str, Any]:
record["error_detail"] = (
f"primary={primary_kind}: {primary_detail}; cleanup: {type(cleanup_error).__name__}: {cleanup_error}"
)[:2000]
cell_finished = time.monotonic()
phase_seconds["cleanup"] = cell_finished - phase_started
record.update(
{
"cell_started_at": cell_started_at,
"cell_finished_at": datetime.now(UTC).isoformat(),
"cell_duration_s": cell_finished - cell_started,
"cell_phase_seconds": phase_seconds,
"task": task["id"],
"class": task.get("class", ""),
"run": run_idx,
"sandbox_backend": ctx.sandbox_backend,
"runtime_exposure": runtime_exposure,
"max_budget_usd": getattr(args, "max_budget_usd", None),
"model": args.model,
"effort": args.effort,
"task_base_sha": ctx.task_sha,
"task_prompt_digest": task_prompt_digest(task),
**getattr(args, "measurement_revisions", {}),
"mcp_tool_loading": (
getattr(args, "mcp_tool_loading", "eager") if execution_arm == "baseline"
@ -2105,6 +2162,10 @@ def build_parser() -> argparse.ArgumentParser:
"subagent spend — token metrics count only the main loop",
)
parser.add_argument("--promotion-min-runs", type=int, default=3)
parser.add_argument(
"--max-budget-usd", type=float,
help="opt-in CLI-reported cost stop per baseline session; provider billing may exceed it",
)
parser.add_argument("--promotion-min-improvement", type=float, default=5.0)
parser.add_argument("--promotion-max-task-regression", type=float, default=20.0)
parser.add_argument("--task-bindings-json", default=None, help=argparse.SUPPRESS)
@ -2126,6 +2187,9 @@ def main() -> None:
parser = build_parser()
args = parser.parse_args()
try:
args.max_budget_usd = session_budget(args.max_budget_usd)
if args.max_budget_usd is not None and any(arm not in ("baseline", "baseline_nomcp", "baseline_cli") for arm in args.arms):
raise ValueError("--max-budget-usd is limited to single-session baseline arms")
args.model = normalized_model_identifier(args.model)
args.proposer_model = (
normalized_model_identifier(args.proposer_model, flag="--proposer-model")
@ -2530,6 +2594,11 @@ def _run_sweep(
total_cells = len(tasks) * args.runs * len(args.arms)
started_cells = 0
sweep_started = time.monotonic()
timing: dict[str, Any] = {"complete": False, "workers": args.workers, "total_s": None,
"before_first_cell_s": None, "dependency_setup_s": None,
"foreground_graph_setup_s": 0.0}
timing_path = out_dir / "sweep-timing.json"
timing_path.write_text(json.dumps(timing) + "\n")
with (
tempfile.TemporaryDirectory(prefix="wfbench-trees-") as trees,
TaskAssetCache(Path(trees) / ".task-assets") as task_asset_cache,
@ -2567,6 +2636,8 @@ def _run_sweep(
except (OSError, SandboxError, ValueError) as exc:
asset_snapshot_errors[str(_task["id"])] = exc
timing["dependency_setup_s"] = time.monotonic() - sweep_started
timing_path.write_text(json.dumps(timing) + "\n")
reuse_source = args.reuse_results.expanduser().resolve() if args.reuse_results is not None else None
reusable_rows: dict[tuple[str, str, int], dict[str, Any]] = {}
if reuse_source is not None:
@ -2659,12 +2730,15 @@ def _run_sweep(
asset_snapshot = asset_snapshots.get(str(task["id"]))
asset_snapshot_error: BaseException | None = asset_snapshot_errors.get(str(task["id"]))
graph_key = (str(repo), task_sha)
graph_started = time.monotonic()
if graph_prefetch is not None and graph_prefetch.key == graph_key:
_join_graph_prefetch()
if paid_cells:
ensure_task_graph(
task=task, repo=repo, task_sha=task_sha, graph_key=graph_key, env=graph_env
)
timing["foreground_graph_setup_s"] += time.monotonic() - graph_started
timing_path.write_text(json.dumps(timing) + "\n")
graph_snapshot = graph_env.graph_snapshots.get(graph_key)
graph_snapshot_error = graph_env.graph_snapshot_errors.get(graph_key)
clone_template, template_head = graph_env.clone_templates.get(graph_key, (None, None))
@ -2692,6 +2766,9 @@ def _run_sweep(
def announce(run_idx: int, arm: str) -> None:
nonlocal started_cells
if timing["before_first_cell_s"] is None:
timing["before_first_cell_s"] = time.monotonic() - sweep_started
timing_path.write_text(json.dumps(timing) + "\n")
started_cells += 1
print(
f"[{task['id']}][{arm}][run {run_idx}] starting "
@ -2756,6 +2833,8 @@ def _run_sweep(
finally:
_join_graph_prefetch()
timing.update(complete=not outage_tripped and not cancel_event.is_set(), total_s=time.monotonic() - sweep_started)
timing_path.write_text(json.dumps(timing) + "\n")
selection_report = [
"## Run provenance",
"",

View file

@ -18,6 +18,8 @@ from pathlib import Path, PurePosixPath
from typing import Any
from .process_control import run_managed
from .session_budget import session_budget
from .session_diagnostics import EventClock, diagnose_session
from .proposer_sandbox import (
SANDBOX_GITNEXUS,
SANDBOX_GITNEXUS_REGISTRY,
@ -354,6 +356,8 @@ GITNEXUS_READ_ONLY_TOOLS = (
"mcp__gitnexus__list_repos",
"mcp__gitnexus__query",
"mcp__gitnexus__context",
"mcp__gitnexus__grep",
"mcp__gitnexus__read_file",
"mcp__gitnexus__check",
"mcp__gitnexus__impact",
"mcp__gitnexus__explain",
@ -728,6 +732,7 @@ def run_claude(
transcript_secrets: tuple[str, ...] = (),
plugin_dirs: Sequence[str] = (),
progress_label: str | None = None,
max_budget_usd: float | None = None,
) -> dict[str, Any]:
"""Run one headless session and return its usage record."""
@ -735,6 +740,7 @@ def run_claude(
# every file under the sandbox HOME is writable by agent tools and cannot
# serve as trusted evidence.
del transcript_projects, transcript_cwd, transcript_wait_seconds
max_budget_usd = session_budget(max_budget_usd)
cmd = [
claude_bin,
@ -766,10 +772,13 @@ def run_claude(
cmd += ["--model", model]
if effort:
cmd += ["--effort", effort]
if max_budget_usd is not None:
cmd += ["--max-budget-usd", str(max_budget_usd)]
for tool in disallowed_tools or []:
cmd += ["--disallowedTools", tool]
managed_cmd = [*(command_prefix or []), *cmd]
started = time.monotonic()
event_clock = EventClock()
with contextlib.ExitStack() as progress_stack:
progress = (
progress_stack.enter_context(
@ -781,6 +790,11 @@ def run_claude(
if progress_label
else None
)
def observe(chunk: bytes) -> None:
event_clock.observe(chunk)
if progress is not None:
progress.observe(chunk)
proc = run_managed(
managed_cmd,
cwd=None if command_prefix else cwd,
@ -789,17 +803,19 @@ def run_claude(
require_pid_namespace=require_pid_namespace,
stdin_data=prompt.encode(),
capture_stdout_bytes=MAX_TRANSCRIPT_BYTES,
stdout_observer=progress.observe if progress is not None else None,
stdout_observer=observe,
)
wall_s = time.monotonic() - started
event_stream_error: str | None = None
events: list[dict[str, Any]] = []
event_offsets: list[float] | None = None
try:
if proc.stdout_capture is None:
raise ValueError("parent process did not capture Claude stdout")
if proc.stdout_capture_overflow:
raise ValueError(f"parent-captured event stream exceeds {MAX_TRANSCRIPT_BYTES} bytes")
events = _parse_parent_event_stream(proc.stdout_capture)
event_offsets = event_clock.finish(len(events))
data, result_index = _session_result(events, proc.duration_s)
# A session that used a background task drains its bookkeeping after
# the final result event (`background_tasks_changed`, `task_updated`,
@ -811,9 +827,12 @@ def run_claude(
# a property of what the evidence readers below can see, rather than an
# assumption that a `system` event never carries a tool_use block.
events = events[: result_index + 1]
if event_offsets is not None:
event_offsets = event_offsets[: result_index + 1]
except (UnicodeError, ValueError) as exc:
event_stream_error = str(exc)
data = {}
event_offsets = None
usage = data.get("usage") or {}
subtype = data.get("subtype")
well_formed = all(field in usage for field in USAGE_FIELDS)
@ -860,6 +879,8 @@ def run_claude(
"transcript_missing": False,
"tool_use_counts": tool_use_counts,
"gitnexus_tool_uses": sum(count for name, count in tool_use_counts.items() if name.startswith("mcp__gitnexus")),
"session_diagnostics": [diagnose_session(events, event_offsets, wall_s)],
"max_budget_usd": max_budget_usd,
**{field: usage.get(field, 0) for field in USAGE_FIELDS},
}
needs_evidence = expected_skill is not None or transcript_output_dir is not None
@ -955,4 +976,7 @@ def sum_sessions(sessions: list[dict[str, Any]]) -> dict[str, Any]:
for name, count in session.get("tool_use_counts", {}).items():
tool_use_counts[name] = tool_use_counts.get(name, 0) + int(count)
total["tool_use_counts"] = tool_use_counts
total["session_diagnostics"] = [
diagnostic for session in sessions for diagnostic in session.get("session_diagnostics", [])
]
return total

View file

@ -0,0 +1,20 @@
"""Validate opt-in CLI-reported session budgets, not provider billing caps."""
from __future__ import annotations
import math
def session_budget(value: float | None) -> float | None:
if value is None:
return None
message = "--max-budget-usd must be finite and greater than zero"
if type(value) not in (int, float):
raise ValueError(message)
try:
budget = float(value)
except OverflowError as exc:
raise ValueError(message) from exc
if not math.isfinite(budget) or budget <= 0:
raise ValueError(message)
return budget

View file

@ -0,0 +1,478 @@
"""Content-free diagnostics from parent-captured agent events.
Tool intervals are observed at the stdout boundary, not inside the server.
The next tool action is a correlation, never proof of a workaround or its cost.
"""
from __future__ import annotations
import hashlib
import json
import math
import shlex
import time
from collections import Counter
from pathlib import PurePosixPath
from typing import Any
from .workload_diagnostics import COUNT_FIELDS, TOKEN_FIELDS, public_workload, workload_metrics
from .verification_diagnostics import public_verification
SCHEMA = "gitnexus.session-diagnostics/v1"
MAX_EVENTS = 100_000
MAX_CALLS = 2048
CELL_PHASES = ("validation", "clone", "graph_materialize", "dependencies", "mcp_startup",
"sandbox_setup", "agent_and_verification", "artifacts", "cleanup")
COMMANDS = frozenset({
"list_repos", "query", "context", "grep", "read_file", "check", "impact", "explain", "pdg_query",
"route_map", "tool_map", "shape_check", "api_impact", "trace", "detect_changes",
})
TOOLS = frozenset({
"Read", "Grep", "Glob", "Edit", "Write", "Bash", "Skill", "ToolSearch", "other",
*(f"mcp__gitnexus__{name}" for name in COMMANDS),
*(f"cli__gitnexus__{name}" for name in COMMANDS),
})
USAGE = ("input_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "output_tokens")
COUNTERS = (
"calls", "errors", "ambiguous", "not_found", "empty", "unknown_risk", "repeated",
"result_bytes", "max_result_bytes", "source_followups", "unfinished", "output_limit_exceeded", "timed_calls",
"stale", "truncated_results",
"lower_bound", "server_timed_calls",
)
class EventClock:
"""Record receipt offsets without retaining or parsing event content.
Called on the stdout reader thread; no I/O, unbounded buffers, or locks.
Several events in one chunk share a receipt time. Missing timing stays unknown.
"""
def __init__(self) -> None:
self.started = time.monotonic()
self.offsets: list[float] = []
self.partial = False
self.last = 0.0
self.overflow = False
def observe(self, chunk: bytes) -> None:
self.last = time.monotonic() - self.started
parts = chunk.split(b"\n")
for part in parts[:-1]:
if self.partial or part.strip():
if len(self.offsets) < MAX_EVENTS:
self.offsets.append(self.last)
else:
self.overflow = True
self.partial = False
self.partial = self.partial or bool(parts[-1].strip())
def finish(self, expected: int) -> list[float] | None:
offsets = [*self.offsets, *([self.last] if self.partial else [])]
return offsets if not self.overflow and len(offsets) <= MAX_EVENTS and len(offsets) == expected else None
def _content(event: dict[str, Any]) -> list[Any]:
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if content is None:
content = event.get("content")
return content if isinstance(content, list) else []
def _has_shell_control(command: str) -> bool:
"""Reject operators/expansions while allowing quoted or escaped literals."""
quote = None
escaped = False
for char in command:
if escaped:
escaped = False
continue
if quote == "'":
if char == "'":
quote = None
continue
if char == "\\":
escaped = True
elif char == '"':
quote = None if quote == '"' else '"'
elif char == "'" and quote is None:
quote = "'"
elif char in "`$" or (quote is None and char in ";|&\n<>()"):
return True
return False
def _tool_name(block: dict[str, Any]) -> str:
name = block.get("name")
if name == "Bash" and isinstance(block.get("input"), dict):
# Only a standalone shim command. Compound shell commands retain Bash
# attribution: their elapsed time cannot be called GitNexus server time.
command = block["input"].get("command")
try:
words = shlex.split(command) if isinstance(command, str) else []
except ValueError:
words = []
if (len(words) >= 2 and PurePosixPath(words[0]).name == "gitnexus"
and words[1] in COMMANDS and not _has_shell_control(command)):
return f"cli__gitnexus__{words[1]}"
return name if isinstance(name, str) and name in TOOLS else "other"
def _result_signals(content: Any) -> dict[str, Any]:
text = content if isinstance(content, str) else "\n".join(
block["text"] for block in content
if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str)
) if isinstance(content, list) else ""
signals = {
"output_limit_exceeded": "exceeds maximum allowed tokens" in text,
"semantic_error": text.lstrip().lower().startswith("error:"),
}
try:
body = json.loads(text.split("\n\n---", 1)[0])
except (ValueError, TypeError, RecursionError):
return signals
if not isinstance(body, dict):
return signals
error = body.get("error")
signals["semantic_error"] = isinstance(error, str)
signals["not_found"] = body.get("status") == "not_found" or isinstance(error, str) and "not found" in error.lower()
signals["ambiguous"] = body.get("status") == "ambiguous"
signals["unknown_risk"] = body.get("risk") == "UNKNOWN"
sections = [body[key] for key in ("processes", "process_symbols", "definitions") if key in body]
signals["empty"] = bool(sections) and all(section == [] for section in sections)
signals["truncated"] = body.get("truncated") is True or body.get("partial") is True
signals["lower_bound"] = body.get("epistemic") == "lower-bound"
timing = body.get("timing")
wall = timing.get("wall") if isinstance(timing, dict) else None
if type(wall) in (int, float) and 0 <= wall <= 10**12:
signals["server_wall_ms"] = wall
staleness = body.get("staleness")
if isinstance(staleness, dict) and staleness.get("status") in ("current", "behind", "diverged", "unknown"):
signals["staleness"] = staleness["status"]
return signals
def _union_seconds(intervals: list[tuple[float, float]]) -> float:
total, end = 0.0, 0.0
for start, stop in sorted(intervals):
total += max(0.0, stop - max(end, start))
end = max(end, stop)
return round(total, 6)
def diagnose_session(
events: list[dict[str, Any]], offsets: list[float] | None, duration_s: float,
) -> dict[str, Any]:
"""Explain observed tool work; never estimate tool dollars or causal savings."""
timing = offsets is not None and len(offsets) == len(events) and all(
type(value) in (int, float) and math.isfinite(value) and 0 <= value <= duration_s
and (index == 0 or value >= offsets[index - 1]) for index, value in enumerate(offsets)
)
calls: list[dict[str, Any]] = []
inputs: dict[int, Any] = {}
results: dict[int, Any] = {}
pending: dict[str, dict[str, Any]] = {}
signatures: dict[str, int] = {}
turns: dict[str, dict[str, Any]] = {}
retries = 0
truncated = False
duplicate_ids = False
seen_ids: set[str] = set()
for index, event in enumerate(events):
if event.get("type") == "system" and event.get("subtype") == "api_retry":
retries += 1
if event.get("type") == "assistant":
message = event.get("message") or {}
# Streaming can repeat a message id. Retain its last usage snapshot
# rather than adding cumulative snapshots as separate model requests.
if isinstance(message, dict) and isinstance(message.get("usage"), dict):
key = str(message.get("id", index))
if key in turns or len(turns) < MAX_CALLS:
turns[key] = {"event_index": index, **{
field: value for field in USAGE
if type(value := message["usage"].get(field)) is int and value >= 0
}}
else:
truncated = True
for block in _content(event):
if not isinstance(block, dict) or block.get("type") != "tool_use":
continue
if len(calls) >= MAX_CALLS:
truncated = True
continue
tool_id = block.get("id")
if not isinstance(tool_id, str) or not 0 < len(tool_id) <= 256:
duplicate_ids = True
continue
if tool_id in seen_ids:
duplicate_ids = True
continue
seen_ids.add(tool_id)
name = _tool_name(block)
encoded = json.dumps(block.get("input"), sort_keys=True, ensure_ascii=False, separators=(",", ":"))
signature = hashlib.sha256((name + encoded).encode(errors="backslashreplace")).hexdigest()
call = {
"index": len(calls), "tool": name, "request_event": index,
"started_s": offsets[index] if timing else None,
"completed_s": None, "duration_s": None, "result_event": None,
"input_bytes": len(encoded.encode(errors="backslashreplace")), "result_bytes": None,
"repeat_of": signatures.get(signature), "outcome": "unfinished",
"source_followups": 0,
}
signatures[signature] = call["index"]
pending[tool_id] = call
calls.append(call)
inputs[call["index"]] = block.get("input")
elif event.get("type") == "user":
for block in _content(event):
if not isinstance(block, dict) or block.get("type") != "tool_result":
continue
tool_id = block.get("tool_use_id")
call = pending.pop(tool_id, None) if isinstance(tool_id, str) else None
if call is None:
duplicate_ids = True
continue
content = block.get("content")
results[call["index"]] = content
call["result_bytes"] = len(json.dumps(
content, ensure_ascii=False, separators=(",", ":"),
).encode(errors="backslashreplace"))
call["result_event"] = index
if timing:
call["completed_s"] = offsets[index]
call["duration_s"] = round(offsets[index] - call["started_s"], 6)
call["outcome"] = "error" if block.get("is_error") is True else "ok"
if "__gitnexus__" in call["tool"]:
call.update(_result_signals(content))
if call.get("semantic_error") or call.get("output_limit_exceeded"):
call["outcome"] = "error"
# Only inspect the immediately following assistant action; this is a
# temporal signal, not evidence that GitNexus caused the source read.
next_actions: dict[int, list[str]] = {}
next_tools: list[str] = []
for index in range(len(events) - 1, -1, -1):
next_actions[index] = next_tools
if events[index].get("type") == "assistant":
next_tools = [_tool_name(b) for b in _content(events[index])
if isinstance(b, dict) and b.get("type") == "tool_use"]
tools: dict[str, Counter] = {}
intervals, mcp_intervals = [], []
for call in calls:
following = next_actions.get(call["result_event"], [])
call["next_tools"] = following
call["source_followups"] = sum(name in ("Read", "Grep", "Glob") for name in following)
totals = tools.setdefault(call["tool"], Counter({**dict.fromkeys(COUNTERS, 0), "elapsed_s": 0.0, "server_wall_s": 0.0}))
totals["calls"] += 1
totals["errors"] += call["outcome"] == "error"
totals["unfinished"] += call["outcome"] == "unfinished"
totals["repeated"] += call["repeat_of"] is not None
totals["result_bytes"] += call["result_bytes"] or 0
totals["max_result_bytes"] = max(totals["max_result_bytes"], call["result_bytes"] or 0)
for field in ("ambiguous", "not_found", "empty", "unknown_risk", "output_limit_exceeded", "lower_bound"):
totals[field] += call.get(field) is True
if "server_wall_ms" in call:
totals["server_timed_calls"] += 1
totals["server_wall_s"] += call["server_wall_ms"] / 1000
totals["stale"] += call.get("staleness") in ("behind", "diverged")
totals["truncated_results"] += call.get("truncated") is True
call["problematic"] = call["outcome"] == "error" or any(
call.get(field) for field in ("not_found", "ambiguous", "empty", "unknown_risk")
)
totals["source_followups"] += call["source_followups"] if call["problematic"] else 0
if call["duration_s"] is not None:
totals["timed_calls"] += 1
totals["elapsed_s"] += call["duration_s"]
interval = (call["started_s"], call["completed_s"])
intervals.append(interval)
if call["tool"].startswith("mcp__gitnexus__"):
mcp_intervals.append(interval)
complete = bool(events) and timing and not (pending or truncated or duplicate_ids)
wait = _union_seconds(intervals) if complete else None
return {
"schema": SCHEMA, "timing_complete": complete, "truncated": truncated,
"tool_wait_s": wait, "mcp_wait_s": _union_seconds(mcp_intervals) if complete else None,
"other_wall_s": round(max(0.0, duration_s - wait), 6) if wait is not None else None,
"api_retries": retries, "tools": {
name: {**values, "elapsed_s": round(values["elapsed_s"], 6) if values["timed_calls"] else None,
"server_wall_s": round(values["server_wall_s"], 6) if values["server_timed_calls"] else None}
for name, values in tools.items()
}, "calls": calls, "assistant_usage": list(turns.values()),
"workload": workload_metrics(
[(call, inputs.get(call["index"]), results.get(call["index"])) for call in calls],
sorted(turns.values(), key=lambda turn: turn["event_index"]), events,
),
}
def public_diagnostics(value: Any) -> list[dict[str, Any]]:
"""Idempotent metadata whitelist for public summaries and bounded outliers."""
output = []
for session in value[:16] if isinstance(value, list) else []:
if not isinstance(session, dict) or session.get("schema") != SCHEMA:
continue
safe: dict[str, Any] = {"schema": SCHEMA}
for field in ("timing_complete", "truncated"):
safe[field] = session.get(field) is True
for field in ("tool_wait_s", "mcp_wait_s", "other_wall_s", "api_retries"):
number = session.get(field)
safe[field] = number if type(number) in (int, float) and 0 <= number <= 10**12 else None
safe["tools"] = {}
if "workload" in session:
safe["workload"] = public_workload(session["workload"])
tools = session.get("tools")
if isinstance(tools, dict):
for name in sorted(TOOLS):
row = tools.get(name)
if not isinstance(row, dict):
continue
safe["tools"][name] = {
field: number if type(number := row.get(field)) is int and 0 <= number <= 10**12 else None
for field in COUNTERS
}
for field in ("elapsed_s", "server_wall_s"):
elapsed = row.get(field)
safe["tools"][name][field] = elapsed if type(elapsed) in (int, float) and 0 <= elapsed <= 10**12 else None
# Keep a few inspectable outliers in the already-uploaded release JSON.
# Full calls and assistant usage remain in local results.jsonl.
for field, sort_key in (
("largest_results", "result_bytes"), ("slowest_calls", "duration_s"), ("problem_calls", "result_bytes"),
):
candidates = session.get(field, session.get("calls", []))
safe[field] = []
if not isinstance(candidates, list):
continue
for call in sorted(
[c for c in candidates[:MAX_CALLS] if isinstance(c, dict)
and (field != "problem_calls" or c.get("problematic") is True)
and type(c.get(sort_key)) in (int, float) and 0 <= c[sort_key] <= 10**12],
key=lambda c: c[sort_key], reverse=True,
)[:5]:
tool = call.get("tool")
item = {"tool": tool if isinstance(tool, str) and tool in TOOLS else "other"}
for name in ("index", "request_event", "result_event", "input_bytes", "result_bytes", "duration_s", "source_followups", "server_wall_ms"):
number = call.get(name)
item[name] = number if type(number) in (int, float) and 0 <= number <= 10**12 else None
outcome = call.get("outcome")
item["outcome"] = outcome if outcome in ("ok", "error", "unfinished") else "unknown"
for name in ("problematic", "not_found", "ambiguous", "empty", "unknown_risk", "output_limit_exceeded"):
item[name] = call.get(name) is True
safe[field].append(item)
output.append(safe)
return output
def public_cell_diagnostics(row: dict[str, Any]) -> dict[str, Any]:
"""Shared publication boundary for release and optional-arm artifacts."""
safe = {}
verification = row.get("verification_diagnostics")
if isinstance(verification, dict):
safe["verification_diagnostics"] = {
phase: public_verification(verification[phase])
for phase in ("authored", "hidden") if phase in verification
}
if "session_diagnostics" in row:
safe["session_diagnostics"] = public_diagnostics(row["session_diagnostics"])
if "cell_duration_s" in row:
value = row["cell_duration_s"]
safe["cell_duration_s"] = value if type(value) in (int, float) and 0 <= value <= 10**12 else None
phases = row.get("cell_phase_seconds")
if isinstance(phases, dict):
safe["cell_phase_seconds"] = {
field: value if type(value := phases[field]) in (int, float) and 0 <= value <= 10**12 else None
for field in CELL_PHASES if field in phases
}
return safe
def render_diagnostics(rows: list[dict[str, Any]]) -> list[str]:
"""Add tool diagnostics to the existing public report without new CI wiring."""
lines = []
for arm in ("baseline_nomcp", "baseline", "baseline_cli"):
cells = [row for row in rows if row.get("arm") == arm]
sessions = [s for row in cells for s in public_diagnostics(row.get("session_diagnostics"))]
if not sessions:
continue
measured = sum(bool(public_diagnostics(row.get("session_diagnostics"))) for row in cells)
lines += ["", f"### Tool diagnostics: {arm}", "",
f"Diagnostics available for {measured}/{len(cells)} cells.", "",
"| Tool | Calls | Errors | Ambiguous | Not found | Empty | UNKNOWN risk | Repeated args | Result bytes | Largest result | Source follow-ups after problems | Timed calls | Elapsed seconds |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
totals: dict[str, Counter] = {}
coverage: dict[str, Counter] = {}
tool_sessions: Counter = Counter()
for session in sessions:
for name, values in session["tools"].items():
tally = totals.setdefault(name, Counter())
known = coverage.setdefault(name, Counter())
tool_sessions[name] += 1
for field, number in values.items():
if number is not None:
known[field] += 1
tally[field] = max(tally[field], number) if field == "max_result_bytes" else tally[field] + number
for name, tally in sorted(totals.items()):
lines.append("| " + " | ".join(str(v) for v in (
name, *(tally[f] if coverage[name][f] == tool_sessions[name] else "n/a" for f in
("calls", "errors", "ambiguous", "not_found", "empty", "unknown_risk",
"repeated", "result_bytes", "max_result_bytes", "source_followups", "timed_calls")),
f"{tally['elapsed_s']:.1f}" if tally["timed_calls"] else "n/a",
)) + " |")
lines += ["", "| Tool | Stale | Truncated results | Lower bound | Output limit | Unfinished | Server-timed calls | Server seconds |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
for name, tally in sorted(totals.items()):
lines.append("| " + " | ".join(str(v) for v in (
name, *(tally[f] if coverage[name][f] == tool_sessions[name] else "n/a" for f in
("stale", "truncated_results", "lower_bound", "output_limit_exceeded", "unfinished", "server_timed_calls")),
f"{tally['server_wall_s']:.3f}" if tally["server_timed_calls"] else "n/a",
)) + " |")
complete = [s for s in sessions if s["timing_complete"] and s["tool_wait_s"] is not None]
lines += ["", f"Complete host timing: {len(complete)}/{len(sessions)} sessions."]
limited = sum(s["truncated"] for s in sessions)
if limited:
lines.append(f"Diagnostics truncated in {limited} session(s); tool counts are lower bounds.")
if complete:
lines.append(f"Observed tool wait: {sum(s['tool_wait_s'] for s in complete):.1f}s; "
f"MCP wait: {sum(s['mcp_wait_s'] or 0 for s in complete):.1f}s "
"(a subset of tool wait). Parallel intervals are counted once per session.")
for field, label in (("other_wall_s", "Other session wall time (seconds)"), ("api_retries", "Observed API retries")):
values = [s[field] for s in sessions if s.get(field) is not None]
lines.append(f"{label}: {sum(values):.1f} across {len(values)}/{len(sessions)} sessions."
if values else f"{label}: n/a.")
workloads = [s["workload"] for s in sessions if s.get("workload")]
if workloads:
lines += ["", f"Workload signals available for {len(workloads)}/{len(sessions)} sessions.", "",
"| Workload signal | Observed count | Sessions reporting |", "| --- | ---: | ---: |"]
for field in COUNT_FIELDS:
values = [w[field] for w in workloads if w.get(field) is not None]
lines.append(f"| {field} | {sum(values) if values else 'n/a'} | {len(values)}/{len(sessions)} |")
peaks = [w["max_input_tokens"] for w in workloads if w.get("max_input_tokens") is not None]
if peaks:
lines.append(f"\nLargest observed CLI input context: {max(peaks):,} tokens. "
"Uses complete Anthropic-shaped usage snapshots; provider-native usage may differ.")
lines += ["", "| Context/cache signal | Session minimum | Session maximum | Sessions reporting |",
"| --- | ---: | ---: | ---: |"]
for field in TOKEN_FIELDS:
values = [w[field] for w in workloads if w.get(field) is not None]
lines.append(f"| {field} | {min(values) if values else 'n/a'} | {max(values) if values else 'n/a'} | "
f"{len(values)}/{len(sessions)} |")
lines += ["", "<details><summary>Largest, slowest, and problematic tool calls</summary>", "",
"Session and call indexes identify observations; arguments and source content are omitted.", "",
"| View | Session | Call | Tool | Result bytes | Host seconds | Outcome |",
"| --- | ---: | ---: | --- | ---: | ---: | --- |"]
for view, sort_key in (("largest_results", "result_bytes"), ("slowest_calls", "duration_s"),
("problem_calls", "result_bytes")):
calls = [(index, call) for index, session in enumerate(sessions) for call in session[view]]
for index, call in sorted(calls, key=lambda item: item[1][sort_key], reverse=True)[:5]:
duration = call["duration_s"]
lines.append(f"| {view} | {index} | {call['index']} | {call['tool']} | {call['result_bytes']} | "
f"{f'{duration:.3f}' if duration is not None else 'n/a'} | {call['outcome']} |")
lines += ["", "</details>"]
if lines:
lines += ["", ("Receipt timing includes transport and scheduling, not just server execution. Other wall time is not model time. "
"Bytes are serialized payload bytes, not tokens. Repeated arguments may be justified after edits. "
"Source follow-ups are the next assistant action after a problematic result, not proof of a workaround. "
"Per-tool elapsed sums may overlap. Session cost includes all work; no per-tool dollar attribution is available.")]
lines += [("A file read after a graph reference is an association, not duplicated content or low relevance. "
"Repeated verification can be necessary. Bash classification is conservative; mixed commands stay unclassified. "
"Missing usage fields remain unknown. Context totals describe observed snapshots only.")]
return lines

View file

@ -0,0 +1,41 @@
"""Bounded verifier evidence without publishing candidate or hidden-test output."""
from __future__ import annotations
from typing import Any
from .runner_artifacts import VerificationResult
# These are observed text markers, not authoritative diagnoses. Never publish
# matched text: candidate code can print source, credentials, or hidden tests.
MARKERS = {
"assertion_failure": ("AssertionError",),
"startup_failure": ("Failed Suites", "Startup Error", "Unhandled Errors", "Cannot find module"),
"permission_failure": ("EACCES", "EROFS", "Permission denied", "Failed to create Vitest API token"),
"typecheck_failure": ("error TS",),
}
def verification_metrics(result: VerificationResult | tuple[bool, str], elapsed: float) -> dict[str, Any]:
passed, output = result
return {
"passed": passed,
"duration_s": elapsed,
"exit_code": result.process.returncode if isinstance(result, VerificationResult) else None,
"state": result.process.state if isinstance(result, VerificationResult) else "unknown",
**{field: any(marker in output for marker in markers) for field, markers in MARKERS.items()},
}
def public_verification(value: Any) -> dict[str, Any]:
if not isinstance(value, dict):
return {}
code, elapsed, state = value.get("exit_code"), value.get("duration_s"), value.get("state")
return {
"exit_code": code if type(code) is int and -255 <= code <= 255 else None,
"duration_s": elapsed if type(elapsed) in (int, float) and 0 <= elapsed <= 10**12 else None,
"state": state if state in ("exited", "timeout", "cancelled", "input-failure", "spawn-failure", "forced-kill",
"ownership-failure", "reap-failure", "cleanup-failure", "unknown") else "unknown",
**{field: value.get(field) if type(value.get(field)) is bool else None
for field in ("passed", *MARKERS)},
}

View file

@ -0,0 +1,252 @@
"""Content-free workload signals. Associations are not causal cost attribution."""
from __future__ import annotations
import json
import shlex
from collections import Counter
from pathlib import PurePosixPath
from typing import Any
# Also the publication allowlist. No arguments, paths, source, or hashes leave
# this module. Unknown/missing observations must not become measured zeros.
COUNT_FIELDS = (
"source_reads", "repeated_source_reads", "reads_after_graph_reference",
"reads_after_graph_content_reference", "query_context_read_chains",
"graph_calls", "graph_calls_before_first_edit", "edits", "edits_after_verification",
"verification_calls", "verification_repeated_without_observed_change",
"verification_repeated_after_possible_change", "verification_errors",
"bash_test", "bash_typecheck", "bash_lint", "bash_build", "bash_source",
"bash_mutation", "bash_mixed", "bash_other", "compactions",
"usage_snapshots", "complete_input_snapshots", "usage_after_graph_snapshots",
"reference_scan_limited",
)
TOKEN_FIELDS = (
"first_input_tokens", "last_input_tokens", "max_input_tokens", "sum_input_tokens",
"sum_cache_read_tokens", "sum_cache_write_tokens", "sum_output_tokens",
"input_tokens_after_graph",
)
FIELDS = (*COUNT_FIELDS, *TOKEN_FIELDS)
VERIFY = frozenset({"test", "typecheck", "lint", "build"})
_UV_RUN_FLAGS = frozenset({"--locked", "--frozen", "--no-sync", "--offline", "--all-extras",
"--no-dev", "--only-dev", "--isolated", "--active", "--no-project"})
_UV_RUN_VALUES = frozenset({"--extra", "--group", "--python", "--project", "--directory",
"--package", "--with"})
def classify_bash(command: Any) -> str:
"""Conservative lexical classification, never shell evaluation.
Compound commands are explicitly mixed; their time cannot be assigned to
one test/build. A common `cd dir && command` prefix is the sole exception.
"""
if not isinstance(command, str):
return "other"
try:
words = shlex.split(command)
except ValueError:
return "other"
if len(words) >= 4 and words[0] == "cd" and words[2] == "&&":
words = words[3:]
# Inspect the remainder too: shell substitutions and pipes stay mixed.
command = shlex.join(words)
if any(op in command for op in (";", "|", "&", "\n", "`", "$(", ">", "<")):
return "mixed"
if not words:
return "other"
executable = PurePosixPath(words[0]).name
args = words[1:]
if executable == "uv" and args[:1] == ["run"]:
args = args[1:]
while args and args[0].startswith("-"):
option = args.pop(0)
if option == "--":
break
name, separator, value = option.partition("=")
if name in _UV_RUN_FLAGS and not separator:
continue
if name in _UV_RUN_VALUES:
if separator and value:
continue
if not separator and args and args[0] and not args[0].startswith("-"):
args.pop(0)
continue
# Unknown or malformed options may change what uv executes.
return "other"
return classify_bash(shlex.join(args))
if executable in ("python", "python3") and args[:2] == ["-m", "pytest"]:
return "test"
if executable in ("npm", "pnpm", "yarn", "bun", "npx") and args:
if args[0] in ("run", "exec", "x"):
args = args[1:]
executable = args[0] if args else ""
if executable in ("test", "pytest", "vitest", "jest") or executable.startswith("test:"):
return "test"
if executable in ("tsc", "mypy", "pyright", "typecheck"):
return "typecheck"
if executable in ("eslint", "ruff", "prettier", "lint"):
mutates = any(arg in ("--fix", "--write", "-w") for arg in args)
if executable == "ruff" and "format" in args and not any(arg in ("--check", "--diff") for arg in args):
mutates = True
return "mutation" if mutates else "lint"
if executable in ("build", "vite", "tsup"):
return "build"
if executable in ("rg", "grep", "cat", "head", "tail", "sed"):
return "mutation" if executable == "sed" and any(
arg.startswith(("-i", "--in-place")) for arg in args
) else "source"
if executable in ("apply_patch", "patch", "touch", "rm", "mv", "cp", "tee"):
return "mutation"
return "other"
def _references(content: Any) -> tuple[dict[str, bool], bool]:
"""Structured result references only; cap traversal of large tool results."""
if isinstance(content, list):
text = "\n".join(b["text"] for b in content
if isinstance(b, dict) and isinstance(b.get("text"), str))
else:
text = content if isinstance(content, str) else ""
if len(text) > 2_000_000:
return {}, True
try:
body = json.loads(text.split("\n\n---", 1)[0])
except (ValueError, RecursionError):
return {}, False
stack, paths, visited = [body], {}, 0
while stack and visited < 10_000 and len(paths) < 1000:
item = stack.pop()
visited += 1
if isinstance(item, dict):
path = item.get("filePath", item.get("file_path"))
if isinstance(path, str) and path and len(path) <= 4096:
# Same-object source only. Descriptions and arbitrary text are
# not evidence that the response supplied the file's contents.
supplied = any(isinstance(item.get(key), str) and bool(item[key].strip())
for key in ("content", "source", "code"))
paths[path] = paths.get(path, False) or supplied
stack.extend(value for value in item.values() if isinstance(value, (dict, list)))
elif isinstance(item, list):
stack.extend(item)
return paths, bool(stack)
def _same_path(read: str, reference: str) -> bool:
read_parts, ref_parts = PurePosixPath(read).parts, PurePosixPath(reference).parts
if ".." in read_parts or ".." in ref_parts:
return False
if read_parts == ref_parts:
return True
# Absolute sandbox paths can match repo-relative graph paths. Never match
# unrelated absolute roots or a basename-only relative reference.
return (len(ref_parts) > 1 and not reference.startswith("/")
and len(read_parts) >= len(ref_parts) and read_parts[-len(ref_parts):] == ref_parts)
def workload_metrics(
observations: list[tuple[dict[str, Any], Any, Any]],
usage: list[dict[str, Any]], events: list[dict[str, Any]],
) -> dict[str, int | None]:
"""Compute bounded numeric signals from accepted, deduplicated calls.
`observations` holds (call metadata, input, result content) only in memory.
No assumption that an unknown Bash action left the checkout unchanged.
"""
counts = Counter(dict.fromkeys(COUNT_FIELDS, 0))
counts["compactions"] = sum(e.get("type") == "system" and e.get("subtype") == "compact_boundary" for e in events)
reads: set[str] = set()
checks: dict[str, int] = {}
mutation_epoch = 0
verified = False
graph_results: list[tuple[int, str, dict[str, bool]]] = []
reference_budget = 5000
for call, args, content in observations:
args = args if isinstance(args, dict) else {}
tool = call["tool"]
if "__gitnexus__" in tool:
counts["graph_calls"] += 1
counts["graph_calls_before_first_edit"] += counts["edits"] == 0
if call.get("result_event") is not None:
paths, limited = _references(content)
if len(paths) > reference_budget:
paths = dict(list(paths.items())[:reference_budget])
limited = True
reference_budget -= len(paths)
counts["reference_scan_limited"] += limited
graph_results.append((call["result_event"], tool.rsplit("__", 1)[-1], paths))
elif tool == "Read":
counts["source_reads"] += 1
signature = json.dumps(args, sort_keys=True, ensure_ascii=True)
counts["repeated_source_reads"] += signature in reads
reads.add(signature)
path = args.get("file_path")
prior = sorted((g for g in graph_results if g[0] < call["request_event"]), key=lambda g: g[0])
if isinstance(path, str):
matching = [supplied for _, _, paths in prior for ref, supplied in paths.items() if _same_path(path, ref)]
counts["reads_after_graph_reference"] += bool(matching)
counts["reads_after_graph_content_reference"] += any(matching)
counts["query_context_read_chains"] += (
len(prior) >= 2 and [g[1] for g in prior[-2:]] == ["query", "context"]
and any(_same_path(path, ref) for ref in prior[-1][2])
)
elif tool in ("Edit", "Write"):
counts["edits"] += 1
counts["edits_after_verification"] += verified
mutation_epoch += 1
elif tool == "Bash":
category = classify_bash(args.get("command"))
counts[f"bash_{category}"] += 1
if category in VERIFY:
counts["verification_calls"] += 1
counts["verification_errors"] += call["outcome"] == "error"
signature = json.dumps(args, sort_keys=True, ensure_ascii=True)
if signature in checks:
field = "verification_repeated_without_observed_change" if checks[signature] == mutation_epoch else "verification_repeated_after_possible_change"
counts[field] += 1
checks[signature] = mutation_epoch
verified = True
elif category in ("mutation", "mixed", "other"):
mutation_epoch += 1
if category == "mutation":
counts["edits"] += 1
counts["edits_after_verification"] += verified
counts["usage_snapshots"] = len(usage)
# Some CLI/provider adapters emit all-zero placeholders for completed
# assistant turns. They are not evidence of an empty model context.
usage = [u if any(type(u.get(f)) is int and u[f] > 0 for f in (
"input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens", "output_tokens",
)) else {"event_index": u["event_index"]} for u in usage]
input_fields = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens")
totals = [sum(u[f] for f in input_fields)
if all(type(u.get(f)) is int and u[f] >= 0 for f in input_fields) else None for u in usage]
complete = [total for total in totals if total is not None]
counts["complete_input_snapshots"] = len(complete)
after_graph = []
for u, total in zip(usage, totals, strict=True):
preceding = [call for call, _, _ in observations
if call.get("result_event") is not None and call["result_event"] < u["event_index"]]
latest = max((call["result_event"] for call in preceding), default=-1)
if any("__gitnexus__" in c["tool"] and c["result_event"] == latest for c in preceding):
after_graph.append(total)
counts["usage_after_graph_snapshots"] = len(after_graph)
tokens = {
"first_input_tokens": totals[0] if totals else None,
"last_input_tokens": totals[-1] if totals else None,
"max_input_tokens": max(complete) if complete else None,
"sum_input_tokens": sum(complete) if totals and len(complete) == len(totals) else None,
"input_tokens_after_graph": sum(after_graph) if after_graph and all(t is not None for t in after_graph) else None,
}
for dest, source in (("sum_cache_read_tokens", "cache_read_input_tokens"),
("sum_cache_write_tokens", "cache_creation_input_tokens"), ("sum_output_tokens", "output_tokens")):
# A partial sum labelled as total would understate spend. These fields
# stay unknown unless every observed usage snapshot supplies the field.
tokens[dest] = sum(u[source] for u in usage) if usage and all(
type(u.get(source)) is int and u[source] >= 0 for u in usage
) else None
return {**counts, **tokens}
def public_workload(value: Any) -> dict[str, int | None]:
if not isinstance(value, dict):
return {}
return {field: n if type(n := value.get(field)) is int and 0 <= n <= 10**15 else None for field in FIELDS}

View file

@ -87,6 +87,7 @@ Generates repository documentation from the knowledge graph using an LLM. HTTP p
| `--concurrency <n>` | Parallel LLM calls (default: 3) |
| `--timeout <seconds>` | LLM request timeout in seconds (default: disabled) |
| `--retries <n>` | Max LLM retry attempts per request (default: 3) |
| `--max-completion-tokens <n>` | Max completion tokens per LLM request (default: 16384). `GITNEXUS_MAX_COMPLETION_TOKENS` overrides `maxCompletionTokens` in `~/.gitnexus/config.json`; the flag overrides both. |
| `--lang <lang>` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) |
| `--gist` | Publish wiki as a public GitHub Gist |

View file

@ -5084,9 +5084,9 @@
}
},
"node_modules/follow-redirects": {
"version": "1.16.1",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz",
"integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==",
"version": "1.16.2",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.2.tgz",
"integrity": "sha512-/kVP2U8z596Ia0gwFrwyIeo74ta5aVmSeU1FfAUMAmcEP5EP7zydmag3SLiN64NaRKef3lxsq7i/g7ltJQDGow==",
"funding": [
{
"type": "individual",

View file

@ -409,6 +409,21 @@ anchors are deliberately omitted. Add common infrastructure fields such as `/hea
`--spring-actuator` is explicitly opt-in. The path may be a JSON bundle keyed by `mappings`, `beans`, `conditions`, `configprops`, and/or `env`, or a directory containing endpoint-named JSON files. Runtime mappings and beans confirm matching static nodes; conditions and configuration property keys enrich existing evidence, with conservative runtime-only nodes added when no match exists. The configured input is excluded from source scanning; only normalized repository-relative exclusions are retained for future scans, never absolute paths. Env/configprops values, origins, condition messages, and source names are never persisted or printed. Enabled runs always rebuild because runtime snapshots are external to git freshness; omitting the option later rebuilds once to remove runtime evidence. Project config can set the same path with `springActuator` in `.gitnexusrc`.
Go gin/echo routes are indexed when a file imports `github.com/gin-gonic/gin` or `github.com/labstack/echo` directly. A service that registers routes through an internal wrapper package — `func InitRoute(s *core.Engine)`, where the dependency declares `type Engine = gin.Engine` — produces no routes, because the wrapper's source is usually not in the repository and the equivalence cannot be proven. Declare it in `.gitnexusrc`, together with any application adapters that wrap the real handler (`common.Wrapper(api.CreateJob)`):
```json
{
"goRouterAliases": [
{ "importPath": "example.com/infra/pkg", "type": "Engine", "framework": "gin" }
],
"goHandlerWrappers": ["common.Wrapper", "common.WrapperWithSuccessCode"]
}
```
`framework` is `gin` or `echo` and selects the handler argument order. A wrapper entry is the callee exactly as written at the registration site; its first argument becomes the route's handler. Registrations on an undeclared wrapper type are dropped, and `analyze` logs how many were dropped per `importPath.Type` so you can see what to declare. Changing either key rebuilds Route evidence on the next `analyze` without `--force`.
To keep this configuration out of the working tree, set `GITNEXUS_RC` to the absolute path of a JSON file with the same shape as `.gitnexusrc`. Its keys override the repository's `.gitnexusrc` key by key, and CLI flags still win. Hooks, MCP servers, and watch processes that re-index must see the same variable; if a later run no longer has it, `analyze` warns before removing the Routes it produced.
`--asyncapi-spec` is explicitly opt-in and accepts a directory of AsyncAPI documents or a single document; the path is resolved against the repository root, so a committed `docs/asyncapi` and an absolute cache written by something else both work. Each `operations[]` entry of an **AsyncAPI 3.x** document can contribute a `Destination` node keyed by broker and address, with `action: send` emitting `PUBLISHES_TO` and `action: receive` emitting `CONSUMES_FROM`, so a document and source code that name one address on one broker land on the same node. Edges start at the document, not at a callable — a document states that the service talks to an address, not which method does — and no address a document names is ever attached to an unresolved source site.
An operation must name a protocol, either through its own `bindings` or through the `servers[].protocol` of the servers its channel resolves to (a channel that lists no `servers` resolves to all of them); operations that name none are refused, as are operations whose two readings name different brokers, and channels that inherit a multi-protocol server set without choosing. HTTP and WebSocket documents are refused for destination minting: there the host rather than the address names the place, and an HTTP endpoint is already modelled as a `Route`. A parameterized address — a channel declaring `parameters`, or an address containing `{` — is refused rather than keyed: two services publishing `{env}.orders` share a pattern, not a queue. AsyncAPI **2.x is refused** under its own counted reason and never mapped, because its `publish`/`subscribe` are inverted relative to 3.x `send`/`receive` and a naive mapping would reverse the async graph while leaving it connected. Every refusal is counted, and a configured path that yields nothing is reported rather than passed over in silence.

View file

@ -1,8 +1,12 @@
# Analyze phase breakdown — where the time actually goes
Measured 2026-09-06/07 while landing #3194, #3196 and #3200. Records what the
`analyze` pipeline costs per phase, and — as importantly — the optimizations
that were measured and **rejected**, so the next person does not re-derive them.
Updated 2026-11-01 with scope-emission measurements after #3796 merged, plus
production-hardening measurements on LadybugDB **0.22.0** for unchanged dirty checkouts.
Records what the `analyze`
pipeline costs per phase and the results of optimization experiments.
The September measurements remain below as historical
evidence; the corpus and analyzer have changed, so they are not a controlled
before/after comparison.
Unlike `parse-throughput.md` (a synthetic-fixture scaffold), these numbers come
from a real repo corpus. They are not a CI gate; the gate for the parse
@ -10,7 +14,392 @@ dispatch path is `bench/parse-dispatch-rounds/`.
---
## Method, and one trap that invalidates everything
## Current measurements — 2026-10-31
### Environment and timing boundaries
- Analyzer and corpus: main at `82966afa64ec0cbd0e09084cf7af89067ccf0c2d`.
Built once with `npm ci`; no changes to the build between samples.
- Dependencies: `@ladybugdb/core` **0.22.0**, packaged FTS **0.22.0**,
`tree-sitter` **0.25.1**. Node **v22.23.3**, Linux x64, Intel Core i9-12900K,
24 logical CPUs, about 31.2 GiB RAM. This is a shared development host.
- Corpus: a detached git checkout of that commit, **7,726 tracked paths**,
**3,353 indexed files**, **3,090 parseable files** in **276 chunks**.
Both runtime and corpus are on local `/tmp` overlay storage, outside the
workspace's 9p mount. Index storage and GitNexus home are disposable and
isolated; shared-store reuse is disabled.
- Options: 16 workers, 8 GiB parent V8 heap limit, full content retention and
FTS, no embeddings or PDG, `skipAgentsMd: true`, `skipSkills: true`,
`verbose: true`. No `force` flag. Other analysis limits retain their defaults.
- Total is wall time around `await runFullAnalysis(...)`, after importing the
analyzer. It excludes CLI startup, the analyzer import, post-run graph
hashing, and process shutdown. Each sample runs in a new Node process.
- "Fresh" means no index or parse cache. OS filesystem caches were not flushed;
an earlier setup run had already read the corpus. That run disabled FTS with
`GITNEXUS_LBUG_EXTENSION_INSTALL=never` and is excluded from the results.
Valid runs use `load-only`, which successfully loads the packaged extension
without downloading it.
Pipeline timings use the existing `NODE_ENV=development` phase logs.
`PROF_SCOPE_RESOLUTION=1` supplies per-language internals and `PROF_LBUG_LOAD=1`
supplies the graph-load interval. FTS is timed from the `fts` progress callback
at 85% to `Search indexes ready` at 90%; verbose callbacks also time each index.
All samples use this same instrumentation. The residual is total time minus
these intervals and the remaining pipeline phases; it is not a CPU profile.
The parse interval includes cache hydration and worker startup; graph loading
includes CSV emission and native COPY. Neither isolates JavaScript CPU time.
### Fresh index, one-file edits, and unchanged controls
Run order: one fresh index, an unchanged clean-checkout control, three successive
one-file edits, then a repeat with **no further edit** to the dirty checkout.
The edit replaces `gitnexus/src/cli/update-notice.ts` with its pristine contents
plus `\n// phase-breakdown trial N\n`, for `N = 1, 2, 3`. These are uncommitted,
comment-only changes; they measure the smallest edit path, not API changes,
deletions, or large change sets.
All values below are seconds. Each column is one complete sample; rounding can
make the components differ from the total by 1ms.
| Step | Fresh | Edit 1 | Edit 2 | Edit 3 | Dirty repeat |
| -------------------------------- | ----------: | ----------: | ----------: | ----------: | -----------: |
| **Total** | **166.015** | **120.361** | **122.634** | **126.342** | **110.820** |
| Parsing | 62.823 | 6.598 | 6.995 | 6.750 | 6.748 |
| Scope resolution | 70.335 | 73.987 | 75.941 | 77.268 | 87.054 |
| Graph load | 10.145 | 5.141 | 5.251 | 4.813 | 0.007 |
| FTS | 16.072 | 12.792 | 13.095 | 15.731 | 0.510 |
| Other pipeline phases | 3.143 | 3.307 | 3.255 | 3.308 | 1.025 |
| Residual outside those intervals | 3.498 | 18.536 | 18.097 | 18.472 | 15.476 |
The edit median is **122.634s**, range **120.361–126.342s**, with three samples
on a shared host. This sequence measures the unmodified main baseline.
The graph contains **117,437 nodes / 352,781 relationships / 3,463 communities /
767 processes**. These reported counts matched across runs. The fresh index
built all **23** configured FTS indexes. Every edit reported:
- `changedFiles: 1`, `deletedFiles: 0`, `affectedDependents: 6`,
`writeMode: "incremental"`, and no rebuild reasons.
- **22 files reparsed / 3,068 cached**; 273 chunk hits and 3 misses.
- A row-removal set of **71 paths**, followed by a load of **6,941 nodes /
59,204 relationships**. The write set extends beyond the changed file and its
six importer dependents.
- **9 FTS indexes rebuilt**: File, Function, Class, Method, Interface, TypeAlias,
Const, Property, Route. In Edit 2, Function took **4.885s**, File **4.423s**,
and the remaining seven indexes plus orchestration **3.786s**.
**The two unchanged controls behave differently.** Immediately after the fresh
index, the clean checkout returned `alreadyUpToDate: true` in **1.245s**, without
running the pipeline. Repeating analysis after Edit 3, with the same uncommitted
contents, took **110.820s** and did not return `alreadyUpToDate`. It reported
**zero changed files**, two reparsed files and 3,088 cache hits, but still spent
**87.054s in scope resolution**. Community/process phases were skipped, the
graph loader received zero nodes and relationships, and only `Class.class_fts`
was rebuilt. This is a measured dirty-worktree repeat, not a full-rebuild
fallback or another edit sample.
The fresh run and three edits also matched a filtered in-memory graph hash
(excluding node content, Folder/Community/Process nodes, and CONTAINS,
MEMBER_OF, STEP_IN_PROCESS edges). The dirty repeat skips derived graph phases
and is excluded from that comparison. This is not a persisted-graph or query
equivalence test. Runs emitted warnings for two dropped scope sites and for
callable-flow/process limits; the timings describe that default bounded
configuration, not complete graph coverage.
### Where the edit time goes now
In Edit 2, the sample with the median total, scope resolution is **61.9%** of
wall time. Summing the existing per-language profiler gives:
| Scope-resolution work | Seconds |
| ---------------------------------------- | ---------: |
| `emit` | **49.604** |
| `resolve` | 7.043 |
| Other work inside the internal timers | 4.482 |
| Phase time outside those internal timers | 14.812 |
| **Whole phase** | **75.941** |
`emit` alone is **40.4% of the entire edit run**. TypeScript reports **2,865 /
2,865 pre-extracted hits** in every edit sample, yet resolution and emission
still process the corpus. The 14.812s outside the internal timers includes
ParsedFile loading and other phase work; it was not separately instrumented,
so it must not all be labeled rehydration.
The separate **18.097s run residual** includes work outside the pipeline,
graph-load and FTS timers, such as incremental deletion, reconciliation,
checkpoints, and publication. Those costs were not individually timed. The
graph-load profiler reports `mode=overlap`; its CSV and COPY subfields must not
be interpreted as separate CPU costs.
These measurements identified two targets: an early exit for an unchanged dirty
snapshot and reducing repeated scope emission. The early-exit experiment below
demonstrates a speedup for unchanged snapshots. Repeated scope emission remains
the target for actual edits. No scriptc build or alternate native-linking
strategy was measured; these results do not establish a compiler migration's
benefit.
### Experimental unchanged-dirty fast path — 2026-10-31
Candidate: `experiment/unchanged-dirty-fastpath`, based on the same main commit
`82966afa64ec0cbd0e09084cf7af89067ccf0c2d`. The production change is confined to
`src/core/run-analyze.ts`; native dependencies and the storage schema are
unchanged. This is an experimental result, not a released optimization.
Each variant used its own runtime, corpus checkout, index, and GitNexus home.
Both corpora started with the identical comment-only edit from Edit 3 above.
Both indexes were seeded before timing, with stable runner identities throughout
the measured sequence. Setup runs, including the candidate's rebuild after a
code revision, are excluded. Hardware, options, instrumentation, and timing
boundaries match the baseline section. Our regression tests started after the
timing runs; other workloads on the shared host were not isolated.
Run order was candidate 1, baseline 1, baseline 2, candidate 2, candidate 3,
baseline 3. Each run used a new Node process and made no source change.
| Unchanged dirty snapshot | Main baseline (s) | Candidate (s) |
| ------------------------ | ----------------: | ------------: |
| Sample 1 | 102.494 | 1.657 |
| Sample 2 | 101.905 | 1.640 |
| Sample 3 | 102.688 | 1.552 |
| **Median** | **102.494** | **1.640** |
The ratio of medians is **62.50×**, a **98.40% reduction** in elapsed analysis
time for this workload. All three baseline runs reported zero changed files,
but ran the pipeline and rewrote the database. All three candidate runs returned
`alreadyUpToDate: true` without a pipeline result. SHA-256 of the complete
LadybugDB file was identical before and after each candidate run. Reported graph
counts remained **117,437 nodes / 352,781 relationships** in both variants.
The candidate retains the existing rebuild and configuration gates. For an
eligible dirty checkout, it verifies the recorded dirty files, enumerates the
current covered paths, and compares their content hashes with the indexed
snapshot. Missing or changed bytes, coverage changes, scan failures, incomplete
scope receipts, and unknown dirty paths fall through to analysis. The full
coverage check also catches edits hidden by Git's `assume-unchanged` flag.
This deliberately conservative path does not accelerate every dirty checkout,
including snapshots containing deleted or unindexed dirty paths.
For a functional control, both corpora then received the same new exported
function and unique search token in `gitnexus/src/cli/update-notice.ts`. Baseline
took **168.291s**, candidate **149.265s**. Each reported one changed file,
22 reparsed files, 3,068 cached files, an incremental write, and no rebuild
reasons. Both produced **117,438 nodes / 352,782 relationships**, with matching
filtered in-memory graph hashes under the exclusions documented above.
Read-only queries against the persisted databases also returned identical
results: the new function and its location, one File FTS hit for the token, and
all **34** Function-to-Function call edges touching the edited file, including
confidence and reason. These are targeted checks, not exhaustive graph equality.
This real-edit pair is **not evidence of an edit speedup**: it has one sample per
variant and the shared host's one-minute load rose from 5.62 to 13.85 during the
baseline run. Repeating the candidate after that edit took **1.821s**, returned
already up to date, and again preserved the database bytes.
Validation covered **158 targeted tests across seven files**, including **13 new
fast-path cases** for native FTS/database preservation, real edits, additions,
deletions, renames, partial reverts, hidden edits, scan-policy changes, recovery,
and scan failures. The initial run passed 136 tests; 22 existing cases failed
with changing-runtime guards or incomplete reads of compiled JavaScript. All
22 passed when rerun in a fresh runtime copy with identical source. Hashes of
all **2,340 compiled build files** stayed unchanged throughout that rerun.
The build, `tsc --noEmit`, formatting, and diff checks passed; targeted ESLint
reported no errors. This was targeted validation, not the full repository suite.
To reproduce, seed each variant independently with the same dirty source, then
use the harness below in separate processes with no further edits. Keep the
runtime fixed after seeding. Record `alreadyUpToDate`, rebuild reasons, graph
counts, and a database-file hash before/after each repeat. Follow with the same
real edit in both corpora and verify persisted symbol, call-edge, and FTS results.
Do not combine these no-op timings with the actual-edit baseline.
### Production hardening — 2026-11-01
Review found a freshness race in the experiment: per-file hashes were read from
disk after parsing. A file edited between parsing and hashing could therefore
have a new hash beside an older graph; the next dirty repeat incorrectly returned
already up to date. A regression test reproduced that behavior before the fix.
Git repositories now capture hashes before parsing and retain only paths that
the pipeline actually indexed. An edit during analysis differs from that receipt
on the next run, which reconciles the graph before allowing another unchanged
repeat. This moves the existing content-hash pass and adds a path-only scan; it
does not introduce another full-content hashing pass. The analyzer still reads a
live working tree, not an atomic filesystem snapshot.
The dedicated native integration suite, `test/integration/analyze-dirty-snapshot.test.ts`,
covers both fresh and incremental versions of the race, as well as the original
fast-path and fallback cases. It is registered in the sequential LadybugDB
project and in the Windows/macOS CI subset. Database preservation uses an exact
byte comparison; the FTS check fails rather than skips when CI requires FTS.
The hardened build was reseeded and measured against the same pinned main
runtime, with three unchanged dirty repeats per variant. The corpora retained
the identical exported function from the October functional control, so both
contained **117,438 nodes / 352,782 relationships**. The seed rebuild took
150.305s and is excluded. Options and timing boundaries are unchanged.
| Unchanged dirty snapshot | Main baseline (s) | Hardened candidate (s) |
| ------------------------ | ----------------: | ---------------------: |
| Sample 1 | 134.649 | 2.200 |
| Sample 2 | 127.837 | 1.868 |
| Sample 3 | 101.765 | 1.960 |
| **Median** | **127.837** | **1.960** |
This is **65.21×** by ratio of medians (**98.47% less elapsed time**). Every
candidate repeat returned already up to date without running the pipeline and
preserved the complete database file's SHA-256. All six runs retained native
FTS and matched graph counts. Run order was candidate, baseline, baseline,
candidate, candidate, baseline. Other validation ran on the same host; observed
one-minute load ranged from 9.23 to 16.69. These results confirm the benefit for
unchanged dirty repeats, not a speedup for real edits or an improvement over the
October speedup ratio.
A subsequent real-edit control added the same exported function and search
token to both corpora. Main took **121.055s**, candidate **140.828s**. Both
reported one changed file, 22 reparsed files, six importer dependents, an
incremental write and no rebuild reasons. Filtered in-memory graph hashes and
the persisted new function, its File FTS hit, and all 34 call edges touching the
edited file matched. Counts were **117,439 nodes / 352,783 relationships**.
The next candidate repeat returned already up to date in **1.545s**.
The candidate was slower in this single real-edit pair. Scope-resolution time
was 73.939s on main and 91.838s on the candidate, accounting for most of the
19.773s difference; the first pipeline progress event arrived at 0.923s and
1.569s respectively. Those phase timings locate the difference, but do not
establish its cause. Host load changed and this pair has one sample per
variant: neither a real-edit speedup nor a stable regression is established.
### Scope-emission experiment after #3796 — 2026-11-01
Baseline runtime and corpus: merged main at
`568ebdc1d9e6fce80a4da306bcb1ecebb20b60e4`. This includes the dirty-snapshot
shortcut and the intervening Java overload changes. The corpus now has **3,375
indexed files**, **3,111 parseable files**, **118,517 nodes / 356,369 relationships /
3,495 communities / 767 processes**. These measurements form a new paired
comparison; the earlier corpus timings are not its baseline.
A separate Node CPU profile of a one-file edit identified repeated namespace
lookups inside receiver-bound call emission. The main-thread profile recorded
**29.6s in garbage collection**, **21.7s inclusive in `emitReceiverBoundCalls`**,
and **6.2s inclusive in `collectNamespaceTargets`**. Those intervals overlap and
must not be added. The profile includes startup and post-run graph hashing;
its **140.158s** analysis run is excluded from the timing comparison below.
The candidate adds a proof to `createNamespaceTargetCache`: while traversing a
file's scopes, record whether any resolved namespace import could produce a
target. If that traversal is complete and has none, return one empty map for
covered scopes instead of repeatedly resolving named imports and rebuilding
empty maps. The lexical receiver-position policy remains unchanged. Incomplete
scope views and lookups outside the traversed scopes retain the existing path.
This changes no native bindings, dependencies, public flags or graph schema.
Each runtime and corpus has its own index and stable runner identity. Both are
on local overlay storage. The first use of the separate baseline runtime caused
an identity rebuild; that setup run is excluded. Candidate seeding is also
excluded. The timer, options and instrumentation match the method above:
Node **22.23.3**, LadybugDB/FTS **0.22.0**, Tree-sitter **0.25.1**, 16 workers,
8 GiB parent heap limit, full retention and FTS, no embeddings or PDG. CPU
profiling is disabled for every sample in the table.
Each edit replaces `gitnexus/src/cli/update-notice.ts` with its pristine contents
plus `\n// scope-emission profile trial N\n`, for `N = 3, 4, 5`. These measure the
smallest one-file edit path. Run order is baseline 1, candidate 1, candidate 2,
baseline 2, baseline 3, candidate 3. No local test suite runs alongside them.
The host remains shared; one-minute loads recorded before and after the samples
range from **1.67 to 4.51**.
| One-file edit | Main baseline (s) | Candidate (s) |
| ------------- | ----------------: | ------------: |
| Sample 1 | 122.120 | 111.554 |
| Sample 2 | 126.536 | 114.429 |
| Sample 3 | 124.307 | 116.258 |
| **Median** | **124.307** | **114.429** |
The ratio of medians is **1.086×**, or **7.95% less elapsed time**. Every paired
candidate sample is faster. This is an edit-path improvement; the earlier 65×
result measures unchanged dirty repeats.
Independent medians of the existing phase timers locate the reduction:
| Interval | Main baseline (s) | Candidate (s) |
| --------------------------------------------------------- | ----------------: | ------------: |
| Scope resolution | 77.028 | 66.340 |
| `emit`, summed across languages (inside scope resolution) | 50.616 | 38.961 |
| Parsing | 6.995 | 6.966 |
| Graph load | 5.044 | 5.179 |
| FTS | 13.041 | 12.510 |
All six runs reported **one changed file**, **22 reparsed / 3,089 cached files**,
six importer dependents, an incremental write, no rebuild reasons, and available
native FTS. Their reported graph counts and filtered in-memory node/relationship
hashes matched exactly under the exclusions defined above. The same default
bounded-analysis configuration applies; this is not exhaustive persisted-graph
equivalence or a claim about every repository, language mix or change-set size.
A functional control then added two exported functions, one calling the other,
and a unique search token to the same file in both corpora. Both produced
**118,519 nodes / 356,372 relationships** and matching filtered graph hashes.
Read-only native queries returned identical function IDs and source spans, the
same File FTS hit, and all **35** Function-to-Function call edges touching the
file, including confidence and reason. The new caller-to-callee edge was present
with confidence **0.85** and reason `local-call`. This control took 138.832s on
main and 117.450s on the candidate; it is one pair and is excluded from the
comment-edit medians above.
### Reproducing the timing boundaries
Use separate runtime and corpus checkouts of the pinned commit on local storage.
Build the runtime once with `npm ci`, then leave `dist` and dependencies unchanged.
Use a fresh corpus index for the first sample; retain that index and its parse
cache for the unchanged and edit samples. Set `BENCH_ROOT` to an isolated scratch
directory and `CORPUS` to the git checkout's absolute path. From the runtime's
`gitnexus/` directory, this collects the same timing boundaries:
```bash
mkdir -p "$BENCH_ROOT/home" "$BENCH_ROOT/xdg"
NODE_ENV=development PROF_SCOPE_RESOLUTION=1 PROF_LBUG_LOAD=1 \
GITNEXUS_HOME="$BENCH_ROOT/home" XDG_RUNTIME_DIR="$BENCH_ROOT/xdg" \
GITNEXUS_SHARED_STORE=off GITNEXUS_LBUG_EXTENSION_INSTALL=load-only \
node --max-old-space-size=8192 --input-type=module - "$CORPUS" <<'JS'
import { performance } from 'node:perf_hooks';
import { runFullAnalysis } from './dist/core/run-analyze.js';
const start = performance.now();
const events = [];
const result = await runFullAnalysis(process.argv[2], {
skipAgentsMd: true, skipSkills: true, workerPoolSize: 16, verbose: true,
}, {
onProgress(phase, percent, message) {
events.push({ ms: performance.now() - start, phase, percent, message });
},
onLog(message) {
events.push({ ms: performance.now() - start, message });
},
});
const wallMs = performance.now() - start;
console.log(JSON.stringify({
wallMs, events, stats: result.stats, incrementalStats: result.incrementalStats,
alreadyUpToDate: result.alreadyUpToDate, rebuildReasons: result.rebuildReasons,
ftsSkipped: result.ftsSkipped,
reparsedFiles: result.pipelineResult?.reparsedFileCount,
cachedFiles: result.pipelineResult?.parseCacheHitFileCount,
}));
JS
```
Capture stdout and stderr for each sample. Reject an edit sample if it has
rebuild reasons, reports a full write, or skips FTS. The clean unchanged control
must return `alreadyUpToDate: true`; record the dirty repeat separately. Do not
reuse `--force` to measure edits, change the runtime between samples, or compare
a no-op with an actual edit.
## Historical measurements — 2026-09-06/07
The following measurements were taken while landing #3194, #3196 and #3200.
Their optimization experiments have not been repeated against the October
corpus or LadybugDB 0.22.0. Claims about limits and rejected approaches below
describe that September baseline.
### Method, and one trap that invalidates everything
**The corpus must be a git repository.** A non-git checkout cannot record a
schema fingerprint, so the tool forces a full rebuild on every run and prints:
@ -53,7 +442,7 @@ measured by injecting timestamp marks into a disposable copy of the built
---
## Cold analyze
### Cold analyze
| | before #3194 | after #3196 |
| ---------------- | -----------: | ----------: |
@ -66,7 +455,7 @@ measured by injecting timestamp marks into a disposable copy of the built
Graph output identical throughout: 51,286 nodes / 163,092 edges / 2106 clusters
/ 759 flows.
## Re-analyze after a one-file edit — the developer loop
### Re-analyze after a one-file edit — the developer loop
Leaf file (`cli/update-notice.ts`, 2 importers), stable runner identity, so the
incremental path is genuinely taken. **31.7s total.**
@ -94,7 +483,7 @@ this.**
A FULL-rebuild run of the same repo is 36-39s, with the graph write at ~6.3s and
FTS at ~9.7s. Do not quote those as edit-loop numbers.
## FTS index rebuild — the largest non-resolution cost, and it is a floor
### FTS index rebuild — the largest non-resolution cost, and it is a floor
A true incremental leaf edit rebuilds **8** of the 20 configured indexes — the
tables the writeback actually DMLs. Per-index cost (measured against a copy of
@ -118,7 +507,7 @@ trap above), not a leaf edit. There is nothing to fix there.
`File.file_fts` dominates because File rows carry whole-file content: 2442 rows,
32.9 MB, and Ladybug tokenizes at ~9.8 MB/s.
### Four ways out, all measured, all closed
#### Four ways out, all measured, all closed
**Narrow further — no.** The 8 tables are exactly the ones holding rows for the
6 files in the write set (1 changed + 5 importers). There is no fat.
@ -144,7 +533,7 @@ is deleting full-file keyword search (#2317/#2323), not optimizing it. Capping
the indexed content is a bad trade — the size distribution is flat, so a 64 KB
cap still indexes 90% of the bytes while truncating the 72 largest files.
### The one lever left: overlap
#### The one lever left: overlap
The FTS build runs on a libuv thread, not the main thread, and fully overlaps
blocking JS:
@ -169,7 +558,7 @@ Two things block it today, and neither is small:
would then leave the live index with fresh File content and stale symbols,
instead of untouched.
## scopeResolution — 14.7s, and it is two different problems
### scopeResolution — 14.7s, and it is two different problems
`PROF_SCOPE_RESOLUTION=1` already exists and reports the internal split. Marks
injected around the phase supply the rest. On a one-file edit:
@ -186,7 +575,7 @@ injected around the phase supply the rest. On a one-file edit:
`extract` is small because the parse cache works: 2121/2121 pre-extracted hits
on the TypeScript pass. Nothing here re-parses.
### Rehydration: three full store scans, one per language
#### Rehydration: three full store scans, one per language
The corpus resolves three languages — python (54 files), typescript (2121),
javascript (59) — and `loadParsedFilesForPaths` walks **every** shard on each
@ -213,7 +602,7 @@ CRC-32 would buy ~90 ms and cost an envelope-format bump. The remainder is
(~1085 ms), and the intern walk is not optional — dropping it regresses
retained heap ~59%, which is #2649's constraint.
### `emit` is the real target
#### `emit` is the real target
7.2s, 49% of the phase and ~21% of the whole edit loop. It is a fan of passes,
each walking every reference site in the repo:
@ -239,7 +628,7 @@ is a design, not a patch, and #2649 rules out "just cache the emitted edges".
---
## Measured and rejected
### Measured and rejected
Recorded because each cost real time to establish and each looks attractive
from the armchair.
@ -267,7 +656,7 @@ run (73.0s vs 69.6s on overlay). Measure on a local filesystem.
---
## Open
### Open at the September baseline
The post-pipeline tail is fully accounted (99.7%): FTS rebuild, the graph write,
and 0.8s of post-FTS event-loop drain between them.

View file

@ -0,0 +1,114 @@
{
"_what": "Go route-fact collection and serialized replay (#3750). Run node --expose-gc --import tsx bench/go-route-facts/measure.js --check.",
"_method": "Three 4x sizes; eight warmups, seven interleaved adaptive batches per size, median per-call time. Parsing, serialization and identity checks are untimed. manyParameters grows parameters and routes together. Serialized bytes exclude worker envelopes and heap.",
"_triage": "Check identities before timing. A normalized factor near 1 is linear; quadratic growth produces factors near 4 per adjacent size and 16 across the full range. Re-run timing failures on an idle machine; do not raise budgets or regenerate fingerprints to conceal regressions.",
"version": 1,
"sizes": [128, 512, 2048],
"budgets": {
"collect_ms": 1.5,
"replay_ms": 1.5,
"serialized_bytes": 1.15
},
"workloads": {
"noRoutes": [
{
"size": 128,
"routes": 0,
"candidates": 0,
"fingerprint": "95d423889096132b39fc71a1ac71377b51265e98194b64803edf5db1fe7bf7b1"
},
{
"size": 512,
"routes": 0,
"candidates": 0,
"fingerprint": "95d423889096132b39fc71a1ac71377b51265e98194b64803edf5db1fe7bf7b1"
},
{
"size": 2048,
"routes": 0,
"candidates": 0,
"fingerprint": "95d423889096132b39fc71a1ac71377b51265e98194b64803edf5db1fe7bf7b1"
}
],
"direct": [
{
"size": 128,
"routes": 128,
"candidates": 128,
"fingerprint": "1db15d356dbdbf66795f95bedf9c241ac47e9de19140e169cdabf12f4ba64771"
},
{
"size": 512,
"routes": 512,
"candidates": 512,
"fingerprint": "8450e2f92cf78ce784f1412f52dead89ef7b994285643eef7da0f12ba646b83e"
},
{
"size": 2048,
"routes": 2048,
"candidates": 2048,
"fingerprint": "1d568993ecb20a35e23c21ffd2c89e8a6cbb4279e05252d1293fe510d5c92cdf"
}
],
"wrapped": [
{
"size": 128,
"routes": 128,
"candidates": 128,
"fingerprint": "f687fcd72ed2b722e6fa2210761a2ae1a9d090db935ba526da65764b3397bb68"
},
{
"size": 512,
"routes": 512,
"candidates": 512,
"fingerprint": "be46370d5d280ab594de5099814be36f3faf2ddbabe997b7cddd0a8df8850e0b"
},
{
"size": 2048,
"routes": 2048,
"candidates": 2048,
"fingerprint": "a5728b94aed95cd572a76cc5c6c3515795fe5c84101fc852968271d728231545"
}
],
"mixed": [
{
"size": 128,
"routes": 128,
"candidates": 128,
"fingerprint": "ae5a80d0b61dd6afb781ce1d763766ba42b91310d5b229bf271ea49958765dd2"
},
{
"size": 512,
"routes": 512,
"candidates": 512,
"fingerprint": "f930a0c5fb25631ca12c471bd8793d3fe7b354e2fcfa53accc11ea298496cf4e"
},
{
"size": 2048,
"routes": 2048,
"candidates": 2048,
"fingerprint": "b365479d3bfee17800dd01d748ebfa7e5fda39b8d650b1c04bb8bb75e81975f4"
}
],
"manyParameters": [
{
"size": 128,
"routes": 128,
"candidates": 128,
"fingerprint": "7e39887e880c081eb858b5d4d43543ac9e79835434d93230f8ddda0dc3999984"
},
{
"size": 512,
"routes": 512,
"candidates": 512,
"fingerprint": "97afb59ccc0fcd32aaa569f97b8d9e6d190830b1c064b3bf9dfb17bfba30eedf"
},
{
"size": 2048,
"routes": 2048,
"candidates": 2048,
"fingerprint": "be631b09b19df157c595b057469cc2c72a2fc60fae7995e43657a81752a72b11"
}
]
}
}

View file

@ -0,0 +1,299 @@
#!/usr/bin/env node
/**
* Go route-fact collection and replay, using the production provider hooks.
* Run: node --expose-gc --import tsx bench/go-route-facts/measure.js --check
* Record a reviewed baseline: same command with --record instead of --check.
*
* Parsing, serialization and fingerprints are outside timing. Each invocation
* collects fresh facts or replays serialized facts with a fresh resolution
* context. Parameters AND registrations grow in the manyParameters workload.
* Adaptive batches avoid sub-millisecond timing ratios; sample order rotates
* across three 4x sizes. Check adjacent sizes and the full 16x range.
* Serialized bytes cover collected routes/facts, not worker envelopes or heap.
*/
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { readFileSync, writeFileSync } from 'node:fs';
import { performance } from 'node:perf_hooks';
import { parseArgs } from 'node:util';
import Parser from 'tree-sitter';
import Go from 'tree-sitter-go';
import { goProvider } from '../../src/core/ingestion/languages/go.ts';
const { values } = parseArgs({
options: {
check: { type: 'boolean' },
record: { type: 'boolean' },
baseline: { type: 'string' },
output: { type: 'string' },
},
});
assert(!(values.check && values.record), 'Choose --check or --record');
const baselinePath = values.baseline ?? new URL('./baseline.json', import.meta.url);
const shapes = ['noRoutes', 'direct', 'wrapped', 'mixed', 'manyParameters'];
const sizes = [128, 512, 2048];
const metrics = ['collect_ms', 'replay_ms', 'serialized_bytes'];
const budgets = { collect_ms: 1.5, replay_ms: 1.5, serialized_bytes: 1.15 };
const warmups = 8;
const samples = 7;
const batchMs = 40;
const filePath = 'routes.go';
const alias = { importPath: 'example.com/infra/pkg', type: 'Engine', framework: 'gin' };
const routeHookConfig = {
routeFrameworkAliases: [alias],
routeHandlerWrappers: ['common.Wrapper'],
};
const parsedFile = {
filePath,
moduleScope: 'bench',
scopes: [],
parsedImports: [],
localDefs: [],
referenceSites: [],
};
const expectedCount = (shape, size) => (shape === 'noRoutes' ? 0 : size);
const id = (i) => String(i).padStart(8, '0');
const lines = (size, line) => Array.from({ length: size }, (_, i) => line(i)).join('\n');
const median = (numbers) => [...numbers].sort((a, b) => a - b)[Math.floor(numbers.length / 2)];
function source(shape, size) {
if (shape === 'noRoutes') {
return `package routes\nimport "fmt"\n${lines(
size,
(i) => `func F${id(i)}(x int) int { fmt.Println(x); return x + 1 }`,
)}`;
}
const imports =
shape === 'direct'
? 'import "github.com/gin-gonic/gin"'
: shape === 'mixed'
? 'import "github.com/gin-gonic/gin"\nimport core "example.com/infra/pkg"'
: 'import core "example.com/infra/pkg"\nimport "example.com/app/common"';
const parameters =
shape === 'direct'
? 'r *gin.Engine'
: shape === 'mixed'
? 'r *gin.Engine, s *core.Engine'
: `r *core.Engine${
shape === 'manyParameters'
? `, ${Array.from({ length: size }, (_, i) => `x${id(i)} *core.T${id(i)}`).join(', ')}`
: ''
}`;
return `package routes\n${imports}\nfunc Register(${parameters}) {
g := r.Group("/api")
${shape === 'mixed' ? 'a := s.Group("/api")' : ''}
${lines(
size,
(i) =>
`${shape === 'mixed' && i % 2 ? 'a' : 'g'}.GET("/${id(i)}", Middleware, ${
shape === 'wrapped' ? `common.Wrapper(Handler${id(i)})` : `Handler${id(i)}`
})`,
)}
}`;
}
function collect(tree) {
return {
direct: goProvider.extractDecoratorRoutes?.(tree, filePath, 0) ?? [],
facts: goProvider.extractRouteFacts(tree, parsedFile, 0) ?? null,
};
}
function replay(collected) {
return [
...goProvider.resolveRouteFacts([{ filePath, facts: collected.facts }], {
resolveImportTargets: () => [],
routeHookConfig,
}),
...collected.direct,
];
}
function identity(shape, size, collected) {
const routes = replay(collected);
const count = expectedCount(shape, size);
assert.equal(routes.length, count, `${shape}/${size}: route count`);
assert.deepEqual(
routes.map((r) => r.routePath).sort(),
Array.from({ length: count }, (_, i) => `/api/${id(i)}`),
`${shape}/${size}: route paths`,
);
for (const route of routes) {
const handler = `Handler${route.routePath.slice(-8)}`;
assert.equal(route.httpMethod, 'GET');
assert.equal(route.source, 'gin-route');
assert.equal(route.filePath, filePath);
if (shape === 'wrapped') {
assert.deepEqual(route.handlerWrapper, { callee: 'common.Wrapper', handlerName: handler });
} else {
assert.equal(route.handlerName, handler);
}
}
const candidates = (collected.facts?.roots ?? []).reduce((n, root) => n + root.routes.length, 0);
assert.equal(candidates, count, `${shape}/${size}: candidate count`);
return {
size,
routes: routes.length,
candidates,
fingerprint: createHash('sha256').update(JSON.stringify({ collected, routes })).digest('hex'),
};
}
function validateBaseline(baseline) {
assert(baseline && typeof baseline === 'object' && !Array.isArray(baseline), 'Invalid baseline');
assert.equal(baseline.version, 1, 'Unknown baseline version');
assert.deepEqual(baseline.sizes, sizes, 'Baseline must retain all three sizes');
assert.deepEqual(baseline.budgets, budgets, 'Baseline must retain the reviewed scaling budgets');
assert.deepEqual(Object.keys(baseline.workloads).sort(), [...shapes].sort(), 'Missing workload');
for (const shape of shapes) {
const rows = baseline.workloads[shape];
assert(Array.isArray(rows) && rows.length === sizes.length, `${shape}: missing sizes`);
for (const [index, row] of rows.entries()) {
assert.deepEqual(Object.keys(row).sort(), ['candidates', 'fingerprint', 'routes', 'size']);
assert.equal(row.size, sizes[index]);
assert.equal(row.routes, expectedCount(shape, row.size));
assert.equal(row.candidates, row.routes);
assert.match(row.fingerprint, /^[a-f0-9]{64}$/);
}
}
}
function batch(action, iterations) {
let result;
const start = performance.now();
for (let i = 0; i < iterations; i++) result = action();
const elapsed = performance.now() - start;
assert(result !== undefined, 'Benchmark action returned no result');
return elapsed;
}
function calibrate(action) {
let iterations = 1;
while (iterations < 65536 && batch(action, iterations) < batchMs) iterations *= 2;
return iterations;
}
function scaling(rows) {
const pairs = [
[rows[0], rows[1]],
[rows[1], rows[2]],
[rows[0], rows[2]],
];
return pairs.map(([small, large]) => {
const inputRatio = large.size / small.size;
const factors = Object.fromEntries(
metrics.map((metric) => {
assert(Number.isFinite(small[metric]) && small[metric] > 0, `Invalid ${metric}`);
assert(Number.isFinite(large[metric]) && large[metric] > 0, `Invalid ${metric}`);
return [metric, large[metric] / small[metric] / inputRatio];
}),
);
return {
from: small.size,
to: large.size,
input_ratio: inputRatio,
factors,
passed: metrics.every((metric) => factors[metric] <= budgets[metric]),
};
});
}
const baseline = values.check ? JSON.parse(readFileSync(baselinePath, 'utf8')) : undefined;
if (values.check) validateBaseline(baseline);
assert.equal(typeof goProvider.extractRouteFacts, 'function');
assert.equal(typeof goProvider.resolveRouteFacts, 'function');
const parser = new Parser();
parser.setLanguage(Go);
const report = {
node: process.version,
warmups,
samples,
batch_ms: batchMs,
workloads: {},
errors: [],
};
const recorded = {};
for (const shape of shapes) {
const states = sizes.map((size) => {
const tree = parser.parse(source(shape, size));
assert(!tree.rootNode.hasError, `${shape}/${size}: invalid Go fixture`);
// The replay input survives JSON serialization, just like cached worker facts.
const collected = JSON.parse(JSON.stringify(collect(tree)));
const actual = identity(shape, size, collected);
if (values.check)
assert.deepEqual(
actual,
baseline.workloads[shape][sizes.indexOf(size)],
`${shape}/${size}: identity drift`,
);
const actions = { collect_ms: () => collect(tree), replay_ms: () => replay(collected) };
for (let i = 0; i < warmups; i++) for (const action of Object.values(actions)) action();
return {
size,
actual,
actions,
collected,
timings: { collect_ms: [], replay_ms: [] },
iterations: Object.fromEntries(
Object.entries(actions).map(([key, action]) => [key, calibrate(action)]),
),
};
});
for (let round = 0; round < samples; round++) {
for (let offset = 0; offset < states.length; offset++) {
const state = states[(round + offset) % states.length];
for (const [metric, action] of Object.entries(state.actions)) {
global.gc?.();
state.timings[metric].push(
batch(action, state.iterations[metric]) / state.iterations[metric],
);
}
}
}
const rows = states.map((state) => ({
...state.actual,
collect_ms: median(state.timings.collect_ms),
replay_ms: median(state.timings.replay_ms),
serialized_bytes: Buffer.byteLength(JSON.stringify(state.collected)),
iterations: state.iterations,
samples_ms: state.timings,
}));
const comparisons = scaling(rows);
for (const comparison of comparisons) {
for (const metric of metrics)
if (comparison.factors[metric] > budgets[metric]) {
report.errors.push(
`${shape} ${comparison.from}->${comparison.to}: ${metric} linear factor ${comparison.factors[metric].toFixed(3)} exceeds ${budgets[metric]}`,
);
}
}
report.workloads[shape] = { rows, comparisons };
recorded[shape] = states.map((state) => state.actual);
}
report.passed = report.errors.length === 0;
const json = `${JSON.stringify(report, null, 2)}\n`;
if (values.output) writeFileSync(values.output, json);
console.log(json);
if (values.record) {
assert(report.passed, 'Refusing to record a baseline that fails scaling');
writeFileSync(
baselinePath,
`${JSON.stringify(
{
_what:
'Go route-fact collection and serialized replay (#3750). Run node --expose-gc --import tsx bench/go-route-facts/measure.js --check.',
_method:
'Three 4x sizes; eight warmups, seven interleaved adaptive batches per size, median per-call time. Parsing, serialization and identity checks are untimed. manyParameters grows parameters and routes together. Serialized bytes exclude worker envelopes and heap.',
_triage:
'Check identities before timing. A normalized factor near 1 is linear; quadratic growth produces factors near 4 per adjacent size and 16 across the full range. Re-run timing failures on an idle machine; do not raise budgets or regenerate fingerprints to conceal regressions.',
version: 1,
sizes,
budgets,
workloads: recorded,
},
null,
2,
)}\n`,
);
}
if (values.check && !report.passed) process.exitCode = 1;

View file

@ -0,0 +1,67 @@
# Java overload and ingestion scaling
Build the package, then measure the current implementation:
```sh
npm run build
node bench/java-overload-resolution/measure.js --check --output /tmp/java-overload-check.json
```
CI runs this check in the scope-resolution benchmark job and uploads
`java-overload-resolution-benchmark.json` with that job's measurements.
`measure.js` runs three fresh-process samples at 128, 512 and 2,048 units. Each
unit adds seven methods and four calls. Two shapes exercise increasing file
counts and increasing declarations within one class. Every overload family has
two candidates; package width and inheritance depth stay fixed. The spread
shape uses separate packages with an explicit import so package-size limits
cannot silently reduce resolution work.
Each sample runs real worker parsing, scope resolution, graph construction and
database writes, with a cold cache, a warm cache and one edited source file.
The edit casts one argument to `Object`, changing exactly one selected overload.
The calls cover reference specificity, widening before boxing, `var` receiver
inference through a factory call and cross-file member resolution. Expected target IDs
are generated independently of the implementation. Missing methods or calls,
incorrect targets, suppressed calls, extraction failures, incorrect cache reuse
and truncated process detection all fail the run.
The harness uses one parser worker, private temporary storage, a 128 MiB database
buffer pool, and no FTS or embeddings. It raises the file-size and process limits
to include the entire generated corpus. Each scenario forces graph reconstruction
and persistence; warm and changed scenarios reuse parse-cache chunks. Editing one
file can reparse other files in its cache chunk. This measures full graph writes,
not the separate selective database-update path.
The report includes raw samples, median wall time and process CPU time for the
whole analysis and the scope-resolution phase, main-thread heap at completion,
and process peak RSS (including workers/native memory). Peak RSS is cumulative
across the three scenarios in each child. Heap is sampled before an explicit
post-analysis collection. The built JavaScript digest identifies the measured
runtime and detects changes to it during the run.
`--check` verifies the committed corpus, call-target and cache fingerprints, then
gates every adjacent size pair and the full span. For every time and memory
metric, `(large cost / small cost) / (large size / small size)` must be at most
**1.5**. The shared gate and its quadratic-growth tests come from
`../method-declaration-facts/scaling.mjs`. Absolute baseline timings are reference
measurements, not machine-independent thresholds.
These are bounded scaling checks, not a proof of linear complexity for arbitrary
Java programs. In particular, this corpus does not increase overload-family
width, inheritance depth or generic constraint complexity. Startup and database
costs can hide growth at small sizes; the separate scope measurements and
deterministic lookup-work regression tests help expose it.
After reviewing an intentional workload or semantic change, record a baseline:
```sh
node bench/java-overload-resolution/measure.js --record --output /tmp/java-overload-record.json
```
Recording also requires all scaling and correctness checks to pass, uses at least
three samples, and is disabled in CI. `--root /path/to/built/gitnexus` selects an
isolated runtime. `--sizes 128,512,2048 --samples 1` is useful for diagnostic runs;
one-sample runs cannot record or check a baseline. Completed samples are saved
after each child so a later failure leaves usable diagnostic evidence.
The output path must not already exist; existing files and symlinks are rejected.

View file

@ -0,0 +1,926 @@
{
"_note": "Measured baseline. --check gates same-run normalized growth at 1.5 and exact call targets/cache counts. Absolute timings and memory are references, not portable budgets.",
"schemaVersion": 1,
"recordedAt": "2026-10-31T19:46:00.972Z",
"runtimeSha256": "4b3e97e7ebb9f21e6aa4048f584fe10fd5ac049d5a4b60488947f45b01fc2057",
"workload": {
"sizes": [128, 512, 2048],
"shapes": ["spread", "concentrated"],
"scenarios": ["cold", "warm", "changed"],
"overloadsPerFamily": 2,
"callsPerUnit": 4,
"workers": 1
},
"environment": {
"node": "v22.23.3",
"platform": "linux",
"arch": "x64",
"cpu": "12th Gen Intel(R) Core(TM) i9-12900K"
},
"correctness": [
{
"shape": "spread",
"size": 128,
"scenario": "cold",
"corpus": "ba00fdce998f6c93ded7b9e48249deb2195f435cd9f498d2585c028a4745b18c",
"correctness": {
"files": 129,
"methods": 898,
"calls": 512,
"fingerprint": "9c7584e6908a263beed06089876413d7d73a0d787d1eff7681dde0dd26f7b672",
"reparsed": 129,
"hits": 0
}
},
{
"shape": "spread",
"size": 128,
"scenario": "warm",
"corpus": "ba00fdce998f6c93ded7b9e48249deb2195f435cd9f498d2585c028a4745b18c",
"correctness": {
"files": 129,
"methods": 898,
"calls": 512,
"fingerprint": "9c7584e6908a263beed06089876413d7d73a0d787d1eff7681dde0dd26f7b672",
"reparsed": 0,
"hits": 129
}
},
{
"shape": "spread",
"size": 128,
"scenario": "changed",
"corpus": "ba00fdce998f6c93ded7b9e48249deb2195f435cd9f498d2585c028a4745b18c",
"correctness": {
"files": 129,
"methods": 898,
"calls": 512,
"fingerprint": "def7d18a8b462dcf231bd2192afebe764911163d3ffc65e75fe5fefeb7e3a54c",
"reparsed": 2,
"hits": 127
}
},
{
"shape": "spread",
"size": 512,
"scenario": "cold",
"corpus": "dd070e75fcba816f5775dff5efbfdd3fdda28e4b01d0c1603fd9e294587896c7",
"correctness": {
"files": 513,
"methods": 3586,
"calls": 2048,
"fingerprint": "f126c9b94d15f6c85f55e802db43430abbb378a6e82a78492b3d6e8fcc18a69b",
"reparsed": 513,
"hits": 0
}
},
{
"shape": "spread",
"size": 512,
"scenario": "warm",
"corpus": "dd070e75fcba816f5775dff5efbfdd3fdda28e4b01d0c1603fd9e294587896c7",
"correctness": {
"files": 513,
"methods": 3586,
"calls": 2048,
"fingerprint": "f126c9b94d15f6c85f55e802db43430abbb378a6e82a78492b3d6e8fcc18a69b",
"reparsed": 0,
"hits": 513
}
},
{
"shape": "spread",
"size": 512,
"scenario": "changed",
"corpus": "dd070e75fcba816f5775dff5efbfdd3fdda28e4b01d0c1603fd9e294587896c7",
"correctness": {
"files": 513,
"methods": 3586,
"calls": 2048,
"fingerprint": "360cdbc01738746de63cca79adba7410797749c171d3e324579a7a53deee56d0",
"reparsed": 5,
"hits": 508
}
},
{
"shape": "spread",
"size": 2048,
"scenario": "cold",
"corpus": "5566903502880e054a6e2779f37d4c71da7333cfef65fb51e60d7ce3d03f8c77",
"correctness": {
"files": 2049,
"methods": 14338,
"calls": 8192,
"fingerprint": "9c07a075de05a15a73b0649a9fb45b730e6a6bf68d5152a315e5fae4fd1365a7",
"reparsed": 2049,
"hits": 0
}
},
{
"shape": "spread",
"size": 2048,
"scenario": "warm",
"corpus": "5566903502880e054a6e2779f37d4c71da7333cfef65fb51e60d7ce3d03f8c77",
"correctness": {
"files": 2049,
"methods": 14338,
"calls": 8192,
"fingerprint": "9c07a075de05a15a73b0649a9fb45b730e6a6bf68d5152a315e5fae4fd1365a7",
"reparsed": 0,
"hits": 2049
}
},
{
"shape": "spread",
"size": 2048,
"scenario": "changed",
"corpus": "5566903502880e054a6e2779f37d4c71da7333cfef65fb51e60d7ce3d03f8c77",
"correctness": {
"files": 2049,
"methods": 14338,
"calls": 8192,
"fingerprint": "8b2269dfa3d017f04e05f86afd8f6cac6364c1228acab0507de5287feb249969",
"reparsed": 15,
"hits": 2034
}
},
{
"shape": "concentrated",
"size": 128,
"scenario": "cold",
"corpus": "4043c0f69f54bcb6409bfeb70b974c2aa82b373d28f92f294a60c8199a8272fb",
"correctness": {
"files": 2,
"methods": 898,
"calls": 512,
"fingerprint": "f2f037938ed3b85bfac7ea91c8460bcef0bea3b73c38b84630e2ac36d8185b5d",
"reparsed": 2,
"hits": 0
}
},
{
"shape": "concentrated",
"size": 128,
"scenario": "warm",
"corpus": "4043c0f69f54bcb6409bfeb70b974c2aa82b373d28f92f294a60c8199a8272fb",
"correctness": {
"files": 2,
"methods": 898,
"calls": 512,
"fingerprint": "f2f037938ed3b85bfac7ea91c8460bcef0bea3b73c38b84630e2ac36d8185b5d",
"reparsed": 0,
"hits": 2
}
},
{
"shape": "concentrated",
"size": 128,
"scenario": "changed",
"corpus": "4043c0f69f54bcb6409bfeb70b974c2aa82b373d28f92f294a60c8199a8272fb",
"correctness": {
"files": 2,
"methods": 898,
"calls": 512,
"fingerprint": "5e4d7c334a4a1f6de8b594ae62d0792bd5895b828893103af8bbc4789f1f11fb",
"reparsed": 1,
"hits": 1
}
},
{
"shape": "concentrated",
"size": 512,
"scenario": "cold",
"corpus": "d6c830c77f6515d7e9157bc48d96a2bfa3ec3b7c81e315c61738ae5a5079a579",
"correctness": {
"files": 2,
"methods": 3586,
"calls": 2048,
"fingerprint": "e1e74e9da813db36ccea611bf2374dbcc079b4dc29466d5da127879d08cdaf00",
"reparsed": 2,
"hits": 0
}
},
{
"shape": "concentrated",
"size": 512,
"scenario": "warm",
"corpus": "d6c830c77f6515d7e9157bc48d96a2bfa3ec3b7c81e315c61738ae5a5079a579",
"correctness": {
"files": 2,
"methods": 3586,
"calls": 2048,
"fingerprint": "e1e74e9da813db36ccea611bf2374dbcc079b4dc29466d5da127879d08cdaf00",
"reparsed": 0,
"hits": 2
}
},
{
"shape": "concentrated",
"size": 512,
"scenario": "changed",
"corpus": "d6c830c77f6515d7e9157bc48d96a2bfa3ec3b7c81e315c61738ae5a5079a579",
"correctness": {
"files": 2,
"methods": 3586,
"calls": 2048,
"fingerprint": "c9beb4f12867432b2e53a6e52d337b39dfc0b05e1a18939575579157b425d29f",
"reparsed": 1,
"hits": 1
}
},
{
"shape": "concentrated",
"size": 2048,
"scenario": "cold",
"corpus": "b0698581ac113f8b46b4cc13cc1a8744b37aa41945e40245423eef383ca296b3",
"correctness": {
"files": 2,
"methods": 14338,
"calls": 8192,
"fingerprint": "7f349aefb5d96c2d46cda31484a7d696e26c4e55253336625d001462799b0de6",
"reparsed": 2,
"hits": 0
}
},
{
"shape": "concentrated",
"size": 2048,
"scenario": "warm",
"corpus": "b0698581ac113f8b46b4cc13cc1a8744b37aa41945e40245423eef383ca296b3",
"correctness": {
"files": 2,
"methods": 14338,
"calls": 8192,
"fingerprint": "7f349aefb5d96c2d46cda31484a7d696e26c4e55253336625d001462799b0de6",
"reparsed": 0,
"hits": 2
}
},
{
"shape": "concentrated",
"size": 2048,
"scenario": "changed",
"corpus": "b0698581ac113f8b46b4cc13cc1a8744b37aa41945e40245423eef383ca296b3",
"correctness": {
"files": 2,
"methods": 14338,
"calls": 8192,
"fingerprint": "1786baf149b46e933b59c54f2db3f6b3e1cb4d6e356497029546cd1afbe738fb",
"reparsed": 1,
"hits": 1
}
}
],
"measured": [
{
"shape": "spread",
"size": 128,
"scenario": "cold",
"corpus": "ba00fdce998f6c93ded7b9e48249deb2195f435cd9f498d2585c028a4745b18c",
"correctness": {
"files": 129,
"methods": 898,
"calls": 512,
"fingerprint": "9c7584e6908a263beed06089876413d7d73a0d787d1eff7681dde0dd26f7b672",
"reparsed": 129,
"hits": 0
},
"wallMs": 6382.313993999999,
"cpuMs": 6573.025,
"scopeWallMs": 265.60164499999973,
"scopeCpuMs": 422.705,
"heapBytes": 72082216,
"peakRssBytes": 563265536
},
{
"shape": "spread",
"size": 128,
"scenario": "warm",
"corpus": "ba00fdce998f6c93ded7b9e48249deb2195f435cd9f498d2585c028a4745b18c",
"correctness": {
"files": 129,
"methods": 898,
"calls": 512,
"fingerprint": "9c7584e6908a263beed06089876413d7d73a0d787d1eff7681dde0dd26f7b672",
"reparsed": 0,
"hits": 129
},
"wallMs": 2412.3159590000005,
"cpuMs": 2055.392,
"scopeWallMs": 236.99802499999987,
"scopeCpuMs": 423.928,
"heapBytes": 106224584,
"peakRssBytes": 693342208
},
{
"shape": "spread",
"size": 128,
"scenario": "changed",
"corpus": "ba00fdce998f6c93ded7b9e48249deb2195f435cd9f498d2585c028a4745b18c",
"correctness": {
"files": 129,
"methods": 898,
"calls": 512,
"fingerprint": "def7d18a8b462dcf231bd2192afebe764911163d3ffc65e75fe5fefeb7e3a54c",
"reparsed": 2,
"hits": 127
},
"wallMs": 3324.9829769999997,
"cpuMs": 2986.027,
"scopeWallMs": 279.4751209999995,
"scopeCpuMs": 570.481,
"heapBytes": 74642032,
"peakRssBytes": 747954176
},
{
"shape": "spread",
"size": 512,
"scenario": "cold",
"corpus": "dd070e75fcba816f5775dff5efbfdd3fdda28e4b01d0c1603fd9e294587896c7",
"correctness": {
"files": 513,
"methods": 3586,
"calls": 2048,
"fingerprint": "f126c9b94d15f6c85f55e802db43430abbb378a6e82a78492b3d6e8fcc18a69b",
"reparsed": 513,
"hits": 0
},
"wallMs": 9657.072656,
"cpuMs": 11590.296,
"scopeWallMs": 787.7755969999998,
"scopeCpuMs": 1342.358,
"heapBytes": 83321312,
"peakRssBytes": 641908736
},
{
"shape": "spread",
"size": 512,
"scenario": "warm",
"corpus": "dd070e75fcba816f5775dff5efbfdd3fdda28e4b01d0c1603fd9e294587896c7",
"correctness": {
"files": 513,
"methods": 3586,
"calls": 2048,
"fingerprint": "f126c9b94d15f6c85f55e802db43430abbb378a6e82a78492b3d6e8fcc18a69b",
"reparsed": 0,
"hits": 513
},
"wallMs": 4034.202557999999,
"cpuMs": 4545.678,
"scopeWallMs": 876.6461720000007,
"scopeCpuMs": 1818.994,
"heapBytes": 140587192,
"peakRssBytes": 788180992
},
{
"shape": "spread",
"size": 512,
"scenario": "changed",
"corpus": "dd070e75fcba816f5775dff5efbfdd3fdda28e4b01d0c1603fd9e294587896c7",
"correctness": {
"files": 513,
"methods": 3586,
"calls": 2048,
"fingerprint": "360cdbc01738746de63cca79adba7410797749c171d3e324579a7a53deee56d0",
"reparsed": 5,
"hits": 508
},
"wallMs": 4542.365702000001,
"cpuMs": 4986.714,
"scopeWallMs": 745.671574,
"scopeCpuMs": 1462.018,
"heapBytes": 141094984,
"peakRssBytes": 897118208
},
{
"shape": "spread",
"size": 2048,
"scenario": "cold",
"corpus": "5566903502880e054a6e2779f37d4c71da7333cfef65fb51e60d7ce3d03f8c77",
"correctness": {
"files": 2049,
"methods": 14338,
"calls": 8192,
"fingerprint": "9c07a075de05a15a73b0649a9fb45b730e6a6bf68d5152a315e5fae4fd1365a7",
"reparsed": 2049,
"hits": 0
},
"wallMs": 22421.167622,
"cpuMs": 28309.111,
"scopeWallMs": 2729.000151,
"scopeCpuMs": 4387.326,
"heapBytes": 110154224,
"peakRssBytes": 760012800
},
{
"shape": "spread",
"size": 2048,
"scenario": "warm",
"corpus": "5566903502880e054a6e2779f37d4c71da7333cfef65fb51e60d7ce3d03f8c77",
"correctness": {
"files": 2049,
"methods": 14338,
"calls": 8192,
"fingerprint": "9c07a075de05a15a73b0649a9fb45b730e6a6bf68d5152a315e5fae4fd1365a7",
"reparsed": 0,
"hits": 2049
},
"wallMs": 7275.022228999998,
"cpuMs": 9205.167,
"scopeWallMs": 2452.0143179999977,
"scopeCpuMs": 4076.376,
"heapBytes": 216411400,
"peakRssBytes": 931454976
},
{
"shape": "spread",
"size": 2048,
"scenario": "changed",
"corpus": "5566903502880e054a6e2779f37d4c71da7333cfef65fb51e60d7ce3d03f8c77",
"correctness": {
"files": 2049,
"methods": 14338,
"calls": 8192,
"fingerprint": "8b2269dfa3d017f04e05f86afd8f6cac6364c1228acab0507de5287feb249969",
"reparsed": 15,
"hits": 2034
},
"wallMs": 8042.547480000001,
"cpuMs": 10680.432,
"scopeWallMs": 2782.9499240000005,
"scopeCpuMs": 4797.166,
"heapBytes": 221611264,
"peakRssBytes": 1018769408
},
{
"shape": "concentrated",
"size": 128,
"scenario": "cold",
"corpus": "4043c0f69f54bcb6409bfeb70b974c2aa82b373d28f92f294a60c8199a8272fb",
"correctness": {
"files": 2,
"methods": 898,
"calls": 512,
"fingerprint": "f2f037938ed3b85bfac7ea91c8460bcef0bea3b73c38b84630e2ac36d8185b5d",
"reparsed": 2,
"hits": 0
},
"wallMs": 5724.462823000001,
"cpuMs": 5990.56,
"scopeWallMs": 216.9987230000006,
"scopeCpuMs": 359.581,
"heapBytes": 66736864,
"peakRssBytes": 549969920
},
{
"shape": "concentrated",
"size": 128,
"scenario": "warm",
"corpus": "4043c0f69f54bcb6409bfeb70b974c2aa82b373d28f92f294a60c8199a8272fb",
"correctness": {
"files": 2,
"methods": 898,
"calls": 512,
"fingerprint": "f2f037938ed3b85bfac7ea91c8460bcef0bea3b73c38b84630e2ac36d8185b5d",
"reparsed": 0,
"hits": 2
},
"wallMs": 2307.437070000001,
"cpuMs": 1697.642,
"scopeWallMs": 165.59543699999995,
"scopeCpuMs": 366.701,
"heapBytes": 89981272,
"peakRssBytes": 673837056
},
{
"shape": "concentrated",
"size": 128,
"scenario": "changed",
"corpus": "4043c0f69f54bcb6409bfeb70b974c2aa82b373d28f92f294a60c8199a8272fb",
"correctness": {
"files": 2,
"methods": 898,
"calls": 512,
"fingerprint": "5e4d7c334a4a1f6de8b594ae62d0792bd5895b828893103af8bbc4789f1f11fb",
"reparsed": 1,
"hits": 1
},
"wallMs": 3410.6907119999996,
"cpuMs": 3338.962,
"scopeWallMs": 166.35203,
"scopeCpuMs": 366.192,
"heapBytes": 69753416,
"peakRssBytes": 766402560
},
{
"shape": "concentrated",
"size": 512,
"scenario": "cold",
"corpus": "d6c830c77f6515d7e9157bc48d96a2bfa3ec3b7c81e315c61738ae5a5079a579",
"correctness": {
"files": 2,
"methods": 3586,
"calls": 2048,
"fingerprint": "e1e74e9da813db36ccea611bf2374dbcc079b4dc29466d5da127879d08cdaf00",
"reparsed": 2,
"hits": 0
},
"wallMs": 9268.474702000001,
"cpuMs": 10475.483,
"scopeWallMs": 625.0715480000008,
"scopeCpuMs": 1136.577,
"heapBytes": 116100256,
"peakRssBytes": 584364032
},
{
"shape": "concentrated",
"size": 512,
"scenario": "warm",
"corpus": "d6c830c77f6515d7e9157bc48d96a2bfa3ec3b7c81e315c61738ae5a5079a579",
"correctness": {
"files": 2,
"methods": 3586,
"calls": 2048,
"fingerprint": "e1e74e9da813db36ccea611bf2374dbcc079b4dc29466d5da127879d08cdaf00",
"reparsed": 0,
"hits": 2
},
"wallMs": 2986.604102000001,
"cpuMs": 3221.225,
"scopeWallMs": 582.4847769999997,
"scopeCpuMs": 1302.585,
"heapBytes": 115896680,
"peakRssBytes": 776753152
},
{
"shape": "concentrated",
"size": 512,
"scenario": "changed",
"corpus": "d6c830c77f6515d7e9157bc48d96a2bfa3ec3b7c81e315c61738ae5a5079a579",
"correctness": {
"files": 2,
"methods": 3586,
"calls": 2048,
"fingerprint": "c9beb4f12867432b2e53a6e52d337b39dfc0b05e1a18939575579157b425d29f",
"reparsed": 1,
"hits": 1
},
"wallMs": 6360.436007999999,
"cpuMs": 7743.869,
"scopeWallMs": 510.34782500000074,
"scopeCpuMs": 1284.846,
"heapBytes": 123689144,
"peakRssBytes": 958959616
},
{
"shape": "concentrated",
"size": 2048,
"scenario": "cold",
"corpus": "b0698581ac113f8b46b4cc13cc1a8744b37aa41945e40245423eef383ca296b3",
"correctness": {
"files": 2,
"methods": 14338,
"calls": 8192,
"fingerprint": "7f349aefb5d96c2d46cda31484a7d696e26c4e55253336625d001462799b0de6",
"reparsed": 2,
"hits": 0
},
"wallMs": 21677.11593,
"cpuMs": 25262.452,
"scopeWallMs": 2939.4483809999983,
"scopeCpuMs": 4160.173,
"heapBytes": 97430168,
"peakRssBytes": 883941376
},
{
"shape": "concentrated",
"size": 2048,
"scenario": "warm",
"corpus": "b0698581ac113f8b46b4cc13cc1a8744b37aa41945e40245423eef383ca296b3",
"correctness": {
"files": 2,
"methods": 14338,
"calls": 8192,
"fingerprint": "7f349aefb5d96c2d46cda31484a7d696e26c4e55253336625d001462799b0de6",
"reparsed": 0,
"hits": 2
},
"wallMs": 4958.608856999999,
"cpuMs": 6291.745,
"scopeWallMs": 2304.6741980000006,
"scopeCpuMs": 3572.083,
"heapBytes": 123585896,
"peakRssBytes": 1016422400
},
{
"shape": "concentrated",
"size": 2048,
"scenario": "changed",
"corpus": "b0698581ac113f8b46b4cc13cc1a8744b37aa41945e40245423eef383ca296b3",
"correctness": {
"files": 2,
"methods": 14338,
"calls": 8192,
"fingerprint": "1786baf149b46e933b59c54f2db3f6b3e1cb4d6e356497029546cd1afbe738fb",
"reparsed": 1,
"hits": 1
},
"wallMs": 18368.502542000002,
"cpuMs": 22049.709,
"scopeWallMs": 2615.8797609999965,
"scopeCpuMs": 4184.537,
"heapBytes": 126797336,
"peakRssBytes": 1626734592
}
],
"scaling": {
"spread/cold": {
"budget": 1.5,
"comparisons": [
{
"from": 128,
"to": 512,
"inputRatio": 4,
"ratios": {
"wallMs": 0.37827473958029156,
"cpuMs": 0.44082808143891133,
"scopeWallMs": 0.7415010522619322,
"scopeCpuMs": 0.7939094640470304,
"heapBytes": 0.2889801279139365,
"peakRssBytes": 0.28490502923296196
},
"passed": true
},
{
"from": 512,
"to": 2048,
"inputRatio": 4,
"ratios": {
"wallMs": 0.5804338545612368,
"cpuMs": 0.6106209668847111,
"scopeWallMs": 0.8660461689193454,
"scopeCpuMs": 0.8170931301485893,
"heapBytes": 0.33051035010106417,
"peakRssBytes": 0.29599721789734296
},
"passed": true
},
{
"from": 128,
"to": 2048,
"inputRatio": 16,
"ratios": {
"wallMs": 0.21956346517773664,
"cpuMs": 0.2691788693181602,
"scopeWallMs": 0.6421741455611097,
"scopeCpuMs": 0.648697969032777,
"heapBytes": 0.09551092324908546,
"peakRssBytes": 0.08433109601791791
},
"passed": true
}
],
"passed": true
},
"spread/warm": {
"budget": 1.5,
"comparisons": [
{
"from": 128,
"to": 512,
"inputRatio": 4,
"ratios": {
"wallMs": 0.41808397268079406,
"cpuMs": 0.5528967223770453,
"scopeWallMs": 0.9247399551114415,
"scopeCpuMs": 1.0727022041478742,
"heapBytes": 0.33087254076702244,
"peakRssBytes": 0.284196239211215
},
"passed": true
},
{
"from": 512,
"to": 2048,
"inputRatio": 4,
"ratios": {
"wallMs": 0.45083397055592267,
"cpuMs": 0.506259297292945,
"scopeWallMs": 0.6992599740685332,
"scopeCpuMs": 0.5602514356836802,
"heapBytes": 0.38483484327647716,
"peakRssBytes": 0.29544450622833596
},
"passed": true
},
{
"from": 128,
"to": 2048,
"inputRatio": 16,
"ratios": {
"wallMs": 0.1884864574294763,
"cpuMs": 0.27990910614617553,
"scopeWallMs": 0.6466336370313632,
"scopeCpuMs": 0.6009829499348947,
"heapBytes": 0.12733128237056687,
"peakRssBytes": 0.08396421756570746
},
"passed": true
}
],
"passed": true
},
"spread/changed": {
"budget": 1.5,
"comparisons": [
{
"from": 128,
"to": 512,
"inputRatio": 4,
"ratios": {
"wallMs": 0.3415330043357393,
"cpuMs": 0.4175040949060407,
"scopeWallMs": 0.6670285814099364,
"scopeCpuMs": 0.6406953079944818,
"heapBytes": 0.47257215612779674,
"peakRssBytes": 0.29985734313220813
},
"passed": true
},
{
"from": 512,
"to": 2048,
"inputRatio": 4,
"ratios": {
"wallMs": 0.44264090606238904,
"cpuMs": 0.5354443828140134,
"scopeWallMs": 0.9330347370865449,
"scopeCpuMs": 0.8202987240923163,
"heapBytes": 0.3926632572565443,
"peakRssBytes": 0.28390054925738395
},
"passed": true
},
{
"from": 128,
"to": 2048,
"inputRatio": 16,
"ratios": {
"wallMs": 0.15117647848938148,
"cpuMs": 0.22355022241928824,
"scopeWallMs": 0.6223608370850311,
"scopeCpuMs": 0.525561543679807,
"heapBytes": 0.18556172211388886,
"peakRssBytes": 0.08512966441409373
},
"passed": true
}
],
"passed": true
},
"concentrated/cold": {
"budget": 1.5,
"comparisons": [
{
"from": 128,
"to": 512,
"inputRatio": 4,
"ratios": {
"wallMs": 0.4047748665936266,
"cpuMs": 0.43716626659277263,
"scopeWallMs": 0.7201327493526298,
"scopeCpuMs": 0.7902092991565183,
"heapBytes": 0.43491800873352393,
"peakRssBytes": 0.26563454234006106
},
"passed": true
},
{
"from": 512,
"to": 2048,
"inputRatio": 4,
"ratios": {
"wallMs": 0.5847001968220941,
"cpuMs": 0.6028946827559168,
"scopeWallMs": 1.1756447683489806,
"scopeCpuMs": 0.9150662471614329,
"heapBytes": 0.20979748744051005,
"peakRssBytes": 0.37816383606580356
},
"passed": true
},
{
"from": 128,
"to": 2048,
"inputRatio": 16,
"ratios": {
"wallMs": 0.23667194416593032,
"cpuMs": 0.2635652176090382,
"scopeWallMs": 0.8466202992931869,
"scopeCpuMs": 0.7230938578512212,
"heapBytes": 0.09124470547492312,
"peakRssBytes": 0.10045337752290162
},
"passed": true
}
],
"passed": true
},
"concentrated/warm": {
"budget": 1.5,
"comparisons": [
{
"from": 128,
"to": 512,
"inputRatio": 4,
"ratios": {
"wallMs": 0.3235845671405461,
"cpuMs": 0.4743675344978505,
"scopeWallMs": 0.87937926846378,
"scopeCpuMs": 0.8880429832479322,
"heapBytes": 0.3220022273079225,
"peakRssBytes": 0.2881828570733872
},
"passed": true
},
{
"from": 512,
"to": 2048,
"inputRatio": 4,
"ratios": {
"wallMs": 0.4150708202067551,
"cpuMs": 0.4883037509022189,
"scopeWallMs": 0.9891564076017054,
"scopeCpuMs": 0.6855757973567944,
"heapBytes": 0.2665863595057253,
"peakRssBytes": 0.32713816396589274
},
"passed": true
},
{
"from": 128,
"to": 2048,
"inputRatio": 16,
"ratios": {
"wallMs": 0.1343105116892743,
"cpuMs": 0.23163544640153813,
"scopeWallMs": 0.8698436381130483,
"scopeCpuMs": 0.6088207763273075,
"heapBytes": 0.08584140153075409,
"peakRssBytes": 0.09427561074943316
},
"passed": true
}
],
"passed": true
},
"concentrated/changed": {
"budget": 1.5,
"comparisons": [
{
"from": 128,
"to": 512,
"inputRatio": 4,
"ratios": {
"wallMs": 0.4662131914821363,
"cpuMs": 0.579811105966465,
"scopeWallMs": 0.766969638122241,
"scopeCpuMs": 0.8771668960545288,
"heapBytes": 0.4433085542362542,
"peakRssBytes": 0.3128119822564267
},
"passed": true
},
{
"from": 512,
"to": 2048,
"inputRatio": 4,
"ratios": {
"wallMs": 0.7219828372967103,
"cpuMs": 0.7118440730337767,
"scopeWallMs": 1.281420059446708,
"scopeCpuMs": 0.8142098352643041,
"heapBytes": 0.25628226515982677,
"peakRssBytes": 0.42408839873398796
},
"passed": true
},
{
"from": 128,
"to": 2048,
"inputRatio": 16,
"ratios": {
"wallMs": 0.33659792277142725,
"cpuMs": 0.4127350992613872,
"scopeWallMs": 0.9828102792764222,
"scopeCpuMs": 0.7141979139358589,
"heapBytes": 0.11361212044439516,
"peakRssBytes": 0.13265993265993267
},
"passed": true
}
],
"passed": true
}
}
}

View file

@ -0,0 +1,382 @@
#!/usr/bin/env node
/** Build first. Each sample measures real worker parsing, resolution and DB writes. */
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { parseArgs } from 'node:util';
import { evaluateScaling } from '../method-declaration-facts/scaling.mjs';
const script = fileURLToPath(import.meta.url);
const baselinePath = new URL('./baseline.json', import.meta.url);
const { values } = parseArgs({
options: {
check: { type: 'boolean' },
record: { type: 'boolean' },
root: { type: 'string', default: path.resolve(path.dirname(script), '../..') },
sizes: { type: 'string', default: '128,512,2048' },
samples: { type: 'string', default: '3' },
output: { type: 'string' },
child: { type: 'string' },
size: { type: 'string' },
},
});
const root = path.resolve(values.root);
const load = (name) => import(pathToFileURL(path.join(root, 'dist', name)));
const hash = (rows) => createHash('sha256').update(JSON.stringify(rows)).digest('hex');
const marker = 'JAVA_OVERLOAD_BENCH=';
const shapes = ['spread', 'concentrated'];
const scenarios = ['cold', 'warm', 'changed'];
const metrics = ['wallMs', 'cpuMs', 'scopeWallMs', 'scopeCpuMs', 'heapBytes', 'peakRssBytes'];
const median = (numbers) => [...numbers].sort((a, b) => a - b)[Math.floor(numbers.length / 2)];
async function runtimeFingerprint() {
const digest = createHash('sha256');
const dist = path.join(root, 'dist');
for (const file of (await readdir(dist, { recursive: true }))
.filter((file) => file.endsWith('.js'))
.sort()) {
digest.update(file.replaceAll(path.sep, '/')).update('\0');
digest.update(await readFile(path.join(dist, file))).update('\0');
}
return digest.digest('hex');
}
function corpus(shape, size, changed = false) {
const files = new Map([
[
'Product.java',
`package bench;
public class Product {
public void accept(Object value) {}
public void accept(String value) {}
}
`,
],
]);
const expected = [];
const declarations = [];
for (let i = 0; i < size; i++) {
const owner = shape === 'spread' ? `Case${i}` : 'Cases';
const file = `${owner}.java`;
const suffix = shape === 'spread' ? '' : `_${i}`;
const body = `
String choose${suffix}(Object value) { return "object"; }
String choose${suffix}(String value) { return "string"; }
long widen${suffix}(long value) { return value; }
long widen${suffix}(Integer value) { return 0; }
Product make${suffix}(Object value) { return null; }
Product make${suffix}(String value) { return null; }
void run${suffix}() {
choose${suffix}(${changed && i === 0 ? '(Object) ' : ''}"text");
widen${suffix}(1);
var product = make${suffix}("text");
product.accept("text");
}
`;
if (shape === 'spread')
files.set(
file,
`package clients.p${i}; import bench.Product; public class ${owner} {${body}}\n`,
);
else declarations.push(body);
const caller = `Method:${file}:${owner}.run${suffix}#0`;
for (const [name, type] of [
['choose', changed && i === 0 ? 'Object' : 'String'],
['widen', 'long'],
['make', 'String'],
])
expected.push([caller, `Method:${file}:${owner}.${name}${suffix}#1~${type}`]);
expected.push([caller, 'Method:Product.java:Product.accept#1~String']);
}
if (shape === 'concentrated')
files.set('Cases.java', `package bench; public class Cases {${declarations.join('')}}\n`);
return {
files,
expected: expected.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
};
}
async function sample(shape, size) {
assert(shapes.includes(shape) && Number.isSafeInteger(size) && size > 0);
const directory = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-java-overload-bench-'));
const repo = path.join(directory, 'repo');
const input = corpus(shape, size);
Object.assign(process.env, {
GITNEXUS_HOME: path.join(directory, 'home'),
GITNEXUS_SHARED_STORE: 'off',
GITNEXUS_MEMORY: 'off',
GITNEXUS_LANG: 'en',
GITNEXUS_LBUG_BUFFER_POOL_SIZE: '134217728',
GITNEXUS_LBUG_EXTENSION_INSTALL: 'never',
XDG_RUNTIME_DIR: path.join(directory, 'runtime'),
GITNEXUS_MAX_FILE_SIZE: String(
Math.ceil(
Math.max(...[...input.files.values()].map((source) => Buffer.byteLength(source))) / 1024,
) + 1,
),
});
delete process.env.GITNEXUS_STORAGE_PATH;
delete process.env.GITNEXUS_STORAGE_ROOT;
await mkdir(repo);
await mkdir(process.env.XDG_RUNTIME_DIR, { mode: 0o700 });
const { runFullAnalysis } = await load('core/run-analyze.js');
const { scopeResolutionPhase } = await load('core/ingestion/scope-resolution/pipeline/phase.js');
const execute = scopeResolutionPhase.execute;
let scopeWallMs = 0,
scopeCpuMs = 0,
scopeRuns = 0;
scopeResolutionPhase.execute = async function (...args) {
const cpu = process.cpuUsage(),
start = performance.now();
try {
return await execute.apply(this, args);
} finally {
scopeWallMs += performance.now() - start;
const used = process.cpuUsage(cpu);
scopeCpuMs += (used.user + used.system) / 1000;
scopeRuns++;
}
};
try {
for (const [file, source] of input.files) await writeFile(path.join(repo, file), source);
const runs = [];
for (const scenario of scenarios) {
const fixture = scenario === 'changed' ? corpus(shape, size, true) : input;
if (scenario === 'changed') {
const edited = shape === 'spread' ? 'Case0.java' : 'Cases.java';
await writeFile(path.join(repo, edited), fixture.files.get(edited));
}
global.gc();
scopeWallMs = scopeCpuMs = scopeRuns = 0;
const cpu = process.cpuUsage(),
start = performance.now();
const analysis = await runFullAnalysis(
repo,
{
force: true,
skipAgentsMd: true,
skipSkills: true,
skipFts: true,
streamGraphEmit: false,
workerPoolSize: 1,
maxEntryPointCandidates: 7 * size + 2,
maxProcesses: 7 * size + 2,
},
{ onProgress: () => {} },
);
const wallMs = performance.now() - start;
const used = process.cpuUsage(cpu);
const heapBytes = process.memoryUsage().heapUsed;
const peakRssBytes = process.resourceUsage().maxRSS * 1024;
const pipeline = analysis.pipelineResult;
assert(pipeline, 'Missing pipeline output');
assert.equal(scopeRuns, 1, 'Every analysis must execute scope resolution');
assert.deepEqual(pipeline.scopeExtractionFailures, []);
assert.equal(pipeline.unavailableScopeLanguageFiles, 0);
assert.equal(
pipeline.processResult?.stats.truncation.truncated,
false,
'Process detection must cover the workload',
);
const calls = [...pipeline.graph.iterRelationshipsByType('CALLS')]
.map((edge) => [edge.sourceId, edge.targetId])
.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
assert.equal(
calls.length,
fixture.expected.length,
`${shape}/${size}/${scenario}: lost calls`,
);
assert.equal(
hash(calls),
hash(fixture.expected),
`${shape}/${size}/${scenario}: incorrect overload targets`,
);
const methodCount = [...pipeline.graph.iterNodes()].filter(
(node) => node.label === 'Method',
).length;
assert.equal(methodCount, 7 * size + 2, 'Workload lost method declarations');
const reparsed = pipeline.reparsedFileCount;
if (scenario === 'cold') assert.equal(reparsed, input.files.size);
else if (scenario === 'warm') assert.equal(reparsed, 0);
else {
// Cache entries contain chunks: editing one file can reparse its neighbors.
assert(reparsed > 0, 'The edited source must reach parser workers');
if (shape === 'spread')
assert(reparsed < input.files.size, 'The edit must retain cache hits');
}
assert.equal(pipeline.parseCacheHitFileCount, input.files.size - reparsed);
assert.equal(pipeline.usedWorkerPool, scenario !== 'warm');
assert(
!pipeline.resolutionOutcomes.some((outcome) => outcome.kind === 'suppressed'),
'Unexpected suppressed calls',
);
runs.push({
scenario,
wallMs,
cpuMs: (used.user + used.system) / 1000,
scopeWallMs,
scopeCpuMs,
heapBytes,
peakRssBytes,
correctness: {
files: input.files.size,
methods: methodCount,
calls: calls.length,
fingerprint: hash(calls),
reparsed,
hits: pipeline.parseCacheHitFileCount,
},
});
}
return { shape, size, corpus: hash([...input.files]), runs };
} finally {
scopeResolutionPhase.execute = execute;
await rm(directory, { recursive: true, force: true });
}
}
async function measure() {
assert(!(values.check && values.record), '--check cannot record a baseline');
assert(!values.record || !process.env.CI, 'CI must not record baselines');
assert(
!values.output || path.resolve(values.output) !== fileURLToPath(baselinePath),
'--output must not overwrite the baseline',
);
const sizes = values.sizes.split(',').map(Number),
samples = Number(values.samples);
assert(
sizes.length >= 3 &&
sizes.every((n, i) => Number.isSafeInteger(n) && n > 0 && (!i || n > sizes[i - 1])),
'Supply at least three increasing positive sizes',
);
assert(
Number.isSafeInteger(samples) && samples >= (values.check || values.record ? 3 : 1),
'Checks and baseline recording require at least three samples',
);
// Refuse existing files (including symlinks), as in the other ingestion harness.
if (values.output) await writeFile(values.output, '', { flag: 'wx' });
const report = {
schemaVersion: 1,
recordedAt: new Date().toISOString(),
runtimeSha256: await runtimeFingerprint(),
workload: { sizes, shapes, scenarios, overloadsPerFamily: 2, callsPerUnit: 4, workers: 1 },
environment: {
node: process.version,
platform: process.platform,
arch: process.arch,
cpu: os.cpus()[0]?.model,
},
samples: [],
measured: [],
scaling: {},
};
for (let repetition = 0; repetition < samples; repetition++) {
for (const size of repetition % 2 ? [...sizes].reverse() : sizes) {
for (const shape of repetition % 2 ? [...shapes].reverse() : shapes) {
console.error(
`java-overload-resolution: ${shape}/${size}, sample ${repetition + 1}/${samples}`,
);
const output = execFileSync(
process.execPath,
['--expose-gc', script, '--child', shape, '--size', String(size), '--root', root],
{
encoding: 'utf8',
timeout: 300_000,
maxBuffer: 16 * 1024 * 1024,
env: { ...process.env, GITNEXUS_MEMORY: 'off' },
},
);
const result = output.split('\n').find((line) => line.startsWith(marker));
assert(result, 'Child did not report measurements');
report.samples.push(JSON.parse(result.slice(marker.length)));
if (values.output) await writeFile(values.output, JSON.stringify(report, null, 2) + '\n');
}
}
}
for (const shape of shapes)
for (const size of sizes) {
const group = report.samples.filter((row) => row.shape === shape && row.size === size);
assert.equal(group.length, samples);
for (const scenario of scenarios) {
const rows = group.map((row) => row.runs.find((run) => run.scenario === scenario));
for (const row of rows) assert.deepEqual(row.correctness, rows[0].correctness);
report.measured.push({
shape,
size,
scenario,
corpus: group[0].corpus,
correctness: rows[0].correctness,
...Object.fromEntries(
metrics.map((metric) => [metric, median(rows.map((row) => row[metric]))]),
),
});
}
}
assert.equal(
await runtimeFingerprint(),
report.runtimeSha256,
'Built runtime changed during measurement',
);
for (const shape of shapes)
for (const scenario of scenarios) {
report.scaling[`${shape}/${scenario}`] = evaluateScaling(
report.measured.filter((row) => row.shape === shape && row.scenario === scenario),
metrics,
);
}
if (values.output) await writeFile(values.output, JSON.stringify(report, null, 2) + '\n');
console.log(JSON.stringify({ measured: report.measured, scaling: report.scaling }, null, 2));
if (values.check || values.record) {
assert(
Object.values(report.scaling).every((row) => row.passed),
'Time or memory growth exceeds the 1.5 normalized scaling budget',
);
const correctness = report.measured.map(({ shape, size, scenario, corpus, correctness }) => ({
shape,
size,
scenario,
corpus,
correctness,
}));
if (values.check) {
const baseline = JSON.parse(await readFile(baselinePath, 'utf8'));
assert.equal(baseline.schemaVersion, report.schemaVersion);
assert.deepEqual(report.workload, baseline.workload, 'Workload differs from baseline');
assert.deepEqual(
correctness,
baseline.correctness,
'Semantic or cache results differ from baseline',
);
} else {
await writeFile(
baselinePath,
JSON.stringify(
{
_note:
'Measured baseline. --check gates same-run normalized growth at 1.5 and exact call targets/cache counts. Absolute timings and memory are references, not portable budgets.',
schemaVersion: report.schemaVersion,
recordedAt: report.recordedAt,
runtimeSha256: report.runtimeSha256,
workload: report.workload,
environment: report.environment,
correctness,
measured: report.measured,
scaling: report.scaling,
},
null,
2,
) + '\n',
);
}
console.log(`[java-overload-resolution --${values.record ? 'record' : 'check'}] PASS`);
}
}
if (values.child)
console.log(marker + JSON.stringify(await sample(values.child, Number(values.size))));
else await measure();

View file

@ -29,10 +29,13 @@ candidate. Without `--check`, `--sessions 1 --rounds 1` is a smoke run and canno
establish a performance verdict. For an identical-code control, point both roots
at the same build. Do not discard slow samples or rerun until a check turns green.
The candidate indexes one fixed generated repository containing 32 short and 32
oversized TypeScript functions. Both servers read that same index. Index creation
is outside the timing window. Schemas must be compatible; an incompatible base
fails explicitly. Corpus generation and dependency installation use no model.
Both runtimes index the same fixed generated repository containing 32 short and
32 oversized TypeScript functions. Each server uses its own index and registry,
so engine upgrades can compare different native storage formats. Index creation
is outside the timing window. Both indexes must be complete and their responses
must pass the parity checks below. The report retains both indexes' metadata in
`indexes`; `index` remains the candidate metadata for existing report consumers.
Corpus generation and dependency installation use no model.
Eight fresh server pairs each run a warmup cycle and five measured rounds.
Request order alternates between baseline and candidate. Workloads cover small
@ -44,7 +47,9 @@ first query are recorded separately. Filesystem caches are not flushed.
- Response identities, order and diagnostics must agree; timing, guidance and
source representation are excluded from query equality. Indexed context source
must agree. At least ten known matches must survive each nonempty query.
must agree. Each staleness receipt timestamp must match its own index metadata;
only that timestamp is excluded from cross-runtime staleness equality. At least
ten known matches must survive each nonempty query.
- Candidate query source cannot exceed 16,000 Unicode characters, cannot disappear
entirely, and large previews must announce truncation. Context must retain the
original indexed snippet (the indexer separately caps long function snippets).

View file

@ -60,15 +60,12 @@ const runtimes = Object.fromEntries(
}),
);
const repo = path.join(out, 'repo');
const index = path.join(out, 'index');
const env = {
PATH: process.env.PATH,
HOME: process.env.HOME,
LANG: 'C.UTF-8',
TZ: 'UTC',
...(process.env.LD_LIBRARY_PATH ? { LD_LIBRARY_PATH: process.env.LD_LIBRARY_PATH } : {}),
GITNEXUS_HOME: path.join(out, 'home'),
GITNEXUS_STORAGE_PATH: index,
GITNEXUS_SHARED_STORE: 'off',
GITNEXUS_MEMORY: 'off',
GITNEXUS_CONTENT_RETENTION: 'full',
@ -76,6 +73,16 @@ const env = {
GITNEXUS_MCP_READ_ONLY: '1',
GITNEXUS_LBUG_EXTENSION_INSTALL: 'load-only',
};
const runtimeEnvs = Object.fromEntries(
['baseline', 'candidate'].map((variant) => [
variant,
{
...env,
GITNEXUS_HOME: path.join(out, 'home', variant),
GITNEXUS_STORAGE_PATH: path.join(out, 'index', variant),
},
]),
);
const queries = [
{ name: 'small_source', symbol: 'lookupSmallSource', source: true },
{ name: 'small_metadata', symbol: 'lookupSmallSource', source: false },
@ -109,7 +116,8 @@ function canonical(value) {
}
function inspect(response, entry, variant) {
assert(!response.isError, `MCP error in ${entry.name}`);
if (response.isError)
assert.fail(`MCP error in ${variant}/${entry.name}: ${JSON.stringify(response)}`);
if (entry.tool === 'tools/list') {
for (const name of ['query', 'context'])
assert(
@ -128,6 +136,12 @@ function inspect(response, entry, variant) {
!body.error && !body.partial,
`Incomplete ${variant}/${entry.name} response: ${JSON.stringify({ error: body.error, partial: body.partial, warning: body.warning })}`,
);
// Separate builds have different receipt timestamps. Validate each against its
// own index before excluding that timestamp from cross-runtime parity.
if (body.staleness) {
assert.equal(body.staleness.indexedAt, report.indexes[variant].indexedAt);
delete body.staleness.indexedAt;
}
const rows = entry.tool === 'query' ? [...body.process_symbols, ...body.definitions] : [];
if (entry.tool === 'query') {
if (entry.name === 'empty') assert.equal(rows.length, 0);
@ -185,7 +199,7 @@ async function open(variant, clients) {
command: process.execPath,
args: [runtimes[variant].cli, 'mcp'],
cwd: repo,
env,
env: runtimeEnvs[variant],
stderr: 'pipe',
});
const stderr = fs.createWriteStream(path.join(out, `${variant}.log`), { flags: 'a' });
@ -288,22 +302,33 @@ try {
execFileSync('git', args, { cwd: repo, env: gitEnv, stdio: 'pipe' });
}
report.fixtureSha256 = hash(JSON.stringify(source));
report.indexes = {};
const log = fs.openSync(path.join(out, 'index.log'), 'wx');
try {
execFileSync(process.execPath, [runtimes.candidate.cli, 'analyze', repo, '--index-only'], {
cwd: repo,
env,
stdio: ['ignore', log, log],
});
// Native storage formats can differ across engine upgrades. Each runtime
// builds its own index from the same fixture; response parity is checked below.
for (const variant of ['baseline', 'candidate']) {
const runtimeEnv = runtimeEnvs[variant];
fs.writeSync(log, `\n${variant} index (${runtimes[variant].sha})\n`);
execFileSync(process.execPath, [runtimes[variant].cli, 'analyze', repo, '--index-only'], {
cwd: repo,
env: runtimeEnv,
stdio: ['ignore', log, log],
});
const metadata = JSON.parse(
fs.readFileSync(path.join(runtimeEnv.GITNEXUS_STORAGE_PATH, 'gitnexus.json'), 'utf8'),
);
report.indexes[variant] = metadata;
assert(
metadata.stats.nodes >= 128 && metadata.stats.files === 64,
`Incomplete ${variant} fixture index`,
);
assert.equal(metadata.stats.embeddings ?? 0, 0, 'Benchmark must not generate embeddings');
}
} finally {
fs.closeSync(log);
}
report.index = JSON.parse(fs.readFileSync(path.join(index, 'gitnexus.json'), 'utf8'));
assert(
report.index.stats.nodes >= 128 && report.index.stats.files === 64,
'Incomplete fixture index',
);
assert.equal(report.index.stats.embeddings ?? 0, 0, 'Benchmark must not generate embeddings');
report.index = report.indexes.candidate; // Preserve the existing report field.
for (let session = 0; session < sessions; session++) {
const clients = {};
try {

View file

@ -172,7 +172,7 @@ export function formatReport(report) {
Base \`${report.runtimes.baseline.sha}\`; candidate \`${report.runtimes.candidate.sha}\`.
${report.sessions} fresh process pairs × ${report.rounds} rounds. Real MCP stdio, one shared keyword-search index, alternating requests. No model calls.
${report.sessions} fresh process pairs × ${report.rounds} rounds. Real MCP stdio, the same source fixture indexed by each runtime, alternating requests. No model calls.
| Workload | Median ms, base → candidate | p95 ms, base → candidate | Paired change (95% interval) | Assessment |
| --- | ---: | ---: | --- | --- |

View file

@ -1 +1 @@
283fafc9b2c8ba02f45ee25bf79d1619f9951db765c99474bad24f401c6f7882
55d64d78e2663a7d781f607d021d7e989caf7eb3ea928c85b93dce646a125222

View file

@ -202,14 +202,14 @@
}
},
"countArm": {
"callDrops": 140,
"totalDropsAllKinds": 171,
"callDrops": 141,
"totalDropsAllKinds": 175,
"bySiteKind": {
"call": 140,
"read": 31
"call": 141,
"read": 34
},
"callDropsByExtension": {
".java": 49,
".java": 50,
".py": 24,
".ts": 18,
".zig": 10,
@ -224,16 +224,16 @@
".swift": 1
},
"callDropsByShape": {
"chain-field": 56,
"no-chain": 38,
"chain-call": 32,
"chain-field": 57,
"no-chain": 39,
"chain-call": 31,
"<<unclassified>>": 13,
"chain-unwrap": 1
},
"callDropsByOrigin": {
"in-program": 58,
"unknown": 49,
"external": 31,
"unknown": 64,
"in-program": 53,
"external": 22,
"<<unclassified>>": 2
}
}

View file

@ -194,7 +194,8 @@
"_rebaselined_3498_prefixed_call_bindings": "#3498: a local initialized by exactly `prefix.member(...)` now gets a call-derived @type-binding.constructor whose type keeps the dotted callee, so the shared chain follow can type namespace factories (`final x = rp.make()`). Two existing fixture sites gain a binding, both in dart-method-enrichment/app.dart: `var sound = dog.speak()` and `var category = Animal.classify(\"dog\")`; neither variable is used as a receiver, and a full lang-resolution A/B against origin/main emits the identical CALLS/ACCESSES edges outside the new fixture. The new dart-prefixed-factory-return fixture adds its own captures. Prior 4690403d2be0d270e28a55791df82eaa627d26d28c52d81bfafa59761badcaf3 (897dfc745d4ba0dfe0ae5fcfffcd983563ddab6d0a870fc8ee0de1d8df2c7c87 with the existing fixtures alone) -> a8b2e674a7f695caba32b5a0912686119944e8e4874a6ceeb65b3b5ac420bebb. scaling_ratio 1.173 against the 1.5 budget. Review follow-up: the fixture gains an awaited `rp.makeAsync()` row (corpus growth only; the Dart emitter change in that commit is a docstring) -> 1c5b892a136318eddee3435225284896c2effb52dd51ca5717c92887ae4a39a3."
},
"java": {
"fingerprint": "fbd822c3b6848dc6f5e708218c1ab9c1244f2f9f02f4e3df5ed9296137b776d8",
"fingerprint": "a0fc49d8d1e80678ab05004eabc4e2229b5954df3b884a8149017e48a6ebe425",
"_rebaselined_3789_overloads": "#3789: Compared base 82966afa captures with this provider on the same corpus. Existing entries differ only in 387 method return-type captures and four added var call-result assignments. Seven new fixture files add 284 capture groups; total groups 3515 -> 3803, entries 222 -> 229. Removing those two capture changes reproduces every existing entry. The base emitter reproduces the prior fingerprint exactly. Scaling budget remains 1.5.",
"scaling_budget": 1.5,
"_rebaselined_3190_ctor_qualified_name": "#3190: qualified constructors (`new pkg.Foo()`, `new pkg.Box<T>()`) now copy @reference.call.constructor.qualified onto @reference.qualified-name so constructor fallback treats the written qualifier as precise. DIGEST DRIFT ONLY: the tag is added to existing F35 matches. Prior 2bf47cc19b595a9889ac21ec0154c6ce6786271d68551f21d1bc14c626bcd4ff -> e22a753b9f59331898a3a27c43072b588c33639937b1f618f6c70ce04f072a9f; capture_groups_fp 3586, fixture_count 209.",
"_rebaselined_2935_synthetic_declarations": "PR #2935 review follow-up: synthesized Java anonymous classes and bodied enum constants now carry the presence-only @declaration.is-synthetic sidecar used to preserve source-written dispatch targets at the fanout cap. DIGEST DRIFT ONLY, NOT A CAPTURE-SET CHANGE: the tag is attached to existing synthetic declaration matches; capture groups and fixture count remain 5755/18405, 3512, and 206. Prior 36d689c58526c4482fbd701d1d9ca156623a3970734ead145717858712271ab5 -> 2e2150b4f4d64519e3f4c6d7a2c12259178d3117872203c904fab8cba96a694a; CI scaling 0.971 < 1.5.",
@ -212,24 +213,25 @@
"_rebaselined_2562_local_classes": "#2562: Java block-local classes, enums, records, and interfaces use source-type-relative JLS 13.1 Host$NLocal identities with javac-compatible per-(host, simple-name) numbering; anonymous numbering remains separate. Lexical aliases begin at each declaration and end with its immediate block. Expanded java-local-class-naming fixtures cover declaration order, disjoint blocks, initializers, lambdas, local type kinds, and recursive local/member/anonymous host chains. Prior d04298a91beec76d0fa7099b3d71265723be60c1df688969aa954f135dd49686 -> 6dd5913a58400a191ff54abf9b852b03d5add657d16c11e60a7c4608ba186197; scaling 1.204 < 1.5.",
"_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior 6dd5913a58400a191ff54abf9b852b03d5add657d16c11e60a7c4608ba186197 -> 310adbc2e0827b5ac749acaa981cd12d256fc5b7cbc5592c5bee219e92abf9ee.",
"_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 310adbc2e0827b5ac749acaa981cd12d256fc5b7cbc5592c5bee219e92abf9ee -> a9943355e945e03ddb87c800f4cc1f62b3d04feefb3ec64c258d8e0bb3b3fcd9.",
"capture_groups_small": 6255,
"capture_groups_large": 20005,
"capture_groups_fp": 3515,
"fixture_count": 222,
"capture_groups_small": 6505,
"capture_groups_large": 20805,
"capture_groups_fp": 3803,
"fixture_count": 229,
"_rebaselined_2910_declared_package_fixtures": "#2910 adds three Java resolver fixture files covering an external JDK lookalike, a path/package mismatch, and wildcard package membership. Fixture-corpus growth only: Java query rules and synthetic scaling sources are unchanged; capture_groups_small/large remain 6255/20005. capture_groups_fp 3560 -> 3586 and fixture_count 206 -> 209.",
"_rebaselined_3486_semantic_rename": "#3486: Compared actual origin/main (191a954b), committed dd3bf and frozen provider source over the same unchanged corpus. 39 constructor-parameter type bindings are added, 6 member reads gain receiver chains, and 196 duplicate qualified free-call matches are removed. Every removed free-call match retains a member-call match with the identical call anchor and name capture; no qualified call disappears. Fixture entries 209; changed entries 89; capture groups 3586 -> 3429. Fingerprint regenerated from the runner's canonicalized capture records. Scaling budgets unchanged.",
"_rebaselined_same_namespace_heritage_fixture": "Same-namespace heritage fixture: adds `java-same-package-extends-collision` (9 files). FIXTURE ADDITION ONLY, no capture code changed: with that directory parked, --check passes against the prior fingerprint 41c4338f5813a73199dd5c2609313c11076812df21fd9c429039a833325870e0 (measured fixture_count 209, capture_groups_fp 3429); with it, fixture_count 218, capture_groups_fp 3483. Synthetic counts and scaling unchanged.",
"_rebaselined_duplicate_fqn_import_fixture": "Duplicate-FQN import fixture: adds `java-duplicate-fqn-source-roots` (4 files). FIXTURE ADDITION ONLY, no capture code changed: with that directory parked, --check passes against the prior fingerprint 82a4b09708caed2350ab36d2b7a1ff00639a34bca73dd1536e37f231e800c2d2 (fixture_count 218, capture_groups_fp 3483); with it, fixture_count 222, capture_groups_fp 3515. Synthetic counts and scaling unchanged."
},
"java-local-types": {
"fingerprint": "6ae163cd36c889454c8672b28944c765d3a3a5e8c49ea4db1cd6cc1787ede583",
"fingerprint": "41f1d8a9dcac9fad0889ae698b5a6710ffb43b264ad383a3fd0c172f2da2b596",
"_rebaselined_3789_overloads": "#3789: All six entries differ only in 53 method return-type captures; capture groups remain 644. The base 82966afa emitter reproduces the prior fingerprint exactly, and removing the return-type captures makes every entry identical. Corrects the stale group-count field left at 680 after #3486. Scaling budget remains 1.5.",
"scaling_budget": 1.5,
"_rebaselined_2935_synthetic_declarations": "PR #2935 review follow-up: the local-type stress corpus includes synthesized anonymous declarations, which now carry the presence-only @declaration.is-synthetic sidecar. DIGEST DRIFT ONLY, NOT A CAPTURE-SET CHANGE. Prior 8c50bbc83dff4f7f5abd06078aa6abc6b64af05fddb17ee826b5f3df3d346633 -> 560734cd053fb4f4b23aa04bc7870c22089a8deedb0217fa9c1b4db689e02a97; CI scaling 1.002 < 1.5.",
"_rebaselined_2917_record_component_accessors": "#2917: the focused local-type fixture corpus contains local records, so their implicit component accessors add the same bounded scope/declaration captures as the general Java corpus. No local-type naming logic changed. Prior 8c50bbc83dff4f7f5abd06078aa6abc6b64af05fddb17ee826b5f3df3d346633 -> 3e22f368a4ee139be7cb91ff4fb77ddadf60c55efe8d66955ec81f366a46e460; scaling 1.032 < 1.5, capture_groups_fp 680. Re-measured on top of #2935's is-synthetic sidecar after merging origin/main: 560734cd053fb4f4b23aa04bc7870c22089a8deedb0217fa9c1b4db689e02a97 -> bdde823fa725e636e257940efb4c8655aa23124c1727cbaa8856d1ad8f71729e; scaling 0.997 < 1.5, capture_groups_fp 680.",
"_added": "#2562 performance follow-up: co-scales same-host, same-name local classes and anonymous classes to gate JLS binary-name ordinal allocation. Precomputed per-sequence ordinals reduce the focused 100->800 workload from 176->6655ms to 141->752ms; normalized 250->800 scaling is 1.054.",
"_rebaselined_receiver_chain_2747": "#2747 receiver-chain rollout: call matches whose receiver is itself an expression now carry `@reference.receiver-chain`, a compact encoding of the receiver's structure, so resolution types it by folding instead of re-parsing receiver source text. Capture GROUP counts are unchanged \u2014 the tag is added to existing call matches, never a new match \u2014 so this is digest drift only. Prior a9ad88de21ca6747a923260dbdf677fb74a004abbf9d57781f745e3a9027530b -> 3ca67847ea2b9a71b0a41e09f943767e5a2d3a113d3e203499ee364e37f40236.",
"_rebaselined_2766_receiver_chain_wire_v2": "#2766: receiver-chain wire format v1 -> v2 (name-free `await` / `index` step kinds). The VERSION prefix is part of every emitted `@reference.receiver-chain` capture, so every chain-minting language's capture text changed. WIRE-FORMAT CHANGE, NOT A CAPTURE-SET CHANGE: the same chains are minted for the same sites, spelled `2|\u2026` instead of `1|\u2026`. Exactly the 12 chain-minting languages drifted; c, cobol and dart did not, which is the check that this is the prefix and not a capture regression. Accompanied by SCHEMA_BUMP 34 -> 37 and INCREMENTAL_SCHEMA_VERSION 28 -> 31 so a stale index is rejected rather than replaying chains a v2 decoder refuses. Prior 3ca67847ea2b9a71b0a41e09f943767e5a2d3a113d3e203499ee364e37f40236 -> 8c50bbc83dff4f7f5abd06078aa6abc6b64af05fddb17ee826b5f3df3d346633.",
"capture_groups_fp": 680,
"capture_groups_fp": 644,
"_rebaselined_3486_semantic_rename": "#3486: Compared actual origin/main (191a954b), committed dd3bf and frozen provider source over the same unchanged corpus. 2 constructor-parameter type bindings are added and 38 duplicate qualified free-call matches are removed. Every removed free-call match retains a member-call match with the identical call anchor and name capture. Fixture entries 6; changed entries 6; capture groups 680 -> 644. Fingerprint regenerated from the runner's canonicalized capture records. Scaling budgets unchanged."
},
"typescript": {

View file

@ -125,6 +125,7 @@ try {
pool,
{ LocalBackend },
{ summarizeUnresolvedReceivers },
{ summarizeUnresolvedOverloads },
{ taintModelVersion },
] = await Promise.all([
import('../../src/core/ingestion/pipeline.js'),
@ -132,6 +133,7 @@ try {
import('../../src/core/lbug/pool-adapter.js'),
import('../../src/mcp/local/local-backend.js'),
import('../../src/core/ingestion/scope-resolution/unresolved-receivers.js'),
import('../../src/core/ingestion/scope-resolution/unresolved-overloads.js'),
import('../../src/core/ingestion/taint/typescript-model.js'),
]);
adapter = await import('../../src/core/lbug/lbug-adapter.js');
@ -174,6 +176,10 @@ try {
indexedAt: '2026-10-06T00:00:00.000Z',
scopeExtractionReceipt: 1,
unresolvedReceiverMembers: summarizeUnresolvedReceivers(pipeline.resolutionOutcomes),
unresolvedOverloads: summarizeUnresolvedOverloads(
pipeline.resolutionOutcomes,
pipeline.resolutionOutcomes !== undefined,
),
pdg: { maxFunctionLines: 2000, maxEdgesPerFunction: 0, taintModelVersion },
stats: {
files: pipeline.totalFileCount,

View file

@ -23,7 +23,7 @@
"hasInstallScript": true,
"license": "PolyForm-Noncommercial-1.0.0",
"dependencies": {
"@ladybugdb/core": "0.21.1",
"@ladybugdb/core": "0.22.0",
"@modelcontextprotocol/sdk": "^1.0.0",
"@scarf/scarf": "^1.4.0",
"busboy": "^1.6.0",
@ -34,7 +34,7 @@
"express": "^5.2.1",
"express-rate-limit": "^8.4.1",
"fast-xml-parser": "^5.11.1",
"glob": "^13.0.6",
"glob": "^14.0.2",
"graphology": "^0.26.0",
"graphology-indices": "^0.17.0",
"graphology-utils": "^2.3.0",
@ -73,8 +73,9 @@
"@babel/traverse": "^8.0.0",
"@babel/types": "^8.0.0",
"@mermaid-js/tiny": "12.2.0",
"@stryker-mutator/api": "10.0.0",
"@stryker-mutator/core": "10.0.0",
"@streamparser/json": "0.0.26",
"@stryker-mutator/api": "10.1.1",
"@stryker-mutator/core": "10.1.1",
"@types/busboy": "^1.5.4",
"@types/cli-progress": "^3.11.6",
"@types/cors": "^2.8.17",
@ -83,6 +84,7 @@
"@types/proxy-addr": "^2.0.3",
"@vitest/coverage-v8": "^5.0.0",
"gitnexus-shared": "file:../gitnexus-shared",
"jscpd": "5.4.2",
"jsdom": "30.1.3",
"tsx": "^4.0.0",
"typescript": "^7.0.2",
@ -1717,6 +1719,12 @@
"node": ">=18.0.0"
}
},
"node_modules/@isaacs/nonrecursive-glob-match": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.3.tgz",
"integrity": "sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==",
"license": "BlueOak-1.0.0"
},
"node_modules/@jridgewell/gen-mapping": {
"version": "0.4.0-beta.0",
"resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.4.0-beta.0.tgz",
@ -1757,9 +1765,9 @@
}
},
"node_modules/@ladybugdb/core": {
"version": "0.21.1",
"resolved": "https://registry.npmjs.org/@ladybugdb/core/-/core-0.21.1.tgz",
"integrity": "sha512-G0LWyn/dIqX9yHlyjpR12Rvou2o1dvCB9EkuzGa5ykCYbzur8EQhRUbi4fw3FlbckWvR6rK/tVbnFpBcRLwiHg==",
"version": "0.22.0",
"resolved": "https://registry.npmjs.org/@ladybugdb/core/-/core-0.22.0.tgz",
"integrity": "sha512-kEFtewqyRUk4rQNq2DA5xPgIyC2cOI8PhTixqz0sOhG6/BZ3hhoAP9RkYm0NkcAfCSH3SXs1varNGuccqPcapA==",
"hasInstallScript": true,
"license": "MIT",
"dependencies": {
@ -1768,17 +1776,17 @@
"node-addon-api": "^6.0.0"
},
"optionalDependencies": {
"@ladybugdb/core-darwin-arm64": "0.21.1",
"@ladybugdb/core-darwin-x64": "0.21.1",
"@ladybugdb/core-linux-arm64": "0.21.1",
"@ladybugdb/core-linux-x64": "0.21.1",
"@ladybugdb/core-win32-x64": "0.21.1"
"@ladybugdb/core-darwin-arm64": "0.22.0",
"@ladybugdb/core-darwin-x64": "0.22.0",
"@ladybugdb/core-linux-arm64": "0.22.0",
"@ladybugdb/core-linux-x64": "0.22.0",
"@ladybugdb/core-win32-x64": "0.22.0"
}
},
"node_modules/@ladybugdb/core-darwin-arm64": {
"version": "0.21.1",
"resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-arm64/-/core-darwin-arm64-0.21.1.tgz",
"integrity": "sha512-TEFYNqBdbIojf1t29p3esgRhMDm56lD5eK91dwWFZJTVWAZoGXn+yLKs0NUQgj3lW2ZquuRTLZGglPhFEsDL1Q==",
"version": "0.22.0",
"resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-arm64/-/core-darwin-arm64-0.22.0.tgz",
"integrity": "sha512-UE7UnM/Stb/wDyk/1Jl6BgZOl9KmyBr2EVfiASeLmRka09RKlSI6t5/cVu5j7v0qdJzvi/Csg0CP+lhWNNMurg==",
"cpu": [
"arm64"
],
@ -1789,9 +1797,9 @@
]
},
"node_modules/@ladybugdb/core-darwin-x64": {
"version": "0.21.1",
"resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-x64/-/core-darwin-x64-0.21.1.tgz",
"integrity": "sha512-w9p3oKrqeMOeWpX3xdCc/hQJHfrZtpB9We+Ir+0qyls68dSQoeXXR1GZYzzTElOtxr4OM5USwYbVyxADx+CMZg==",
"version": "0.22.0",
"resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-x64/-/core-darwin-x64-0.22.0.tgz",
"integrity": "sha512-VM4yiEVTNQel4Dz5cO9Rmc/NBiBWIsBxwMnD3MfSl9gZGGSoLTHa9JDD1RcDJoQ2tZvvXtQMjw5q6E7tnYYyAw==",
"cpu": [
"x64"
],
@ -1802,9 +1810,9 @@
]
},
"node_modules/@ladybugdb/core-linux-arm64": {
"version": "0.21.1",
"resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-arm64/-/core-linux-arm64-0.21.1.tgz",
"integrity": "sha512-TodWmzOHxmmKvXXo9buBXGeyolxf/NqBUbo/LcR/tuMViKhGH8TBzIXi8DfVVwmsJ+CQCU065TvXgv2JoHAlZg==",
"version": "0.22.0",
"resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-arm64/-/core-linux-arm64-0.22.0.tgz",
"integrity": "sha512-54K+vxXxQkpxnY6v6nBTPdSY4g9OEoKjXt2PxzRLnoNyZ1OMERxWiwaFLr4jAc5HHgpA3jHLIAvkaY2RSnDaWA==",
"cpu": [
"arm64"
],
@ -1815,9 +1823,9 @@
]
},
"node_modules/@ladybugdb/core-linux-x64": {
"version": "0.21.1",
"resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-x64/-/core-linux-x64-0.21.1.tgz",
"integrity": "sha512-hCcLYv8ds0x4fkTaRkuc4f75KLdH9e0EV51ne6uBNdQSY7kyc0RgSoUG3utUGfFDMMkfgIJAeIxQY4ck9jQiXg==",
"version": "0.22.0",
"resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-x64/-/core-linux-x64-0.22.0.tgz",
"integrity": "sha512-ZUbkNmRTjMb16XBPJSCkdNCW4up30jHpDzn9fn6qifg50XnkUQ3eje200EyrdIjd/LLYi9d/VG72mMRW7+cQTA==",
"cpu": [
"x64"
],
@ -1828,9 +1836,9 @@
]
},
"node_modules/@ladybugdb/core-win32-x64": {
"version": "0.21.1",
"resolved": "https://registry.npmjs.org/@ladybugdb/core-win32-x64/-/core-win32-x64-0.21.1.tgz",
"integrity": "sha512-nBS1XSRJfpexdlhkezmEiRtkUt2qEDp4zBjIhH+oxgRdD7UsRz1Jyit8foJuZAmUsoQGukN5LO+YeyDU2WZTyA==",
"version": "0.22.0",
"resolved": "https://registry.npmjs.org/@ladybugdb/core-win32-x64/-/core-win32-x64-0.22.0.tgz",
"integrity": "sha512-A7XW4cQYXQjfizF+yook55fSppdF07vUzwhi9d3ER1Z1CnkrzmECAF5DIwaDQ+jFAgwUCCEcJ7kQ9HrnYtZ6Zg==",
"cpu": [
"x64"
],
@ -2230,10 +2238,17 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/@streamparser/json": {
"version": "0.0.26",
"resolved": "https://registry.npmjs.org/@streamparser/json/-/json-0.0.26.tgz",
"integrity": "sha512-46597LNFI+MFdUnzX2QJWwmdTRdq0XVD+vVNJTtGVzIrnCuhG9pFo1OAzbNBqci8UJgk/X5KJZ6LcV+y7PTuDQ==",
"dev": true,
"license": "MIT"
},
"node_modules/@stryker-mutator/api": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@stryker-mutator/api/-/api-10.0.0.tgz",
"integrity": "sha512-ZtAJ0ZT3MVRCWJTBE2h90XB/6E+4lifHYtcTyNG6nU2nLekPgTo4gD5esjX6Okxo1b/JB4jJzyxYB54fwKAoJw==",
"version": "10.1.1",
"resolved": "https://registry.npmjs.org/@stryker-mutator/api/-/api-10.1.1.tgz",
"integrity": "sha512-3s/ECsrD1Tf1WEB7amBk0HxjN7zgkabKvmqnI2C831rTNszJ6o2HoV9PVxtElDEWxb2+t4iavK5x25G+DTUOUA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
@ -2247,16 +2262,16 @@
}
},
"node_modules/@stryker-mutator/core": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@stryker-mutator/core/-/core-10.0.0.tgz",
"integrity": "sha512-ZvMsRyaXQQ5e6Thcid9pkuODv6Fn9E3nrBQJUap+hcJuGJ4unm26afo3m6YKSjn8kinyxJ/3TXf0cTWRDaTxVw==",
"version": "10.1.1",
"resolved": "https://registry.npmjs.org/@stryker-mutator/core/-/core-10.1.1.tgz",
"integrity": "sha512-4xJAOCw9utJMLyrTk9gINi434tBxnyuwt8lU/InyoKagKd2/zh/2ui7ncWoD0qZeJrxClYzblg8iUh5ewBAB3Q==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"@inquirer/prompts": "^8.0.0",
"@stryker-mutator/api": "10.0.0",
"@stryker-mutator/instrumenter": "10.0.0",
"@stryker-mutator/util": "10.0.0",
"@stryker-mutator/api": "10.1.1",
"@stryker-mutator/instrumenter": "10.1.1",
"@stryker-mutator/util": "10.1.1",
"ajv": "~8.20.0",
"chalk": "~5.6.0",
"commander": "~14.0.0",
@ -2318,9 +2333,9 @@
"license": "MIT"
},
"node_modules/@stryker-mutator/instrumenter": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@stryker-mutator/instrumenter/-/instrumenter-10.0.0.tgz",
"integrity": "sha512-B7Wmn1KlEWyFeOz6D6oGvQGRfi5Xw3VemG6dEKvFQp4qLvxD9Mf4kcZghfxffgnYwXd3bFgqXsJ+ZGlhdfIOrQ==",
"version": "10.1.1",
"resolved": "https://registry.npmjs.org/@stryker-mutator/instrumenter/-/instrumenter-10.1.1.tgz",
"integrity": "sha512-Vyude2i8LhDciLV1q6J55KtdA7gZpRuMfZcBcmWV6hLHAvWocoPKnCKc5BlZt1mwGWkX3Olc0457I7Ue6F1WEg==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
@ -2332,8 +2347,8 @@
"@babel/preset-react": "~8.0.0",
"@babel/preset-typescript": "~8.0.0",
"@babel/traverse": "~8.0.4",
"@stryker-mutator/api": "10.0.0",
"@stryker-mutator/util": "10.0.0",
"@stryker-mutator/api": "10.1.1",
"@stryker-mutator/util": "10.1.1",
"angular-html-parser": "~10.11.0",
"semver": "~7.8.0",
"tslib": "2.8.1",
@ -2344,9 +2359,9 @@
}
},
"node_modules/@stryker-mutator/util": {
"version": "10.0.0",
"resolved": "https://registry.npmjs.org/@stryker-mutator/util/-/util-10.0.0.tgz",
"integrity": "sha512-LzOpHiJaCp2ABQgnPMlrQQcsK43bd5Vo/2FGL78aN62yDoeRQ+4j3tzeuXxK5OAHdC3fUz6TDoy4IsoAuLAd3w==",
"version": "10.1.1",
"resolved": "https://registry.npmjs.org/@stryker-mutator/util/-/util-10.1.1.tgz",
"integrity": "sha512-Z0zDzsmhoSykE1gT3DvD1knFxySgNMfBlpSSs1j87kWvjEMfeMqfLOk0bqEPRg/rTFIUNSEXLZapxePUhZmfZA==",
"dev": true,
"license": "Apache-2.0"
},
@ -4406,14 +4421,30 @@
"link": true
},
"node_modules/glob": {
"version": "13.0.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz",
"integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==",
"version": "14.0.2",
"resolved": "https://registry.npmjs.org/glob/-/glob-14.0.2.tgz",
"integrity": "sha512-vfMgTR2djMHPUchsJMLpZlYQvtjVL+mHpEr5wPpoCbQs2vr/FBNK+3Gnl9Ara1DSs4Aeg4eG9HiADb5/rkY/ZA==",
"license": "BlueOak-1.0.0",
"dependencies": {
"minimatch": "^10.2.2",
"minimatch": "^11.0.0",
"minipass": "^7.1.3",
"path-scurry": "^2.0.2"
"path-scurry": "^3.0.0"
},
"engines": {
"node": "18 || 20 || >=22"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/glob/node_modules/minimatch": {
"version": "11.0.3",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.3.tgz",
"integrity": "sha512-iny/40PEGkNJQ5/Pv6z3nm6FbRAby32GeZRjCUJk2WnNDbgPSTcdI+DDvLqsgBdD4cD5sMtMheqPPnehCCa8Nw==",
"license": "BlueOak-1.0.0",
"dependencies": {
"@isaacs/nonrecursive-glob-match": "^1.0.3",
"brace-expansion": "^5.0.12"
},
"engines": {
"node": "18 || 20 || >=22"
@ -4792,6 +4823,144 @@
"js-yaml": "bin/js-yaml.mjs"
}
},
"node_modules/jscpd": {
"version": "5.4.2",
"resolved": "https://registry.npmjs.org/jscpd/-/jscpd-5.4.2.tgz",
"integrity": "sha512-bxZ1ObA4NyWBcM7Zh+QoGrdcMovbCxrDofFaLo2Lwgk/Axe/e45VYu8EM+mpPx2CpU/QGCoktONRnaRZl8Yq2w==",
"dev": true,
"license": "MIT",
"bin": {
"jscpd": "run-jscpd.js"
},
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://opencollective.com/jscpd"
},
"optionalDependencies": {
"jscpd-darwin-arm64": "5.4.2",
"jscpd-darwin-x64": "5.4.2",
"jscpd-linux-arm64-gnu": "5.4.2",
"jscpd-linux-arm64-musl": "5.4.2",
"jscpd-linux-x64-gnu": "5.4.2",
"jscpd-linux-x64-musl": "5.4.2",
"jscpd-windows-arm64-msvc": "5.4.2",
"jscpd-windows-x64-msvc": "5.4.2"
}
},
"node_modules/jscpd-darwin-arm64": {
"version": "5.4.2",
"resolved": "https://registry.npmjs.org/jscpd-darwin-arm64/-/jscpd-darwin-arm64-5.4.2.tgz",
"integrity": "sha512-EGoPYQZqESW2kJfxKZrUYsYWhzq2Irk6sBQQWZ7VPAGj2wwk3n8HtDP2FnJOiHWZtSz55I8zQWldzIrD55B5Vw==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/jscpd-darwin-x64": {
"version": "5.4.2",
"resolved": "https://registry.npmjs.org/jscpd-darwin-x64/-/jscpd-darwin-x64-5.4.2.tgz",
"integrity": "sha512-/bz/rgoahqeK0p5r+KjeTTS59d7bZpMxD7hQ4CaQ0qKZWBZIyu7hRdExzJ7/KrCEJ5cmRogvXX1jhFqldYLNXw==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/jscpd-linux-arm64-gnu": {
"version": "5.4.2",
"resolved": "https://registry.npmjs.org/jscpd-linux-arm64-gnu/-/jscpd-linux-arm64-gnu-5.4.2.tgz",
"integrity": "sha512-YW/eX+mOYZmT/A3QEARYmJQoVQNV/cfecKCobNXZcD3mE57drpKHmJnvuL9uS85z0cUKWGCM57JVkVz6Y9oxxw==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/jscpd-linux-arm64-musl": {
"version": "5.4.2",
"resolved": "https://registry.npmjs.org/jscpd-linux-arm64-musl/-/jscpd-linux-arm64-musl-5.4.2.tgz",
"integrity": "sha512-jOLTO8YVRz65NaeYNeXoIf13yOPWhbM61xKXpv51DXQ2BXce6XfgPaJVeDCOM5s+8mt6aDtq0BZ8zptNeqGuIw==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/jscpd-linux-x64-gnu": {
"version": "5.4.2",
"resolved": "https://registry.npmjs.org/jscpd-linux-x64-gnu/-/jscpd-linux-x64-gnu-5.4.2.tgz",
"integrity": "sha512-eAmKLxx+pIOEX4lZ342CCCjL5bvf+a3gdmaaVHXiGIx2Zr3zKkF32YkfuN2sCvCui5I/wnely80XsRvVVFO47Q==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/jscpd-linux-x64-musl": {
"version": "5.4.2",
"resolved": "https://registry.npmjs.org/jscpd-linux-x64-musl/-/jscpd-linux-x64-musl-5.4.2.tgz",
"integrity": "sha512-ers9YFscD6DrmhyjBDXo/L+yBKcclHdWTFLMulYVGLcoGRKfBFhmK08MI3ajN/GMmCpJu7rkDYKL3egdfE8OIw==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/jscpd-windows-arm64-msvc": {
"version": "5.4.2",
"resolved": "https://registry.npmjs.org/jscpd-windows-arm64-msvc/-/jscpd-windows-arm64-msvc-5.4.2.tgz",
"integrity": "sha512-kOhVw4cVhN8T9jr3OVThB3pPpbff0ITHTwpJ/+4EGzMkbf0y6QtPpHDzRgiHJj7u1FSTBXTI6VhPhDUhT/os1g==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
]
},
"node_modules/jscpd-windows-x64-msvc": {
"version": "5.4.2",
"resolved": "https://registry.npmjs.org/jscpd-windows-x64-msvc/-/jscpd-windows-x64-msvc-5.4.2.tgz",
"integrity": "sha512-BuR5DCV7F/G5ukMEIzCrz43ea5C0t0ALcP7v+opOWefoi4aiQYCEmtoRwfXMM/zjOmNYmvJBDDfeH05U9uuzEg==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
]
},
"node_modules/jsdom": {
"version": "30.1.3",
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-30.1.3.tgz",
@ -5343,6 +5512,7 @@
"version": "10.2.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz",
"integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==",
"dev": true,
"license": "BlueOak-1.0.0",
"dependencies": {
"brace-expansion": "^5.0.2"
@ -5698,9 +5868,9 @@
}
},
"node_modules/path-scurry": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz",
"integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==",
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-3.0.0.tgz",
"integrity": "sha512-1pTbQqH1wIwMi/eleQ5leFTTCkzFsPcgXNjwOA8YGGW/W4/kHT3dssE7Z6NH+zuqNLOzOB0uaD61yl7cAcVhgg==",
"license": "BlueOak-1.0.0",
"dependencies": {
"lru-cache": "^11.0.0",

View file

@ -81,7 +81,7 @@
"version": "node scripts/sync-plugin-manifests.mjs"
},
"dependencies": {
"@ladybugdb/core": "0.21.1",
"@ladybugdb/core": "0.22.0",
"@modelcontextprotocol/sdk": "^1.0.0",
"@scarf/scarf": "^1.4.0",
"busboy": "^1.6.0",
@ -92,7 +92,7 @@
"express": "^5.2.1",
"express-rate-limit": "^8.4.1",
"fast-xml-parser": "^5.11.1",
"glob": "^13.0.6",
"glob": "^14.0.2",
"graphology": "^0.26.0",
"graphology-indices": "^0.17.0",
"graphology-utils": "^2.3.0",
@ -137,8 +137,9 @@
"@babel/traverse": "^8.0.0",
"@babel/types": "^8.0.0",
"@mermaid-js/tiny": "12.2.0",
"@stryker-mutator/api": "10.0.0",
"@stryker-mutator/core": "10.0.0",
"@streamparser/json": "0.0.26",
"@stryker-mutator/api": "10.1.1",
"@stryker-mutator/core": "10.1.1",
"@types/busboy": "^1.5.4",
"@types/cli-progress": "^3.11.6",
"@types/cors": "^2.8.17",
@ -147,6 +148,7 @@
"@types/proxy-addr": "^2.0.3",
"@vitest/coverage-v8": "^5.0.0",
"gitnexus-shared": "file:../gitnexus-shared",
"jscpd": "5.4.2",
"jsdom": "30.1.3",
"tsx": "^4.0.0",
"typescript": "^7.0.2",

View file

@ -164,6 +164,8 @@ const PLATFORM_LOGIC = [
// N-API addon which has known platform-specific behavior (Windows
// file-lock lag after close, macOS N-API destructor segfaults)
const LBUG_NATIVE = [
'test/integration/java-overload-impact-persistence.test.ts',
'test/integration/dangling-owner-edges.test.ts',
'test/integration/impact-private-type-file-seed.test.ts',
'test/integration/impact-private-type-source.test.ts',
// Inline-handler findings must survive native persistence and cold explain
@ -193,6 +195,9 @@ const LBUG_NATIVE = [
'test/integration/rename-provider-zig.test.ts',
'test/integration/pdg-explicit-owner.test.ts',
'test/integration/skip-fts.test.ts',
// Dirty snapshot freshness depends on native FTS, Git paths and file reads.
'test/integration/analyze-dirty-snapshot.test.ts',
'test/integration/incremental-worktree-context.test.ts',
'test/integration/lbug-core-adapter.test.ts',
// Native COPY, cached vectors and cold shared-store readers vary by platform.
'test/integration/embedding-identity-integrity.test.ts',
@ -208,12 +213,15 @@ const LBUG_NATIVE = [
'test/integration/lbug-orphan-sidecar-recovery.test.ts',
'test/integration/lbug-interrupted-checkpoint-recovery.test.ts',
'test/integration/lbug-checkpoint-accounting.test.ts',
'test/integration/lbug-rel-copy-page-padding.test.ts',
'test/integration/lbug-incremental-search.test.ts',
// Retained identities and reader metadata must agree on every native platform.
'test/integration/incremental-method-integrity.test.ts',
'test/integration/go-incremental-method-owners.test.ts',
'test/integration/query-symbol-integrity.test.ts',
'test/integration/mcp-index-generation.test.ts',
// A real refresh worker must preserve active readers or report native contention.
'test/integration/mcp-refresh-index-e2e.test.ts',
'test/integration/fts-repair-warm-session.test.ts',
'test/unit/lbug-pool-pinning.test.ts',
'test/unit/hybrid-search.test.ts',
@ -270,8 +278,10 @@ const LBUG_NATIVE = [
// process spawning, which behaves differently across platforms (shell
// quoting, path resolution, signal handling)
const SPAWN_CLI = [
'test/integration/resolvers/java-javac-overloads.test.ts',
// Real shard/blob/JSON roundtrip: path identities and child invocation vary by OS.
'test/unit/shard-timings-roundtrip.test.ts',
'test/unit/run-cross-platform.test.ts',
'test/integration/cli-e2e.test.ts',
'test/integration/cli-limit-e2e.test.ts',
'test/integration/hooks-e2e.test.ts',
@ -351,6 +361,7 @@ const WORKER_THREADS = [
'test/integration/cfg/adonis-taint.test.ts',
'test/integration/function-local-identity.test.ts',
'test/integration/java-imported-return-types.test.ts',
'test/integration/resolvers/java-overload-jls.test.ts',
'test/unit/community-processor.test.ts',
'test/integration/worker-pool.test.ts',
'test/integration/class-expression-owners.test.ts',
@ -397,6 +408,8 @@ const FILESYSTEM = [
'test/integration/filesystem-walker.test.ts',
'test/integration/analyze-directory-symlinks.test.ts',
'test/integration/watch-filesystem.test.ts',
'test/unit/watch-external-config.test.ts',
'test/integration/server-route-hook-config.test.ts',
'test/integration/markdown-processor-crlf.test.ts',
'test/integration/ignore-and-skip-e2e.test.ts',
// Pins that the bridge pairing verdict is measured before the database is

View file

@ -0,0 +1,321 @@
/**
* Reproducible scaling probe for the per-file owner-edge guard.
*
* Run alone, after a build, from gitnexus/:
* node --expose-gc --import tsx scripts/owner-edge-guard-scaling.ts > /tmp/owner-edge-scaling.json
* --operations-only skips timings; --sizes=1024,2048,... --samples=11 --warmup=3 override defaults.
*
* Timings include guard lifecycle and appending preconstructed rows, excluding
* fixture creation, prefix copies, assertions, and explicit GC. Prefix-isolation
* timings call the guard directly on a fixed 16-member slice. Proxy counters run
* separately: they are deterministic regression checks, never timed thresholds.
*/
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { readFileSync } from 'node:fs';
import { arch, cpus, platform, release } from 'node:os';
import { performance } from 'node:perf_hooks';
import { parseArgs } from 'node:util';
import { SupportedLanguages } from 'gitnexus-shared';
import {
createOwnerEdgeGuard,
dropDanglingInFileOwnerEdges,
} from '../src/core/ingestion/workers/owner-edge-guard.js';
import type { ParseWorkerResult } from '../src/core/ingestion/workers/parse-worker.js';
const source = readFileSync(
new URL('../src/core/ingestion/workers/owner-edge-guard.ts', import.meta.url),
);
type Slice = Pick<ParseWorkerResult, 'nodes' | 'relationships'>;
type Rel = Slice['relationships'][number];
type Shape = 'member-free' | 'clean' | 'dangling' | 'predefined' | 'duplicate' | 'owner-last';
type FileFixture = Slice & { path: string; expected: Rel[] };
type Reads = { nodes: number; relationships: number; writes: number };
const { values } = parseArgs({
options: {
sizes: { type: 'string', default: '1024,2048,4096,8192,16384,32768,65536' },
samples: { type: 'string', default: '11' },
warmup: { type: 'string', default: '3' },
'operations-only': { type: 'boolean', default: false },
},
});
const sizes = values.sizes.split(',').map(Number);
const samples = Number(values.samples);
const warmup = Number(values.warmup);
assert(
sizes.length >= 2 &&
sizes.every((n, i) => Number.isSafeInteger(n) && n > 0 && (!i || n > sizes[i - 1])),
);
assert(Number.isSafeInteger(samples) && samples >= 3);
assert(Number.isSafeInteger(warmup) && warmup >= 1);
const edge = (type: Rel['type'], sourceId: string, targetId: string): Rel => ({
id: `${type}:${sourceId}->${targetId}`,
sourceId,
targetId,
type,
confidence: 1,
reason: '',
});
function fixture(shape: Shape, count: number, file: number): FileFixture {
// Fixed-width ids keep the string-length dimension constant across sizes.
const path = `src/file-${String(file).padStart(8, '0')}.ts`;
const owner = `Class:${path}:Owner`;
const result: FileFixture = { path, nodes: [], relationships: [], expected: [] };
const addNode = (id: string, label: string): void => {
result.nodes.push({
id,
label,
properties: {
name: id,
filePath: path,
startLine: 0,
endLine: 0,
language: SupportedLanguages.TypeScript,
isExported: false,
},
});
};
if (shape === 'clean') addNode(owner, 'Class');
for (let i = 0; i < count; i++) {
const target = `Method:${path}:member-${String(i).padStart(8, '0')}`;
const defined = edge('DEFINES', `File:${path}`, target);
const member = edge(i % 2 ? 'HAS_PROPERTY' : 'HAS_METHOD', owner, target);
addNode(target, 'Method');
if (shape === 'member-free' || shape === 'predefined') result.relationships.push(defined);
if (shape !== 'member-free') result.relationships.push(member);
if (shape === 'duplicate') result.relationships.push(edge('HAS_PROPERTY', owner, target));
result.expected.push(shape === 'clean' || shape === 'owner-last' ? member : defined);
}
if (shape === 'owner-last') addNode(owner, 'Class');
return result;
}
function files(shape: Shape, count: number, perFile: number): FileFixture[] {
const result: FileFixture[] = [];
for (let i = 0; i < count; i += perFile)
result.push(fixture(shape, Math.min(perFile, count - i), i / perFile));
return result;
}
function append(target: Slice, file: Slice): void {
for (const node of file.nodes) target.nodes.push(node);
for (const rel of file.relationships) target.relationships.push(rel);
}
function verify(result: Slice, expected: Rel[], nodeCount: number): void {
assert.equal(result.nodes.length, nodeCount);
assert.deepEqual(result.relationships, expected);
}
/** Count numeric array accesses only while the production guard runs. */
function operations(input: FileFixture[], prefix: Slice): Reads {
const raw: Slice = { nodes: prefix.nodes.slice(), relationships: prefix.relationships.slice() };
let nodeStart = raw.nodes.length;
let relStart = raw.relationships.length;
const reads: Reads = { nodes: 0, relationships: 0, writes: 0 };
const watched = <T>(array: T[], kind: 'nodes' | 'relationships'): T[] =>
new Proxy(array, {
get(target, property, receiver) {
if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) {
assert(
Number(property) >= (kind === 'nodes' ? nodeStart : relStart),
`rescanned previous ${kind}`,
);
reads[kind]++;
}
return Reflect.get(target, property, receiver);
},
set(target, property, value, receiver) {
if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) {
assert(
Number(property) >= (kind === 'nodes' ? nodeStart : relStart),
`rewrote previous ${kind}`,
);
reads.writes++;
}
return Reflect.set(target, property, value, receiver);
},
});
const guarded = {
nodes: watched(raw.nodes, 'nodes'),
relationships: watched(raw.relationships, 'relationships'),
};
const guard = createOwnerEdgeGuard(guarded);
for (const file of input) {
guard.open(file.path); // Settles the previous slice before the boundaries advance.
nodeStart = raw.nodes.length;
relStart = raw.relationships.length;
append(raw, file); // Fixture appends bypass the access counters.
}
guard.settle();
const settledReads = { ...reads };
guard.settle();
assert.deepEqual(reads, settledReads, 'settling twice rescans a file');
const n = input.reduce((sum, file) => sum + file.nodes.length, 0);
const e = input.reduce((sum, file) => sum + file.relationships.length, 0);
assert(reads.nodes <= n, 'more than one node pass');
assert(reads.relationships <= 3 * e, 'more than three relationship passes');
assert(reads.writes <= e, 'more than one compaction pass');
verify(
raw,
[...prefix.relationships, ...input.flatMap((file) => file.expected)],
prefix.nodes.length + n,
);
return reads;
}
function trial(input: FileFixture[], prefix: Slice, direct: boolean, expected: Rel[]): number {
const result: Slice = {
nodes: prefix.nodes.slice(),
relationships: prefix.relationships.slice(),
};
const nodeStart = result.nodes.length;
const relStart = result.relationships.length;
if (direct) append(result, input[0]);
globalThis.gc?.();
const start = performance.now();
if (direct) {
dropDanglingInFileOwnerEdges(result, input[0].path, nodeStart, relStart);
} else {
const guard = createOwnerEdgeGuard(result);
for (const file of input) {
guard.open(file.path);
append(result, file);
}
guard.settle();
}
const elapsed = performance.now() - start;
verify(
result,
expected,
prefix.nodes.length + input.reduce((sum, file) => sum + file.nodes.length, 0),
);
return elapsed;
}
const percentile = (values: number[], fraction: number): number => {
const sorted = [...values].sort((a, b) => a - b);
return sorted[Math.ceil(fraction * sorted.length) - 1];
};
function measure(shape: Shape, count: number, perFile: number, prefixCount = 0) {
const input = files(shape, count, perFile);
const prefix = prefixCount
? fixture('member-free', prefixCount, 99999999)
: { nodes: [], relationships: [] };
const nodes = input.reduce((sum, file) => sum + file.nodes.length, 0);
const relationships = input.reduce((sum, file) => sum + file.relationships.length, 0);
const counts = operations(input, prefix);
const direct = prefixCount > 0;
const samplesMs: number[] = [];
if (!values['operations-only']) {
for (let i = 0; i < warmup + samples; i++) {
const freshInput = files(shape, count, perFile);
const expected = [...prefix.relationships, ...freshInput.flatMap((file) => file.expected)];
const elapsed = trial(freshInput, prefix, direct, expected);
if (i >= warmup) samplesMs.push(elapsed);
}
}
const medianMs = samplesMs.length ? percentile(samplesMs, 0.5) : null;
return {
shape,
members: count,
files: input.length,
prefixRows: prefix.nodes.length + prefix.relationships.length,
nodes,
relationships,
indexedReads: counts,
samplesMs,
medianMs,
p95Ms: samplesMs.length ? percentile(samplesMs, 0.95) : null,
nsPerRow: medianMs === null ? null : (medianMs * 1e6) / (nodes + relationships),
};
}
function slope(points: { x: number; y: number }[]): number | null {
if (points.length < 2 || points.some(({ x, y }) => x <= 0 || y <= 0)) return null;
const xs = points.map(({ x }) => Math.log(x));
const ys = points.map(({ y }) => Math.log(y));
const meanX = xs.reduce((a, b) => a + b, 0) / xs.length;
const meanY = ys.reduce((a, b) => a + b, 0) / ys.length;
return (
xs.reduce((sum, x, i) => sum + (x - meanX) * (ys[i] - meanY), 0) /
xs.reduce((sum, x) => sum + (x - meanX) ** 2, 0)
);
}
const cases: ReturnType<typeof measure>[] = [];
const scaling: { shape: Shape; distribution: string; logLogSlope: number | null }[] = [];
const shapes: Shape[] = [
'member-free',
'clean',
'dangling',
'predefined',
'duplicate',
'owner-last',
];
// Alternate ascending/descending sizes to avoid tying every large case to late-run thermal drift.
for (const [shapeIndex, shape] of shapes.entries()) {
for (const distribution of ['one-file', 'many-files']) {
const rows = (shapeIndex % 2 ? [...sizes].reverse() : sizes).map((size) =>
measure(shape, size, distribution === 'one-file' ? size : 16),
);
cases.push(...rows);
scaling.push({
shape,
distribution,
logLogSlope: values['operations-only']
? null
: slope(
rows.flatMap((row) =>
row.medianMs === null ? [] : [{ x: row.nodes + row.relationships, y: row.medianMs }],
),
),
});
process.stderr.write(`Validated ${shape} / ${distribution}\n`);
}
}
const prefixIsolation = sizes.map((size) => measure('dangling', 16, 16, size));
process.stdout.write(
`${JSON.stringify(
{
schema: 'gitnexus.owner-edge-guard-scaling/v1',
recordedAt: new Date().toISOString(),
environment: {
node: process.version,
v8: process.versions.v8,
platform: platform(),
arch: arch(),
release: release(),
cpu: cpus()[0]?.model,
explicitGc: typeof globalThis.gc === 'function',
processMaxRssKiB: process.resourceUsage().maxRSS,
},
sourceSha256: createHash('sha256').update(source).digest('hex'),
config: { sizes, samples, warmup, operationsOnly: values['operations-only'] },
methodology: {
timed:
'guard lifecycle + append preconstructed rows; prefixIsolation times only the direct guard',
excluded:
'fresh fixture generation for every sample, prefix copies, assertions, explicit GC, Proxy instrumentation',
statistics:
'nearest-rank median/p95; least-squares slope of log(median ms) versus log(active rows)',
operations:
'at most one indexed node read, three indexed relationship reads, one relationship write per active row; no earlier-slice indexed accesses',
limits:
'Kernel benchmark, not end-to-end indexing throughput. Timings and slopes are descriptive, not asymptotic proof. maxRSS covers all fixtures and validation and is not guard peak memory. Set operations assume average constant-time hashing; string work scales with id lengths.',
memory:
'Live guard scratch is O(nodes + edges) of the largest active file; output accumulation and delayed garbage collection are excluded from that bound.',
},
cases,
scaling,
prefixIsolation,
},
null,
2,
)}\n`,
);

View file

@ -0,0 +1,264 @@
/**
* Generated, non-confidential #3527 probe. Keeps its indexes and raw responses.
* Usage: node scripts/repro-3527.mjs <built-gitnexus-package> <new-output-dir>
* Exit 1 means an integrity/coverage failure; exit 2 means the probe could not run.
* This does not claim to reproduce the original private macOS checkout.
*/
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import fs from 'node:fs';
import { createRequire } from 'node:module';
import path from 'node:path';
const [runtimeArg, outputArg] = process.argv.slice(2);
assert(runtimeArg && outputArg, 'Supply a built GitNexus package and a NEW output directory');
const runtime = fs.realpathSync(runtimeArg);
const output = path.resolve(outputArg);
fs.mkdirSync(output);
const primary = path.join(output, 'primary');
const repo = path.join(output, 'worktree');
fs.mkdirSync(primary);
const requireRuntime = createRequire(path.join(runtime, 'package.json'));
const lbug = requireRuntime('@ladybugdb/core');
const env = {
PATH: process.env.PATH,
HOME: process.env.HOME,
LANG: 'C.UTF-8',
TZ: 'UTC',
GIT_CONFIG_GLOBAL: '/dev/null',
GIT_CONFIG_NOSYSTEM: '1',
GIT_AUTHOR_NAME: 'fixture',
GIT_AUTHOR_EMAIL: 'fixture@example.com',
GIT_COMMITTER_NAME: 'fixture',
GIT_COMMITTER_EMAIL: 'fixture@example.com',
GITNEXUS_HOME: path.join(output, 'home'),
GITNEXUS_MEMORY: 'off',
NODE_OPTIONS: '--max-old-space-size=2048',
GITNEXUS_LBUG_BUFFER_POOL_SIZE: String(512 * 1024 ** 2),
GITNEXUS_WORKER_POOL_SIZE: '1',
GITNEXUS_PARSE_CHUNK_CONCURRENCY: '1',
GITNEXUS_LBUG_EXTENSION_INSTALL: 'load-only',
};
const expected = {};
const report = {
runtime,
node: process.version,
engine: lbug.VERSION,
platform: `${process.platform}-${process.arch}`,
expectedMethods: 4480,
expectedStructs: 4480,
queries: [],
native: [],
crossLanguageMethodPaths: 0,
nulStructNames: 0,
witnesses: [],
};
const save = (name, value) =>
fs.writeFileSync(path.join(output, name), JSON.stringify(value, null, 2) + '\n');
const git = (args, cwd = primary) => execFileSync('git', args, { cwd, env, stdio: 'pipe' });
function cli(args, name) {
const stderr = fs.openSync(path.join(output, name + '.stderr.txt'), 'w');
try {
const text = execFileSync(
process.execPath,
[path.join(runtime, 'dist/cli/index.js'), ...args],
{
cwd: repo,
env,
encoding: 'utf8',
stdio: ['ignore', 'pipe', stderr],
maxBuffer: 32 * 1024 ** 2,
timeout: 300_000,
},
);
fs.writeFileSync(path.join(output, name + '.txt'), text);
return text;
} finally {
fs.closeSync(stderr);
}
}
function source(kind, batch, edit = false) {
const python = kind === 'python';
const file = python
? `checks/test_integrity_batch_${batch}.py`
: `Sources/Features/Component${batch}/IntegritySentinels_${batch}.swift`;
const lines = python ? [`class IntegrityBatch${batch}:`] : [];
for (let i = 0; i < 64; i++) {
const name = python
? `test_mixed_integrity_${batch}_${i}`
: `MixedIntegritySentinel${batch}_${i}`;
const startLine = lines.length + 1;
const marker = `mixed integrity sentinel ${batch}_${i}${edit && i === 0 ? ' changed' : ''}`;
const block = python
? [` def ${name}(self):`, ` return "${marker}"`]
: [`struct ${name} {`, ` let marker = "${marker}"`, '}'];
const id = python
? `Method:${file}:IntegrityBatch${batch}.${name}#0`
: `Struct:${file}:${name}`;
expected[id] = { id, name, filePath: file, startLine, endLine: startLine + block.length - 1 };
lines.push(...block, '');
}
const root = fs.existsSync(repo) ? repo : primary;
fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
fs.writeFileSync(path.join(root, file), lines.join('\n'));
expected['File:' + file] = { id: 'File:' + file, name: path.basename(file), filePath: file };
}
function mismatches(rows) {
return rows.filter(
(row) =>
!expected[row.id] ||
Object.keys(expected[row.id]).some((k) => row[k] !== expected[row.id][k]),
);
}
async function nativeChecks() {
const entry = JSON.parse(fs.readFileSync(path.join(output, 'home/registry.json'))).find(
(entry) => fs.realpathSync(entry.path) === fs.realpathSync(repo),
);
assert(entry, 'The exact worktree must be registered');
const meta = JSON.parse(fs.readFileSync(path.join(entry.storagePath, 'gitnexus.json')));
const dbPath = meta.graphPath || path.join(entry.storagePath, 'lbug');
report.dbPath = dbPath;
report.lastCommit = meta.lastCommit;
assert.equal(meta.lastCommit, git(['rev-parse', 'HEAD'], repo).toString().trim());
const db = new lbug.Database(dbPath, 512 * 1024 ** 2, false, true);
const conn = new lbug.Connection(db);
const fields =
'n.id AS id,n.name AS name,n.filePath AS filePath,n.startLine+1 AS startLine,n.endLine+1 AS endLine';
async function read(query, params) {
const result = params
? await conn.execute(await conn.prepare(query), params)
: await conn.query(query);
try {
return await result.getAll();
} finally {
await result.close();
}
}
async function check(rows, ids, stage) {
const bad = mismatches(rows);
const found = new Set(rows.map((row) => row.id));
const missing = ids.filter((id) => !found.has(id));
const duplicates = rows.length - found.size;
report.crossLanguageMethodPaths += bad.filter(
(row) => row.id?.startsWith('Method:') && row.filePath?.includes('.swift'),
).length;
report.nulStructNames += bad.filter(
(row) => row.id?.startsWith('Struct:') && row.name?.includes('\0'),
).length;
report.native.push({
stage,
rows: rows.length,
bad: bad.length,
missing: missing.length,
duplicates,
});
if ((bad.length || missing.length || duplicates) && report.witnesses.length < 8) {
const uid = missing[0] || bad.find((row) => expected[row.id])?.id;
const primaryKey = uid
? await read(`MATCH(n:${uid.split(':')[0]} {id:$uid}) RETURN ${fields}`, { uid })
: [];
report.witnesses.push({
stage,
bad: bad.slice(0, 3),
missing: missing.slice(0, 3),
uid,
primaryKey,
});
}
}
try {
const methods = Object.keys(expected).filter((id) => id.startsWith('Method:'));
const structs = Object.keys(expected).filter((id) => id.startsWith('Struct:'));
for (const [table, ids] of [
['Method', methods],
['Struct', structs],
]) {
await check(await read(`MATCH(n:${table}) RETURN ${fields}`), ids, table + '-all');
}
const statement = await conn.prepare(
`MATCH(n) WHERE n.id IN $ids RETURN ${fields} ORDER BY startLine,id`,
);
for (let i = 0; i < 150; i++) {
const ids = [methods[(i * 29) % methods.length], structs[(i * 67) % structs.length]];
if (i % 3) ids.splice(i % 2, 1);
const result = await conn.execute(statement, { ids });
let rows;
try {
rows = await result.getAll();
} finally {
await result.close();
}
await check(rows, ids, 'mixed-' + i);
}
} finally {
await conn.close();
await db.close();
}
}
try {
for (let i = 0; i < 70; i++) for (const kind of ['python', 'swift']) source(kind, i);
fs.writeFileSync(path.join(primary, '.gitignore'), '.gitnexus/\n');
git(['init', '-b', 'main']);
git(['add', '.']);
git(['-c', 'commit.gpgsign=false', 'commit', '-m', 'mixed language fixture']);
git(['worktree', 'add', '-b', 'verification', repo]);
cli(['analyze', '--index-only', '.'], 'baseline-analyze');
for (let round = 0; round < 3; round++) {
source('python', round, true); // Leave every Swift file untouched.
git(['add', '.'], repo);
git(['-c', 'commit.gpgsign=false', 'commit', '-m', `incremental edit ${round}`], repo);
const name = `round-${round}-analyze`;
const analyze = cli(['analyze', '--index-only', '.'], name);
assert(
!/switching to a full DB write|escalated/i.test(
analyze + fs.readFileSync(path.join(output, name + '.stderr.txt')),
),
'Probe requires selective writeback',
);
cli(['status'], `round-${round}-status`);
const terms = [
'mixed integrity sentinel',
'test_mixed_integrity',
'MixedIntegritySentinel',
'integrity batch',
];
for (const [i, term] of terms.entries()) {
const text = cli(['query', term, '--repo', repo], `round-${round}-query-${i}`);
const result = JSON.parse(text.slice(text.indexOf('{')));
const rows = [...(result.process_symbols ?? []), ...(result.definitions ?? [])];
const relevant = rows.filter((row) => /^(Method|Struct|File):/.test(row.id));
report.queries.push({
round,
term,
error: result.error,
partial: result.partial,
rows: rows.length,
methods: rows.filter((r) => r.id?.startsWith('Method:')).length,
structs: rows.filter((r) => r.id?.startsWith('Struct:')).length,
bad: mismatches(relevant),
});
}
save('report.json', report);
}
save('expected.json', expected);
await nativeChecks();
report.passed =
report.queries.every((q) => !q.error && !q.partial && q.rows > 0 && !q.bad.length) &&
report.queries.some((q) => q.methods > 0) &&
report.queries.some((q) => q.structs > 0) &&
report.native.every((q) => !q.bad && !q.missing && !q.duplicates);
save('report.json', report);
console.log(
JSON.stringify({
engine: report.engine,
passed: report.passed,
report: path.join(output, 'report.json'),
}),
);
process.exitCode = report.passed ? 0 : 1;
} catch (error) {
report.probeError = String(error);
save('report.json', report);
console.error(error);
process.exitCode = 2;
}

View file

@ -74,13 +74,18 @@ const reportArgs = reportName
: [];
try {
execFileSync(
'npx',
['vitest', 'run', ...ALL_CROSS_PLATFORM, ...(shardArg ? [shardArg] : []), ...reportArgs],
process.execPath,
[
fileURLToPath(new URL('../node_modules/vitest/vitest.mjs', import.meta.url)),
'run',
...ALL_CROSS_PLATFORM,
...(shardArg ? [shardArg] : []),
...reportArgs,
],
{
cwd: ROOT,
stdio: 'inherit',
timeout: timeoutMs,
shell: true,
},
);
} catch (err) {

View file

@ -87,6 +87,7 @@ Generates repository documentation from the knowledge graph using an LLM. HTTP p
| `--concurrency <n>` | Parallel LLM calls (default: 3) |
| `--timeout <seconds>` | LLM request timeout in seconds (default: disabled) |
| `--retries <n>` | Max LLM retry attempts per request (default: 3) |
| `--max-completion-tokens <n>` | Max completion tokens per LLM request (default: 16384). `GITNEXUS_MAX_COMPLETION_TOKENS` overrides `maxCompletionTokens` in `~/.gitnexus/config.json`; the flag overrides both. |
| `--lang <lang>` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) |
| `--gist` | Publish wiki as a public GitHub Gist |

View file

@ -30,7 +30,9 @@
import fs from 'node:fs';
import path from 'node:path';
import { readRepoControlFile } from '../config/repo-control-file.js';
import { MAX_REPO_CONTROL_FILE_BYTES, readRepoControlFile } from '../config/repo-control-file.js';
import { logger } from '../core/logger.js';
import { normalizeRouterAliases } from '../core/ingestion/languages/go/route-config.js';
import {
InvalidBranchError,
sanitizeDetectedBranch,
@ -64,7 +66,8 @@ type ValueKind =
| 'numeric-string'
| 'embeddings'
| 'branch'
| 'path';
| 'path'
| 'router-aliases';
interface KeySpec {
/** The `AnalyzeOptions` field this config key normalizes into. */
@ -116,6 +119,13 @@ const KEY_SPECS: Record<string, KeySpec> = {
// built-in convention set, is otherwise invisible to route_map consumers.
// Listing it here adds it to the cross-file consumer scan.
fetchWrappers: { target: 'fetchWrappers', kind: 'string-array' },
// Go router types reached through an in-house wrapper package that are the
// gin/echo engine (`type Engine = gin.Engine`). The indexer cannot read the
// dependency's source, so the equivalence is declared, never guessed.
goRouterAliases: { target: 'routeFrameworkAliases', kind: 'router-aliases' },
// Go functions that wrap a route handler (`common.Wrapper(api.CreateJob)`);
// the wrapped first argument becomes the route's handler.
goHandlerWrappers: { target: 'routeHandlerWrappers', kind: 'string-array' },
// Explicit local Actuator snapshot input (#2418). The path itself is safe in
// project config; payload contents are never copied into the graph wholesale.
springActuator: { target: 'springActuator', kind: 'path' },
@ -178,8 +188,13 @@ export function validateBranchName(value: string, source: string): string {
}
}
const normalizeValue = (kind: ValueKind, value: unknown, key: string): unknown => {
const source = `${GITNEXUS_RC_FILENAME} "${key}"`;
const normalizeValue = (
kind: ValueKind,
value: unknown,
key: string,
configSource: string,
): unknown => {
const source = `${configSource} "${key}"`;
switch (kind) {
case 'boolean':
if (typeof value !== 'boolean') {
@ -301,6 +316,12 @@ const normalizeValue = (kind: ValueKind, value: unknown, key: string): unknown =
`${source} must be a boolean or a non-negative integer (node cap; 0 disables the cap).`,
);
}
case 'router-aliases':
try {
return normalizeRouterAliases(value, source);
} catch (error) {
throw new GitNexusRcError((error as Error).message);
}
default:
// Exhaustive — kept for forward-compat if a new kind is added.
throw new GitNexusRcError(`${source}: unsupported config value kind.`);
@ -314,7 +335,7 @@ const normalizeValue = (kind: ValueKind, value: unknown, key: string): unknown =
*/
const normalizeLevel = (
obj: Record<string, unknown>,
{ allowNestedKey }: { allowNestedKey: boolean },
{ allowNestedKey, source }: { allowNestedKey: boolean; source: string },
): Partial<AnalyzeOptions> => {
const out: Partial<AnalyzeOptions> = {};
const setBy = new Map<keyof AnalyzeOptions, string>();
@ -326,19 +347,17 @@ const normalizeLevel = (
// truthy) and would slip past `if (!spec)`, hitting the wrong error branch
// instead of the documented "Unknown key" message (#1996 tri-review P3).
if (!Object.hasOwn(KEY_SPECS, key)) {
throw new GitNexusRcError(
`Unknown key "${key}" in ${GITNEXUS_RC_FILENAME}. ${ALLOWED_KEYS_HINT}`,
);
throw new GitNexusRcError(`Unknown key "${key}" in ${source}. ${ALLOWED_KEYS_HINT}`);
}
const spec = KEY_SPECS[key];
const prev = setBy.get(spec.target);
if (prev && prev !== key) {
throw new GitNexusRcError(
`${GITNEXUS_RC_FILENAME}: "${prev}" and "${key}" both configure the same option; set only one.`,
`${source}: "${prev}" and "${key}" both configure the same option; set only one.`,
);
}
setBy.set(spec.target, key);
(out as Record<string, unknown>)[spec.target] = normalizeValue(spec.kind, value, key);
(out as Record<string, unknown>)[spec.target] = normalizeValue(spec.kind, value, key, source);
}
return out;
@ -356,11 +375,11 @@ export function loadAnalyzeConfig(repoRoot: string): Partial<AnalyzeOptions> | u
try {
raw = fs.readFileSync(filePath, 'utf-8');
} catch (err) {
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return undefined;
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return withEnvConfig(undefined);
throw new GitNexusRcError(`Could not read ${GITNEXUS_RC_FILENAME}: ${(err as Error).message}`);
}
return parseAnalyzeConfig(raw);
return withEnvConfig(parseAnalyzeConfig(raw));
}
/** Load `.gitnexusrc` through the strict bounded reader used by watch mode. */
@ -373,10 +392,76 @@ export async function loadAnalyzeConfigStrict(
} catch (err) {
throw new GitNexusRcError(`Could not read ${GITNEXUS_RC_FILENAME}: ${(err as Error).message}`);
}
return raw === null ? undefined : parseAnalyzeConfig(raw);
return withEnvConfig(raw === null ? undefined : parseAnalyzeConfig(raw));
}
function parseAnalyzeConfig(rawInput: string): Partial<AnalyzeOptions> {
/** Environment variable naming an analyze config file kept outside the repository. */
export const GITNEXUS_RC_ENV = 'GITNEXUS_RC';
/**
* Read an explicitly named config file through one descriptor, never by path
* after the check, so it cannot be swapped between the check and the read.
*
* O_NONBLOCK makes opening a FIFO return at once instead of blocking in
* open(2) until a writer appears; fstat then rejects it. It changes nothing
* for a regular file (same reasoning as `readNestedGitignore`). The read stops
* one byte past the limit, so a file that grows after fstat is rejected
* before it is decoded rather than read to EOF.
*/
function readBoundedConfigFile(filePath: string): string {
const fd = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NONBLOCK ?? 0));
try {
const stat = fs.fstatSync(fd);
if (!stat.isFile()) throw new Error('not a regular file');
const tooLarge = () => new Error(`larger than ${MAX_REPO_CONTROL_FILE_BYTES} bytes`);
if (stat.size > MAX_REPO_CONTROL_FILE_BYTES) throw tooLarge();
const buffer = Buffer.alloc(MAX_REPO_CONTROL_FILE_BYTES + 1);
let length = 0;
for (;;) {
const read = fs.readSync(fd, buffer, length, buffer.length - length, null);
if (read === 0) break;
length += read;
if (length > MAX_REPO_CONTROL_FILE_BYTES) throw tooLarge();
}
return buffer.toString('utf-8', 0, length);
} finally {
fs.closeSync(fd);
}
}
/**
* Layer the `GITNEXUS_RC` file over the repository's `.gitnexusrc`, key by key.
* It lets a checkout carry analyze config without a tracked or untracked file
* in the working tree. Being an explicit request, a relative path, a missing
* file, or an oversized file is an error rather than a silent no-op.
*/
function withEnvConfig(
repoConfig: Partial<AnalyzeOptions> | undefined,
): Partial<AnalyzeOptions> | undefined {
const envPath = process.env[GITNEXUS_RC_ENV]?.trim();
if (!envPath) return repoConfig;
const label = `${GITNEXUS_RC_ENV} file "${envPath}"`;
if (!path.isAbsolute(envPath)) {
throw new GitNexusRcError(`${GITNEXUS_RC_ENV} must be an absolute path (got "${envPath}").`);
}
let raw: string;
try {
raw = readBoundedConfigFile(envPath);
} catch (err) {
throw new GitNexusRcError(`Could not read ${label}: ${(err as Error).message}`);
}
const envConfig = parseAnalyzeConfig(raw, label);
logger.debug(
{ file: envPath, keys: Object.keys(envConfig), repoKeys: Object.keys(repoConfig ?? {}) },
'analyze config: applied GITNEXUS_RC over .gitnexusrc',
);
return { ...repoConfig, ...envConfig };
}
function parseAnalyzeConfig(
rawInput: string,
source = GITNEXUS_RC_FILENAME,
): Partial<AnalyzeOptions> {
let raw = rawInput;
// Strip a leading UTF-8 BOM: Node's 'utf-8' decode keeps it, and JSON.parse
@ -390,25 +475,28 @@ function parseAnalyzeConfig(rawInput: string): Partial<AnalyzeOptions> {
parsed = JSON.parse(raw);
} catch (err) {
throw new GitNexusRcError(
`${GITNEXUS_RC_FILENAME} is not valid JSON: ${(err as Error).message}. ` +
`${source} is not valid JSON: ${(err as Error).message}. ` +
`Expected a JSON object such as {"defaultBranch": "develop", "skipContextFiles": true}.`,
);
}
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new GitNexusRcError(`${GITNEXUS_RC_FILENAME} must contain a JSON object.`);
throw new GitNexusRcError(`${source} must contain a JSON object.`);
}
const obj = parsed as Record<string, unknown>;
const flat = normalizeLevel(obj, { allowNestedKey: true });
const flat = normalizeLevel(obj, { allowNestedKey: true, source });
let nested: Partial<AnalyzeOptions> = {};
if (Object.prototype.hasOwnProperty.call(obj, NESTED_KEY)) {
const nestedRaw = obj[NESTED_KEY];
if (nestedRaw === null || typeof nestedRaw !== 'object' || Array.isArray(nestedRaw)) {
throw new GitNexusRcError(`${GITNEXUS_RC_FILENAME} "${NESTED_KEY}" must be a JSON object.`);
throw new GitNexusRcError(`${source} "${NESTED_KEY}" must be a JSON object.`);
}
nested = normalizeLevel(nestedRaw as Record<string, unknown>, { allowNestedKey: false });
nested = normalizeLevel(nestedRaw as Record<string, unknown>, {
allowNestedKey: false,
source,
});
}
// Nested `analyze` block overrides flat top-level keys for the same option.

View file

@ -14,6 +14,8 @@
* back the other way put the two files, plus `core/run-analyze.ts`, in an
* import cycle. `analyze.ts` re-exports the type for existing importers.
*/
import type { RouteFrameworkAlias } from '../core/ingestion/language-provider.js';
export interface AnalyzeOptions {
/** Keep this repository current with serialized incremental refreshes. */
watch?: boolean;
@ -147,6 +149,16 @@ export interface AnalyzeOptions {
* outside the built-in convention still produces `route_map` consumers.
*/
fetchWrappers?: string[];
/**
* Router types reached through a wrapper package that are a framework's
* router root. Supplied via `.gitnexusrc` / `GITNEXUS_RC` `goRouterAliases`.
*/
routeFrameworkAliases?: RouteFrameworkAlias[];
/**
* Handler-wrapper callees whose first argument is the route handler.
* Supplied via `.gitnexusrc` / `GITNEXUS_RC` `goHandlerWrappers`.
*/
routeHandlerWrappers?: string[];
/**
* Explicit local Spring Boot Actuator snapshot input (#2418). Accepts a JSON
* bundle or a directory containing endpoint JSON files. Disabled by default.

View file

@ -1,6 +1,7 @@
/** Local incremental watch (`gitnexus analyze --watch`). Remote auto-sync lives in `auto-sync.ts`. */
import path from 'node:path';
import fs from 'node:fs/promises';
import { watchFile, unwatchFile } from 'node:fs';
import { watch, type FSWatcher } from 'chokidar';
import { createWatchIgnorePredicate } from '../config/ignore-service.js';
import {
@ -15,10 +16,12 @@ import type { AnalyzerRunnerIdentity } from '../storage/repo-manager.js';
import { ANALYZE_STORAGE_REQUIREMENTS, requireStoragePath } from '../storage/storage-resolver.js';
import { GITNEXUS_DIR } from '../storage/repo-meta.js';
import {
GITNEXUS_RC_ENV,
loadAnalyzeConfigStrict,
mergeAnalyzeOptions,
validateBranchName,
} from './analyze-config.js';
import { logger } from '../core/logger.js';
import type { AnalyzeOptions } from './analyze-options.js';
import { ensureHeap } from './analyze.js';
import { cliError, cliInfo, cliWarn } from './cli-message.js';
@ -60,8 +63,15 @@ function isIgnoreControlPath(filePath: string): boolean {
return normalized === '.gitignore' || normalized === '.gitnexusignore';
}
/**
* Queue entry for a change to the `GITNEXUS_RC` file. It lives outside the
* repository, so it has no repo-relative path; the leading NUL keeps it apart
* from every real path.
*/
export const EXTERNAL_CONFIG_WATCH_PATH = '\u0000GITNEXUS_RC';
function isConfigControlPath(filePath: string): boolean {
return posixWatchPath(filePath) === '.gitnexusrc';
return filePath === EXTERNAL_CONFIG_WATCH_PATH || posixWatchPath(filePath) === '.gitnexusrc';
}
function isAnalyzerOwnedWatchPath(filePath: string): boolean {
@ -202,6 +212,10 @@ export async function resolveWatchOptions(
maxProcessTraceDepth: processDetection.maxProcessTraceDepth,
maxEntryPointCandidates: processDetection.maxEntryPointCandidates,
fetchWrappers: merged.fetchWrappers,
routeHookConfig: {
routeFrameworkAliases: merged.routeFrameworkAliases,
routeHandlerWrappers: merged.routeHandlerWrappers,
},
skipAgentsMd: true,
skipSkills: true,
noStats: true,
@ -279,6 +293,8 @@ export async function startWatchFileLoop(
refresh: (paths: readonly string[]) => Promise<void>,
onError: WatchRefreshError,
onWatcherError: (error: unknown) => void = (error) => onError(error, []),
/** Absolute `GITNEXUS_RC` path; its edits, replacements and removal reload config. */
externalConfigPath?: string,
): Promise<WatchFileLoop> {
let ignorePath = await createWatchIgnorePredicate(repoPath);
let ignoreControlValid = true;
@ -366,6 +382,16 @@ export async function startWatchFileLoop(
let watcher: FSWatcher = createWatcher();
// Poll this one explicit path: stat follows symlinks, and continues observing
// the path after atomic saves, target deletion/restoration or link retargeting.
// Watching only the link's directory misses writes in the target directory.
const configPath =
externalConfigPath === undefined ? undefined : path.resolve(externalConfigPath);
const configChanged = () => {
logger.debug({ file: configPath }, 'watch: GITNEXUS_RC changed; reloading config');
queue.enqueue(EXTERNAL_CONFIG_WATCH_PATH);
};
// Chokidar emits `ready` once per instance and `add()` returns before the
// rescan it starts has finished, with no signal for that completion. A file
// an ignore-rule reload has just unignored is therefore still unregistered
@ -400,11 +426,13 @@ export async function startWatchFileLoop(
};
try {
if (configPath) watchFile(configPath, { interval: 100 }, configChanged);
await waitUntilReady(watcher);
await queue.runInitial();
} catch (error) {
closed = true;
await watcher.close();
if (configPath) unwatchFile(configPath, configChanged);
await queue.close();
throw error;
}
@ -414,6 +442,7 @@ export async function startWatchFileLoop(
close: async () => {
closed = true;
await watcher.close();
if (configPath) unwatchFile(configPath, configChanged);
await queue.close();
},
};
@ -497,7 +526,9 @@ export async function watchCommandWithRunnerIdentity(
repoPath,
debounceMs,
async (paths) => {
if (paths.some(isConfigControlPath) || !configControlValid) {
// Initial validation precedes watcher setup. Read again after it is
// armed so a change during startup cannot leave stale options live.
if (paths.length === 0 || paths.some(isConfigControlPath) || !configControlValid) {
const retryingInvalidConfig = !configControlValid;
try {
analyzeOptions = await resolveWatchOptions(
@ -578,6 +609,7 @@ export async function watchCommandWithRunnerIdentity(
);
stopWatching();
},
process.env[GITNEXUS_RC_ENV]?.trim() || undefined,
);
} catch (error) {
cliError(

View file

@ -1589,6 +1589,10 @@ const analyzeCommandImpl = async (
// Extra fetch-wrapper names from `.gitnexusrc` (#1589/#1852 residual);
// forwarded to the routes phase consumer scan.
fetchWrappers: options.fetchWrappers,
routeHookConfig: {
routeFrameworkAliases: options.routeFrameworkAliases,
routeHandlerWrappers: options.routeHandlerWrappers,
},
springActuatorPath: options.springActuator,
asyncApiSpecPath: options.asyncapiSpec,
// The CLI always process.exit()s after this returns (success path at the

View file

@ -114,6 +114,7 @@ const OPTION_DESCRIPTION_KEYS = {
'wiki|--concurrency <n>': 'help.option.wiki.concurrency',
'wiki|--timeout <seconds>': 'help.option.wiki.timeout',
'wiki|--retries <n>': 'help.option.wiki.retries',
'wiki|--max-completion-tokens <n>': 'help.option.wiki.maxCompletionTokens',
'wiki|--allow-insecure-connection <host>': 'help.option.wiki.allowInsecureConnection',
'wiki|--gist': 'help.option.wiki.gist',
'wiki|-v, --verbose': 'help.option.verbose',
@ -145,6 +146,7 @@ const OPTION_DESCRIPTION_KEYS = {
'impact|-l, --limit <n>': 'help.option.impact.limit',
'impact|--offset <n>': 'help.option.impact.offset',
'impact|--summary-only': 'help.option.impact.summaryOnly',
'impact|--max-tokens <n>': 'help.option.impact.maxTokens',
'cypher|-r, --repo <name>': 'help.option.repo.target',
'cypher|--branch <name>': 'help.option.branch',
'cypher|-l, --limit <n>': 'help.option.cypher.limit',

View file

@ -361,6 +361,8 @@ export const en = {
'help.option.wiki.concurrency': 'Parallel LLM calls (default: 3)',
'help.option.wiki.timeout': 'LLM request timeout in seconds (default: disabled)',
'help.option.wiki.retries': 'Max LLM retry attempts per request (default: 3)',
'help.option.wiki.maxCompletionTokens':
'Max completion tokens for each LLM request (default: 16384)',
'help.option.wiki.allowInsecureConnection':
'Allow exact host(s) for http:// LLM base URLs (comma-separated; HTTPS is preferred)',
'help.option.wiki.gist': 'Publish wiki as a public GitHub Gist after generation',
@ -386,9 +388,11 @@ export const en = {
'help.option.impact.depth': 'Max relationship depth (default: 3)',
'help.option.impact.includeTests': 'Include test files in results',
'help.option.impact.limit':
'Max symbols per depth level (default: 100); explicit --limit also caps affected processes/modules/routes',
'Max symbols per depth level in callgraph mode (default: 30, GITNEXUS_IMPACT_DEFAULT_LIMIT=0 for all); explicit --limit also caps affected processes/modules/routes',
'help.option.impact.offset': 'Skip N symbols per depth level for pagination',
'help.option.impact.summaryOnly': 'Return counts and risk only, omit symbol list',
'help.option.impact.maxTokens':
'Fit compact JSON into about N tokens; trim deepest callgraph symbols or fail if required content cannot fit. PDG/group results must fit intact',
'help.option.trace.fromUid': 'Source symbol UID (zero-ambiguity lookup)',
'help.option.trace.fromFile': 'Source file path to disambiguate common names',
'help.option.trace.toUid': 'Target symbol UID (zero-ambiguity lookup)',

View file

@ -329,6 +329,7 @@ export const zhCN = {
'help.option.wiki.concurrency': '并行 LLM 调用数(默认:3)',
'help.option.wiki.timeout': 'LLM 请求超时时间(秒,默认:禁用)',
'help.option.wiki.retries': '每个请求的最大 LLM 重试次数(默认:3)',
'help.option.wiki.maxCompletionTokens': '每次 LLM 请求的最大 completion token 数(默认:16384)',
'help.option.wiki.allowInsecureConnection':
'允许 http:// LLM base URL 使用的精确主机(逗号分隔;推荐使用 HTTPS)',
'help.option.wiki.gist': '生成后发布 Wiki 为公开 GitHub Gist',
@ -351,9 +352,11 @@ export const zhCN = {
'help.option.impact.depth': '最大关系遍历深度(默认:3)',
'help.option.impact.includeTests': '在结果中包含测试文件',
'help.option.impact.limit':
'每层深度最大符号数(默认:100);显式指定 --limit 时,也限制受影响流程/模块/路由数',
'调用图模式下每层深度最大符号数(默认:30,设 GITNEXUS_IMPACT_DEFAULT_LIMIT=0 返回全部);显式指定 --limit 时,也限制受影响流程/模块/路由数',
'help.option.impact.offset': '每层深度跳过 N 个符号(分页用)',
'help.option.impact.summaryOnly': '仅返回计数和风险等级,省略符号列表',
'help.option.impact.maxTokens':
'将紧凑 JSON 控制在约 N 个 token 内;调用图优先舍弃最深层符号,必要内容无法容纳时返回错误。PDG 和仓库组结果必须完整容纳',
'help.option.trace.fromUid': '源符号 UID(零歧义查找)',
'help.option.trace.fromFile': '源文件路径,用于消除常见名称歧义',
'help.option.trace.toUid': '目标符号 UID(零歧义查找)',

View file

@ -427,6 +427,10 @@ program
.option('--concurrency <n>', 'Parallel LLM calls (default: 3)', '3')
.option('--timeout <seconds>', 'LLM request timeout in seconds (default: disabled)')
.option('--retries <n>', 'Max LLM retry attempts per request (default: 3)')
.option(
'--max-completion-tokens <n>',
'Max completion tokens for each LLM request (default: 16384)',
)
.option(
'--allow-insecure-connection <host>',
'Allow exact host(s) for http:// LLM base URLs (comma-separated; HTTPS is preferred)',
@ -514,10 +518,14 @@ program
.option('--include-tests', 'Include test files in results')
.option(
'-l, --limit <n>',
'Max symbols per depth level (default: 100); explicit --limit also caps affected processes/modules/routes',
'Max symbols per depth level in callgraph mode (default: 30, GITNEXUS_IMPACT_DEFAULT_LIMIT=0 for all); explicit --limit also caps affected processes/modules/routes',
)
.option('--offset <n>', 'Skip N symbols per depth level for pagination')
.option('--summary-only', 'Return counts and risk only, omit symbol list')
.option(
'--max-tokens <n>',
'Fit compact JSON into about N tokens; trim deepest callgraph symbols or fail if required content cannot fit. PDG/group results must fit intact',
)
.action(createLbugLazyAction(() => import('./tool.js'), 'impactCommand'));
program

View file

@ -118,13 +118,23 @@ export const mcpCommand = async (options?: {
// stdout at module init, but transitive deps (pino, pino-pretty, the
// worker-thread transport) could in theory, and the import-closure
// regression test enforces the leaf invariant.
const [{ startMCPServer }, { LocalBackend }, { logger }, { createMcpRepositoryPolicy }] =
await Promise.all([
import('../mcp/server.js'),
import('../mcp/local/local-backend.js'),
import('../core/logger.js'),
import('../mcp/repository-policy.js'),
]);
const [
{ startMCPServer },
{ LocalBackend },
{ logger },
{ createMcpRepositoryPolicy },
{ resolveMcpRefreshLimits, resolveMcpRefreshMode },
] = await Promise.all([
import('../mcp/server.js'),
import('../mcp/local/local-backend.js'),
import('../core/logger.js'),
import('../mcp/repository-policy.js'),
import('../mcp/refresh-policy.js'),
]);
// Local servers only: `gitnexus serve` never reads these (#3524). A malformed
// value fails startup here, before any repository is opened.
const refreshIndex = resolveMcpRefreshMode();
if (refreshIndex) resolveMcpRefreshLimits();
// Missing-optional-grammar warnings are intentionally NOT emitted here.
// `gitnexus analyze` already warns at index time, filtered by the repo's
@ -174,6 +184,7 @@ export const mcpCommand = async (options?: {
host: options.host ?? '127.0.0.1',
authToken: resolveAuthToken(options.authToken, process.env),
repositoryPolicy,
refreshIndex,
});
} catch (err) {
logger.error(
@ -187,6 +198,6 @@ export const mcpCommand = async (options?: {
}
// Start MCP server (serves all repos, discovers new ones lazily)
await startMCPServer(backend, repositoryPolicy);
await startMCPServer(backend, repositoryPolicy, { refreshIndex });
void startMcpUpdateNotifier(logger).catch(() => {});
};

View file

@ -19,6 +19,12 @@ import { writeSync } from 'node:fs';
import { LocalBackend, VALID_NODE_LABELS } from '../mcp/local/local-backend.js';
import { cliErrorKey, cliWarnKey } from './cli-message.js';
import { formatDetectChangesResult } from './detect-changes-format.js';
import {
ImpactBudgetExceededError,
shapeImpactForAgent,
withImpactAgentDefaults,
} from '../mcp/impact-agent-output.js';
import { resolveMcpMaxTokens } from '../mcp/output-budget.js';
let _backend: LocalBackend | null = null;
@ -53,7 +59,9 @@ function output<T>(data: T, render?: (data: T) => string): void {
? render(data)
: typeof data === 'string'
? data
: JSON.stringify(data, null, 2);
: // Indent for a terminal only: agents and pipes re-read this output, and
// indentation made impact results ~45% larger with no added information.
JSON.stringify(data, null, process.stdout.isTTY ? 2 : undefined);
try {
writeSync(1, text + '\n');
} catch (err: any) {
@ -234,6 +242,7 @@ export async function impactCommand(
limit?: string;
offset?: string;
summaryOnly?: boolean;
maxTokens?: string;
},
): Promise<void> {
// A `--`-prefixed uid means Commander swallowed a following flag as the uid
@ -266,7 +275,8 @@ export async function impactCommand(
// rather than on a silently-dropped flag. A non-numeric `--line` parses to
// NaN, which the backend rejects as a non-positive integer (loud, not silent).
const parsedLine = options?.line !== undefined ? parseInt(options.line, 10) : undefined;
const result = await backend.callTool('impact', {
const maxTokens = resolveMcpMaxTokens('impact', { maxTokens: options?.maxTokens }, {});
const impactArgs = withImpactAgentDefaults({
target: target || undefined,
target_uid: options?.uid,
file_path: options?.file,
@ -285,6 +295,7 @@ export async function impactCommand(
offset: parsedOffset,
summaryOnly: options?.summaryOnly ?? undefined,
});
const result = await backend.callTool('impact', impactArgs);
// Client-side cap of the affected-list payload to --limit (parity with the
// other tool commands). The backend already paginates byDepth per level to
// the same limit, so byDepth needs no client-side re-slice.
@ -296,8 +307,15 @@ export async function impactCommand(
if (Array.isArray(result.affected_routes))
result.affected_routes = result.affected_routes.slice(0, parsedLimit);
}
output(result);
const shaped = shapeImpactForAgent(result, maxTokens);
// The budget is measured on compact JSON, even when stdout is a terminal.
output(shaped, maxTokens === undefined ? undefined : (data) => JSON.stringify(data));
} catch (err: unknown) {
if (err instanceof ImpactBudgetExceededError) {
output(err.toJSONText());
process.exitCode = 1;
return;
}
// Belt-and-suspenders: catch infrastructure failures (getBackend, callTool transport)
// The backend's impact() already returns structured errors for graph query failures
output({

View file

@ -47,6 +47,7 @@ export interface WikiCommandOptions {
retries?: string;
lang?: string;
allowInsecureConnection?: string;
maxCompletionTokens?: string;
}
function isLocalProvider(
@ -189,6 +190,7 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions)
let timeoutSeconds: number | undefined;
let retries: number | undefined;
let allowedInsecureHttpHosts: string[] | undefined;
let maxCompletionTokens: number | undefined;
try {
timeoutSeconds =
options?.timeout === undefined
@ -202,6 +204,12 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions)
options?.allowInsecureConnection === undefined
? undefined
: parseLLMAllowedInsecureHttpHosts(options.allowInsecureConnection);
maxCompletionTokens =
options?.maxCompletionTokens === undefined
? undefined
: parseIntegerOption(options.maxCompletionTokens, '--max-completion-tokens', {
minimum: 1,
});
} catch (error) {
console.log(` Error: ${(error as Error).message}\n`);
process.exitCode = 1;
@ -271,15 +279,23 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions)
options?.reasoningModel !== undefined
);
let llmConfig = await resolveLLMConfig({
model: options?.model,
baseUrl: options?.baseUrl,
apiKey: options?.apiKey,
provider: options?.provider,
apiVersion: options?.apiVersion,
isReasoningModel: options?.reasoningModel,
allowedInsecureHttpHosts,
});
let llmConfig;
try {
llmConfig = await resolveLLMConfig({
model: options?.model,
baseUrl: options?.baseUrl,
apiKey: options?.apiKey,
provider: options?.provider,
apiVersion: options?.apiVersion,
isReasoningModel: options?.reasoningModel,
allowedInsecureHttpHosts,
maxTokens: maxCompletionTokens,
});
} catch (error) {
console.log(` Error: ${(error as Error).message}\n`);
process.exitCode = 1;
return;
}
// Run interactive setup if no saved config and no CLI flags provided
// (even if env vars exist — let user explicitly choose their provider)
@ -687,7 +703,10 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions)
bar.stop();
const totalElapsed = ((Date.now() - t0) / 1000).toFixed(1);
console.log(`\n Wiki generated successfully (${totalElapsed}s)\n`);
const incomplete = !!continueResult.failedModules?.length;
console.log(
`\n ${incomplete ? 'Wiki generation incomplete' : 'Wiki generated successfully'} (${totalElapsed}s)\n`,
);
console.log(` Mode: ${continueResult.mode}`);
console.log(` Pages: ${continueResult.pagesGenerated}`);
console.log(` Output: ${wikiDir}`);
@ -698,6 +717,9 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions)
for (const mod of continueResult.failedModules) {
console.log(` - ${mod}`);
}
console.log(' Re-run to retry failed modules (pages will be regenerated).\n');
process.exitCode = 1;
return;
}
console.log('');
@ -714,7 +736,10 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions)
return;
}
console.log(`\n Wiki generated successfully (${elapsed}s)\n`);
const incomplete = !!result.failedModules?.length;
console.log(
`\n ${incomplete ? 'Wiki generation incomplete' : 'Wiki generated successfully'} (${elapsed}s)\n`,
);
console.log(` Mode: ${result.mode}`);
console.log(` Pages: ${result.pagesGenerated}`);
console.log(` Output: ${wikiDir}`);
@ -726,6 +751,9 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions)
console.log(` - ${mod}`);
}
console.log(' Re-run to retry failed modules (pages will be regenerated).');
console.log('');
process.exitCode = 1;
return;
}
console.log('');

View file

@ -15,7 +15,9 @@ export type AutoSyncAnalysisRunner = (
signal?: AbortSignal,
onCancellationRequested?: () => void,
concurrency?: number,
) => Promise<Pick<AnalyzeResult, 'stats'>>;
/** Worker progress until the run settles or cancellation begins. */
onProgress?: (percent: number, message: string) => void,
) => Promise<Pick<AnalyzeResult, 'stats' | 'graphWriteCollapsed'>>;
interface AnalysisWorker extends Pick<ChildProcess, 'send' | 'on'> {
stdout?: Pick<NodeJS.ReadableStream, 'resume'> | null;
@ -36,6 +38,8 @@ export class AutoSyncAnalysisError extends Error {
readonly code?: 'index-lock-timeout';
readonly retryable?: boolean;
readonly abandonedWorker: boolean;
/** Resolves on actual process exit when the request stopped waiting first. */
readonly workerExited?: Promise<void>;
constructor(
message: string,
@ -43,6 +47,7 @@ export class AutoSyncAnalysisError extends Error {
code?: 'index-lock-timeout';
retryable?: boolean;
abandonedWorker?: boolean;
workerExited?: Promise<void>;
} = {},
) {
super(message);
@ -50,6 +55,7 @@ export class AutoSyncAnalysisError extends Error {
this.code = options.code;
this.retryable = options.retryable;
this.abandonedWorker = options.abandonedWorker === true;
this.workerExited = options.workerExited;
}
}
@ -89,8 +95,8 @@ export function createAutoSyncAnalysisRunner(
overrides: Partial<AutoSyncAnalysisLaunchDeps> = {},
): AutoSyncAnalysisRunner {
const deps = { ...DEFAULT_DEPS, ...overrides };
return (repoPath, options, timeoutMs, signal, onCancellationRequested, concurrency) =>
new Promise<Pick<AnalyzeResult, 'stats'>>((resolve, reject) => {
return (repoPath, options, timeoutMs, signal, onCancellationRequested, concurrency, onProgress) =>
new Promise<Pick<AnalyzeResult, 'stats' | 'graphWriteCollapsed'>>((resolve, reject) => {
if (signal?.aborted) {
reject(new Error('Analysis cancelled.'));
return;
@ -135,7 +141,10 @@ export function createAutoSyncAnalysisRunner(
child.channel?.unref?.();
child.unref?.();
};
const settle = (error?: Error, result?: Pick<AnalyzeResult, 'stats'>) => {
const settle = (
error?: Error,
result?: Pick<AnalyzeResult, 'stats' | 'graphWriteCollapsed'>,
) => {
if (settled) return;
settled = true;
cleanup();
@ -163,12 +172,15 @@ export function createAutoSyncAnalysisRunner(
// child is deliberately left running rather than killed mid-write.
graceTimer = deps.setTimeoutFn(() => {
if (settled) return;
const workerExited = new Promise<void>((resolveExited) => {
child.on('exit', () => resolveExited());
});
releaseChild();
settle(
new AutoSyncAnalysisError(
`${error.message} The analyze worker did not exit within ${deps.cancelGraceMs}ms; ` +
'it was left running so its native work is not interrupted.',
{ abandonedWorker: true, retryable: true },
{ abandonedWorker: true, retryable: true, workerExited },
),
);
}, deps.cancelGraceMs);
@ -183,7 +195,19 @@ export function createAutoSyncAnalysisRunner(
child.on('message', (message: WorkerMessage) => {
// Once timeout/cancellation requested shutdown, its reason owns the
// result. A terminal IPC can already be queued behind cancellation.
if (message.type === 'progress' || terminalOutcome || terminationError) return;
if (terminalOutcome || terminationError) return;
if (message.type === 'progress') {
// The worker also sends its log lines as progress with percent -1;
// those are not progress. A listener's failure must not decide the
// run's outcome.
if (message.percent < 0) return;
try {
onProgress?.(message.percent, message.message);
} catch {
// ignored
}
return;
}
terminalOutcome = message;
deps.clearTimeoutFn(timeout);
});
@ -204,7 +228,9 @@ export function createAutoSyncAnalysisRunner(
return;
}
if (terminalOutcome?.type === 'complete') {
settle(undefined, { stats: terminalOutcome.result.stats });
const { stats, graphWriteCollapsed } = terminalOutcome.result;
// Only present on a collapsed write, so a healthy result keeps its shape.
settle(undefined, graphWriteCollapsed ? { stats, graphWriteCollapsed } : { stats });
return;
}
if (terminalOutcome?.type === 'error') {

View file

@ -22,7 +22,7 @@ import type { ParsedImport, SymbolDefinition } from 'gitnexus-shared';
import { yieldToEventLoop } from './utils/event-loop.js';
import type { ExtractedRoute, ExtractedFetchCall } from './workers/parse-worker.js';
import type { ExtractedDecoratorRoute } from './workers/parse-worker.js';
import type { LanguageProvider } from './language-provider.js';
import type { LanguageProvider, RouteHookConfig } from './language-provider.js';
import { normalizeFetchURL, routeMatches } from './route-extractors/nextjs.js';
import {
isTestRouteFile,
@ -59,6 +59,8 @@ interface RouteHandlerResolutionContext {
) => readonly string[];
/** The route file's `LanguageProvider.resolveRouteHandler`, when it defines one. */
readonly providerRouteHandler?: (filePath: string) => LanguageProvider['resolveRouteHandler'];
/** User route configuration forwarded to provider handler hooks. */
readonly routeHookConfig?: RouteHookConfig;
readonly isExportedSymbol: (nodeId: string) => boolean;
/** 0-based graph-node startLine for same-name tRPC handler disambiguation. */
readonly nodeStartLine?: (nodeId: string) => number | undefined;
@ -559,7 +561,11 @@ export function resolveRouteHandlerSymbols(
return routeContext.resolveImportTargets(parsedImport, fromFile);
};
const providerContext = { model, importTargetsFor };
const providerContext = {
model,
importTargetsFor,
routeHookConfig: routeContext?.routeHookConfig,
};
const decoratorHandlerId = (dr: ExtractedDecoratorRoute): string | undefined => {
if (dr.source === DATA_ROUTE_TABLE_SOURCE) return dataHandlerByRoute.get(dr);
const providerHandler = routeContext?.providerRouteHandler?.(dr.filePath);

View file

@ -58,6 +58,7 @@ import type {
import type { KnowledgeGraph } from '../graph/types.js';
import type { ScopeResolutionIndexes } from './model/scope-resolution-indexes.js';
import { toZeroBasedLine } from './utils/line-base.js';
import { referenceEdgeKey } from './scope-resolution/graph-bridge/edges.js';
// ─── Public API ─────────────────────────────────────────────────────────────
@ -255,7 +256,7 @@ function buildRelationship(
// Other kinds omit `step`.
const step = ref.kind === 'read' ? 1 : ref.kind === 'write' ? 2 : undefined;
return {
id: `rel:${type}:${callerId}->${targetDef.nodeId}:${ref.atRange.startLine}:${ref.atRange.startCol}`,
id: `rel:${referenceEdgeKey(type, callerId, targetDef.nodeId, ref)}`,
sourceId: callerId,
targetId: targetDef.nodeId,
type,

View file

@ -53,12 +53,15 @@ export const javaExportChecker: ExportChecker = (node, _name) => {
let isPublic = javaPublicModifierCache.get(parent);
if (isPublic === undefined) {
isPublic = false;
for (let i = 0; i < parent.childCount; i++) {
const child = parent.child(i);
// Java modifiers precede the declaration's other named children.
// Skip comments, but never scan a class body full of sibling methods.
for (let i = 0; i < parent.namedChildCount; i++) {
const child = parent.namedChild(i);
if (child?.type === 'line_comment' || child?.type === 'block_comment') continue;
if (child?.type === 'modifiers' && child.text?.includes('public')) {
isPublic = true;
break;
}
break;
}
if (parent.type === 'method_declaration' || parent.type === 'constructor_declaration') {
isPublic ||= parent.text?.trimStart().startsWith('public') ?? false;

View file

@ -3,6 +3,8 @@
import { SupportedLanguages } from 'gitnexus-shared';
import type { FieldExtractionConfig } from '../generic.js';
import { extractSimpleTypeName } from '../../type-extractors/shared.js';
import { extractPythonInstanceFields } from '../../languages/python/instance-fields.js';
import { pythonFieldVisibility } from '../../languages/python/field-helpers.js';
/**
* Python field extraction config.
@ -20,6 +22,7 @@ export const pythonConfig: FieldExtractionConfig = {
fieldNodeTypes: ['expression_statement'],
bodyNodeTypes: ['block'],
defaultVisibility: 'public',
extractPrimaryFields: extractPythonInstanceFields,
extractName(node) {
// expression_statement wrapping an assignment or type
@ -78,15 +81,12 @@ export const pythonConfig: FieldExtractionConfig = {
const left = inner.childForFieldName('left');
name = left?.text;
}
if (!name) return 'public';
if (name.startsWith('__') && !name.endsWith('__')) return 'private';
if (name.startsWith('_')) return 'protected';
return 'public';
return pythonFieldVisibility(name);
},
isStatic(_node) {
// Reports syntactic static keyword — Python class variables don't use explicit static keyword.
// Instance variables (self.x) live in __init__ and are not extracted here.
// Constructor-assigned instance fields are extracted by extractPrimaryFields.
return false;
},

View file

@ -10,7 +10,7 @@ import {
import fs from 'fs/promises';
import { readdir } from 'node:fs';
import path from 'path';
import { glob } from 'glob';
import { glob, globIterate } from 'glob';
import { createIgnoreFilter } from '../../config/ignore-service.js';
import { mapConcurrent } from '../../lib/utils.js';
@ -254,6 +254,33 @@ export const walkRepositoryPaths = async (
return deduplicatedEntries;
};
/**
* Count the paths `walkRepositoryPaths` would scan, with the same patterns and
* ignore rules, and stop as soon as the count passes `limit`. Nothing is
* stat-ed, so large files, non-regular entries and declaration siblings are
* not subtracted: the result is an upper bound on what analyze indexes. Used
* to refuse an expensive rebuild before it starts (#3524).
*/
export const countScanCandidates = async (
repoPath: string,
limit: number,
options: Pick<WalkRepositoryOptions, 'includeDotDirs'> = {},
): Promise<number> => {
await assertWalkRootIsDirectory(repoPath);
const ignoreFilter = await createIgnoreFilter(repoPath);
const includeDotDirs = normalizeIncludeDotDirs(options.includeDotDirs ?? getIncludeDotDirs());
let count = 0;
for await (const _ of globIterate(indexableGlobPatterns(includeDotDirs), {
cwd: repoPath,
nodir: true,
dot: false,
ignore: includeDotDirs.length > 0 ? dotDirectoryIgnoreFilter(ignoreFilter) : ignoreFilter,
})) {
if (++count > limit) break;
}
return count;
};
/**
* Phase 2: Read file contents for a specific set of relative paths.
* Returns a Map for O(1) lookup. Silently skips files that fail to read.

View file

@ -56,8 +56,28 @@ import type {
import type { ExtractedDecoratorRoute, ExtractedToolDef } from './workers/parse-worker.js';
import type { SemanticModel } from './model/semantic-model.js';
/**
* A declared router-framework equivalence: values of type `type` reached
* through `importPath` are the `framework`'s router root. Framework names are
* opaque here; the provider that owns the framework interprets them.
*/
export interface RouteFrameworkAlias {
readonly importPath: string;
readonly type: string;
readonly framework: string;
}
/** User route configuration handed to provider route hooks on the main thread. */
export interface RouteHookConfig {
/** Declared router-framework aliases; empty when none are configured. */
readonly routeFrameworkAliases?: readonly RouteFrameworkAlias[];
/** Callee designators (`pkg.Wrap`) whose first argument is the real handler. */
readonly routeHandlerWrappers?: readonly string[];
}
/** What a provider's {@link LanguageProviderConfig.resolveRouteHandler} can see. */
export interface RouteHandlerResolutionHookContext {
readonly routeHookConfig?: RouteHookConfig;
/** The model after every file has been parsed; scope resolution has not run. */
readonly model: SemanticModel;
/**
@ -647,6 +667,7 @@ interface LanguageProviderConfig {
readonly resolveRouteFacts?: (
files: readonly { filePath: string; facts: unknown }[],
context: {
readonly routeHookConfig?: RouteHookConfig;
resolveImportTargets: (fromFile: string, moduleSpecifier: string) => readonly string[];
},
) => ExtractedDecoratorRoute[];

View file

@ -25,8 +25,8 @@ import { createMethodExtractor } from '../method-extractors/generic.js';
import { goMethodConfig } from '../method-extractors/configs/go.js';
import { createVariableExtractor } from '../variable-extractors/generic.js';
import { goVariableConfig } from '../variable-extractors/configs/go.js';
import { extractGoGinEchoRoutes } from '../route-extractors/go-gin-echo.js';
import { resolveGoRouteHandler } from './go/route-handler.js';
import { extractGoRouteFacts, resolveGoRouteFacts } from './go/route-facts.js';
import { goPackageDir } from './go/package-clause.js';
import { createCallExtractor } from '../call-extractors/generic.js';
import { goCallConfig } from '../call-extractors/configs/go.js';
@ -162,9 +162,10 @@ export const goProvider = defineLanguage({
wrapperNodeTypes: ['type_declaration'],
}),
builtInNames: GO_BUILT_INS,
// gin / echo verb registrations with in-function Group prefixes (#3402).
extractDecoratorRoutes: extractGoGinEchoRoutes,
resolveRouteHandler: resolveGoRouteHandler,
// One config-free collection for direct engines and wrapper-package roots.
extractRouteFacts: extractGoRouteFacts,
resolveRouteFacts: resolveGoRouteFacts,
// ── RFC #909 Ring 3: scope-based resolution hooks ──────────
emitScopeCaptures: emitGoScopeCaptures,

View file

@ -0,0 +1,92 @@
import type { RouteFrameworkAlias } from '../../language-provider.js';
import { goRouteFramework } from '../../route-extractors/go-gin-echo.js';
const FIELDS = new Set(['importPath', 'type', 'framework']);
// https://go.dev/ref/spec#Keywords — these cannot name a type declaration.
const GO_KEYWORDS = new Set([
'break',
'case',
'chan',
'const',
'continue',
'default',
'defer',
'else',
'fallthrough',
'for',
'func',
'go',
'goto',
'if',
'import',
'interface',
'map',
'package',
'range',
'return',
'select',
'struct',
'switch',
'type',
'var',
]);
/** Upper bound on declared aliases; a longer list is rejected, never truncated. */
export const MAX_ROUTER_ALIASES = 100;
/** Validate the Go-owned alias syntax before analysis starts. */
export function normalizeRouterAliases(value: unknown, source: string): RouteFrameworkAlias[] {
if (!Array.isArray(value) || value.length === 0) {
throw new Error(`${source} must be a non-empty array of alias objects.`);
}
if (value.length > MAX_ROUTER_ALIASES) {
throw new Error(
`${source} declares ${value.length} aliases; at most ${MAX_ROUTER_ALIASES} are allowed.`,
);
}
const out = new Map<string, RouteFrameworkAlias>();
for (const entry of value) {
if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) {
throw new Error(
`${source} entries must be objects like {"importPath": "...", "type": "Engine", "framework": "gin"}.`,
);
}
const record = entry as Record<string, unknown>;
for (const key of Object.keys(record)) {
if (!FIELDS.has(key)) throw new Error(`${source} entry has unknown field "${key}".`);
}
const field = (name: string, pattern: RegExp, hint: string): string => {
const raw = record[name];
if (typeof raw !== 'string' || !raw.trim()) {
throw new Error(`${source} entry field "${name}" must be a non-empty string.`);
}
const trimmed = raw.trim();
if (!pattern.test(trimmed)) throw new Error(`${source} entry field "${name}" ${hint}.`);
return trimmed;
};
const importPath = field(
'importPath',
/^[A-Za-z0-9._~-]+(\/[A-Za-z0-9._~-]+)*$/,
'must be a Go import path (e.g. "example.com/org/pkg")',
);
const type = field('type', /^[\p{L}_][\p{L}\p{Nd}_]*$/u, 'must be a Go type name');
if (type === '_' || GO_KEYWORDS.has(type)) {
throw new Error(
`${source} entry field "type" must be a Go type name, not a keyword or blank identifier.`,
);
}
const framework = field('framework', /^[a-z]+$/, 'must be "gin" or "echo"');
if (!goRouteFramework(framework)) {
throw new Error(`${source} entry field "framework" must be "gin" or "echo".`);
}
const key = `${importPath}\u0000${type}`;
const previous = out.get(key);
if (previous && previous.framework !== framework) {
throw new Error(
`${source} declares ${importPath}.${type} as both "${previous.framework}" and "${framework}".`,
);
}
out.set(key, { importPath, type, framework });
}
return [...out.values()];
}

View file

@ -0,0 +1,93 @@
import type Parser from 'tree-sitter';
import type { ParsedFile } from 'gitnexus-shared';
import { compareCodeUnits } from '../../../../lib/utils.js';
import { logger } from '../../../logger.js';
import type { LanguageProvider } from '../../language-provider.js';
import {
collectGoRouteFacts,
goRouteFramework,
materializeGoRoute,
type GoRouteCandidateFacts,
} from '../../route-extractors/go-gin-echo.js';
import type { ExtractedDecoratorRoute } from '../../workers/parse-worker.js';
/**
* Direct framework routes and registrations on a wrapper package's router
* (`func Init(s *core.Engine)`, where `core.Engine = gin.Engine` in a
* dependency the indexer cannot read).
*
* The worker records every such registration as a configuration-free
* candidate (`collectGoRouteFacts`), so the parse cache never depends on
* user config. Direct roots already identify their framework. Other roots
* become routes only when declared in `goRouterAliases` on the main thread.
*/
export const extractGoRouteFacts = (
tree: Parser.Tree,
_parsedFile: ParsedFile,
lineOffset: number,
): GoRouteCandidateFacts | undefined => collectGoRouteFacts(tree, lineOffset);
const isCandidateFacts = (facts: unknown): facts is GoRouteCandidateFacts =>
typeof facts === 'object' &&
facts !== null &&
Array.isArray((facts as GoRouteCandidateFacts).roots);
/** Root types logged in the "not configured" summary. */
const SUMMARY_LIMIT = 5;
export const resolveGoRouteFacts: NonNullable<LanguageProvider['resolveRouteFacts']> = (
files,
context,
) => {
const aliases = new Map<string, string>();
for (const alias of context.routeHookConfig?.routeFrameworkAliases ?? []) {
aliases.set(`${alias.importPath}\u0000${alias.type}`, alias.framework);
}
const out: ExtractedDecoratorRoute[] = [];
const emittedByRoot = new Map<string, number>();
const droppedByRoot = new Map<string, number>();
let candidates = 0;
for (const { filePath, facts } of files) {
if (!isCandidateFacts(facts)) continue;
for (const root of facts.roots) {
const rootKey = `${root.importPath}.${root.typeName}`;
const frameworkName =
root.framework ?? aliases.get(`${root.importPath}\u0000${root.typeName}`);
const framework = frameworkName && goRouteFramework(frameworkName);
if (!root.framework) candidates += root.routes.length;
if (!framework) {
droppedByRoot.set(rootKey, (droppedByRoot.get(rootKey) ?? 0) + root.routes.length);
continue;
}
if (!root.framework)
emittedByRoot.set(rootKey, (emittedByRoot.get(rootKey) ?? 0) + root.routes.length);
for (const candidate of root.routes) {
out.push(materializeGoRoute(filePath, candidate, framework));
}
}
}
for (const [rootKey, count] of emittedByRoot) {
logger.debug(
{ rootType: rootKey, routes: count },
'go routes: emitted routes through a configured router alias',
);
}
if (droppedByRoot.size > 0) {
const top = [...droppedByRoot.entries()]
.sort((a, b) => b[1] - a[1] || compareCodeUnits(a[0], b[0]))
.slice(0, SUMMARY_LIMIT)
.map(([rootType, dropped]) => ({ rootType, dropped }));
logger.info(
{
candidates,
emitted: [...emittedByRoot.values()].reduce((sum, count) => sum + count, 0),
unconfiguredRootTypes: droppedByRoot.size,
top,
},
'go routes: dropped verb registrations on unconfigured router types; if one is a ' +
'gin/echo engine alias, declare it in goRouterAliases',
);
}
return out;
};

View file

@ -2,6 +2,7 @@ import type { SymbolDefinition } from 'gitnexus-shared';
import type { RouteHandlerResolutionHookContext } from '../../language-provider.js';
import type { SemanticModel } from '../../model/semantic-model.js';
import type { ExtractedDecoratorRoute } from '../../workers/parse-worker.js';
import { logger } from '../../../logger.js';
import { goPackageDir } from './package-clause.js';
/**
@ -22,6 +23,8 @@ export function resolveGoRouteHandler(
route: ExtractedDecoratorRoute,
context: RouteHandlerResolutionHookContext,
): string | undefined {
const unwrapped = unwrapConfiguredHandler(route, context);
if (unwrapped !== route) return unwrapped && resolveGoRouteHandler(unwrapped, context);
const designator = route.handlerName;
if (!designator) return undefined;
const parts = designator.split('.');
@ -47,6 +50,43 @@ export function resolveGoRouteHandler(
: undefined;
}
/** Wrapper callees already reported as unconfigured, so the debug line fires once each. */
const reportedWrappers = new WeakMap<RouteHandlerResolutionHookContext, Set<string>>();
/**
* A route whose handler slot was `wrap(inner)`: the route re-addressed to
* `inner` when `wrap` is a configured handler wrapper, `undefined` when it is
* not, and the route itself when nothing was wrapped.
*/
function unwrapConfiguredHandler(
route: ExtractedDecoratorRoute,
context: RouteHandlerResolutionHookContext,
): ExtractedDecoratorRoute | undefined {
const wrapper = route.handlerWrapper;
if (route.handlerName || !wrapper) return route;
if (context.routeHookConfig?.routeHandlerWrappers?.includes(wrapper.callee)) {
return {
...route,
handlerName: wrapper.handlerName,
handlerReceiver: wrapper.handlerReceiver,
handlerWrapper: undefined,
};
}
let reported = reportedWrappers.get(context);
if (!reported) {
reported = new Set();
reportedWrappers.set(context, reported);
}
if (!reported.has(wrapper.callee)) {
reported.add(wrapper.callee);
logger.debug(
{ callee: wrapper.callee, file: route.filePath, line: route.lineNumber },
'go routes: handler is wrapped by an unconfigured function; add it to goHandlerWrappers to resolve it',
);
}
return undefined;
}
type DefinitionBuckets = Map<string, SymbolDefinition[]>;
type PackageIndex = Map<string, DefinitionBuckets>;
const bucketKey = (dir: string, name: string): string => JSON.stringify([dir, name]);

View file

@ -16,11 +16,19 @@ import type { JavaSpringDiClassFact } from './spring-di.js';
import type { SpringDynamicLookupFact } from '../../frameworks/spring/dynamic-lookups.js';
import type { SpringMessageProducerFact } from '../../frameworks/spring/message-producers.js';
import type { JavaSpringNonHttpHandlerFact } from './spring-non-http-handlers.js';
import {
clearJavaOverloadFacts,
getJavaOverloadFacts,
isJavaOverloadFileFacts,
setJavaOverloadFacts,
type JavaOverloadFileFacts,
} from './overload-argument-facts.js';
export type JavaClassAnnotationFact = ClassAnnotationFact;
export interface JavaCaptureSideChannel {
readonly kind: 'java';
readonly overloadFacts?: JavaOverloadFileFacts;
readonly packageFact: JvmPackageFact;
readonly classAnnotations: readonly JavaClassAnnotationFact[];
readonly springAopFacts?: readonly JavaSpringAopFact[];
@ -43,6 +51,7 @@ const springMessageProducerFacts = new Map<string, readonly SpringMessageProduce
/** Clear facts retained by a prior workspace pass in a long-lived process. */
export function clearJavaClassAnnotationFacts(): void {
clearJavaOverloadFacts();
classAnnotations.clear();
springAopFacts.clear();
springConfigConsumers.clear();
@ -157,6 +166,7 @@ export function collectJavaCaptureSideChannel(
filePath: string,
): JavaCaptureSideChannel | undefined {
const facts = classAnnotations.get(filePath);
const overloadFacts = getJavaOverloadFacts(filePath);
const aopFacts = springAopFacts.get(filePath) ?? [];
const configConsumers = springConfigConsumers.get(filePath) ?? [];
const conditionFacts = springConditionalFacts.get(filePath) ?? [];
@ -166,6 +176,7 @@ export function collectJavaCaptureSideChannel(
const messageProducerFacts = springMessageProducerFacts.get(filePath) ?? [];
const packageFact = getJavaPackageFact(filePath);
if (
overloadFacts === undefined &&
facts.length === 0 &&
aopFacts.length === 0 &&
configConsumers.length === 0 &&
@ -180,6 +191,7 @@ export function collectJavaCaptureSideChannel(
}
return {
kind: 'java',
...(overloadFacts === undefined ? {} : { overloadFacts }),
packageFact: packageFact ?? UNKNOWN_JVM_PACKAGE_FACT,
classAnnotations: facts,
...(aopFacts.length > 0 ? { springAopFacts: aopFacts } : {}),
@ -208,6 +220,7 @@ export function applyJavaCaptureSideChannel(parsed: ParsedFile): void {
data.kind !== 'java' ||
!Array.isArray(data.classAnnotations)
) {
setJavaOverloadFacts(parsed.filePath, undefined);
setJavaClassAnnotationFacts(parsed.filePath, []);
setJavaSpringAopFacts(parsed.filePath, []);
setJavaSpringConfigConsumerFacts(parsed.filePath, []);
@ -219,6 +232,10 @@ export function applyJavaCaptureSideChannel(parsed: ParsedFile): void {
setJavaPackageFact(parsed.filePath, UNKNOWN_JVM_PACKAGE_FACT);
return;
}
setJavaOverloadFacts(
parsed.filePath,
isJavaOverloadFileFacts(data.overloadFacts) ? data.overloadFacts : undefined,
);
setJavaClassAnnotationFacts(parsed.filePath, data.classAnnotations);
setJavaSpringAopFacts(
parsed.filePath,

View file

@ -63,6 +63,7 @@ import {
} from './spring-non-http-handlers.js';
import { synthesizeJavaRecordComponentAccessorCaptures } from './record-components.js';
import { synthesizeLombokAccessorCaptures } from './lombok-synthesizer.js';
import { captureJavaOverloadFacts, setJavaOverloadFacts } from './overload-argument-facts.js';
/** Declaration anchors that carry function-like arity metadata. */
const FUNCTION_DECL_TAGS = ['@declaration.method', '@declaration.constructor'] as const;
@ -143,6 +144,8 @@ export function emitJavaScopeCaptures(
recordCacheHit();
}
captureJavaPackageFact(filePath, tree.rootNode);
const accessorCaptures = synthesizeJavaRecordComponentAccessorCaptures(tree.rootNode);
setJavaOverloadFacts(filePath, captureJavaOverloadFacts(tree.rootNode, accessorCaptures));
const rawMatches = getJavaScopeQuery().matches(tree.rootNode);
const out: CaptureMatch[] = [];
@ -333,6 +336,14 @@ export function emitJavaScopeCaptures(
// directly on the method/constructor node.
const fnNode = findFunctionNode(nodeMap[declTag]);
if (fnNode !== null) {
const returnType = fnNode.childForFieldName('type');
if (returnType !== null && fnNode.type === 'method_declaration') {
grouped['@declaration.return-type'] = syntheticCapture(
'@declaration.return-type',
fnNode,
returnType.text + (fnNode.childForFieldName('dimensions')?.text ?? ''),
);
}
const arity = computeJavaArityMetadata(fnNode);
if (arity.parameterCount !== undefined) {
grouped['@declaration.parameter-count'] = syntheticCapture(
@ -392,6 +403,30 @@ export function emitJavaScopeCaptures(
'object_creation_expression',
);
if (callNode !== null) {
if (callNode.type === 'method_invocation') {
let initializer = callNode;
while (initializer.parent?.type === 'parenthesized_expression')
initializer = initializer.parent;
const declarator = initializer.parent;
const declaration = declarator?.parent;
const name = declarator?.childForFieldName('name');
if (
declarator?.type === 'variable_declarator' &&
declaration?.type === 'local_variable_declaration' &&
declaration.childForFieldName('type')?.text === 'var' &&
declarator.childForFieldName('value')?.id === initializer.id &&
name !== null &&
name !== undefined
) {
out.push({
'@call-result-assignment.call': nodeToCapture(
'@call-result-assignment.call',
callNode,
),
'@call-result-assignment.lhs': nodeToCapture('@call-result-assignment.lhs', name),
});
}
}
const argList = callNode.childForFieldName('arguments');
// Exclude interleaved comments — tree-sitter-java emits `block_comment` /
// `line_comment` as named children of argument_list, which would inflate
@ -452,7 +487,7 @@ export function emitJavaScopeCaptures(
...synthesizeJavaInheritanceReferences(tree.rootNode),
...synthesizeJavaExplicitConstructorReferences(tree.rootNode),
...synthesizeJavaAnonymousClassDeclarations(tree.rootNode),
...synthesizeJavaRecordComponentAccessorCaptures(tree.rootNode),
...accessorCaptures,
...synthesizeLombokAccessorCaptures(tree.rootNode),
...synthesizeCallableFlowCaptures(tree.rootNode, JAVA_CALLABLE_CAPTURE_OPTIONS),
];

View file

@ -0,0 +1,799 @@
import type { CaptureMatch, Range } from 'gitnexus-shared';
import { synthesizeJavaTypeIdentity, type SyntaxNode } from '../../utils/ast-helpers.js';
import { synthesizeJavaRecordComponentAccessorCaptures } from './record-components.js';
/** Written types retain their declaration context; a caller's imports cannot
* reinterpret a parameter declared in another file. These are capture facts,
* not the receiver-oriented (container-unwrapped) TypeRef hints. */
export type JavaArgumentFact =
| { readonly kind: 'typed'; readonly spelling: string; readonly at: Range }
| { readonly kind: 'null' }
| {
readonly kind: 'factory';
readonly owner: string;
readonly name: 'asList' | 'singletonList';
readonly arguments: readonly JavaArgumentFact[];
readonly typeArguments: readonly string[];
readonly at: Range;
readonly shadowed: boolean;
}
| { readonly kind: 'unknown' };
export interface JavaOverloadTypeFact {
readonly name: string;
readonly simpleName: string;
readonly kind: string;
readonly range: Range;
readonly visible: Range;
readonly bases: readonly string[];
readonly typeParameters: readonly string[];
readonly modifiers?: readonly string[];
}
export interface JavaOverloadMethodFact {
readonly name: string;
readonly owner: string;
readonly range: Range;
readonly parameterTypes: readonly string[];
readonly modifiers: readonly string[];
readonly typeParameters: readonly string[];
}
export interface JavaOverloadCallFact {
readonly name: string;
readonly range: Range;
readonly arguments: readonly JavaArgumentFact[];
readonly receiver?: JavaArgumentFact;
readonly receiverSpelling?: string;
readonly receiverCall?: { readonly name: string; readonly range: Range };
readonly receiverIsValue: boolean;
readonly staticContext: boolean;
}
export interface JavaOverloadFileFacts {
readonly version: 1;
readonly packageName: string;
readonly imports: readonly {
readonly name: string;
readonly wildcard: boolean;
readonly static: boolean;
}[];
readonly types: readonly JavaOverloadTypeFact[];
readonly methods: readonly JavaOverloadMethodFact[];
readonly calls: readonly JavaOverloadCallFact[];
}
const files = new Map<string, JavaOverloadFileFacts>();
const callIndexes = new Map<string, ReadonlyMap<string, JavaOverloadCallFact>>();
const UNKNOWN: JavaArgumentFact = { kind: 'unknown' };
const typeNodes = new Set([
'class_declaration',
'interface_declaration',
'enum_declaration',
'record_declaration',
'annotation_type_declaration',
]);
const callableNodes = new Set([
'method_declaration',
'constructor_declaration',
'compact_constructor_declaration',
'lambda_expression',
]);
const lexicalScopes = new Set([
'block',
'for_statement',
'enhanced_for_statement',
'catch_clause',
'lambda_expression',
'switch_block',
]);
export const javaOverloadRangeKey = (range: Range): string =>
`${range.startLine}:${range.startCol}:${range.endLine}:${range.endCol}`;
export function setJavaOverloadFacts(
filePath: string,
facts: JavaOverloadFileFacts | undefined,
): void {
if (facts === undefined) files.delete(filePath);
else files.set(filePath, facts);
callIndexes.delete(filePath);
}
export function clearJavaOverloadFacts(): void {
files.clear();
callIndexes.clear();
}
export function getJavaOverloadFacts(filePath: string): JavaOverloadFileFacts | undefined {
return files.get(filePath);
}
export function getJavaOverloadCall(
filePath: string,
range: Range,
name: string,
): JavaOverloadCallFact | undefined {
let index = callIndexes.get(filePath);
if (index === undefined) {
const facts = files.get(filePath);
if (facts === undefined) return undefined;
index = new Map(
facts.calls.map((call) => [`${javaOverloadRangeKey(call.range)}:${call.name}`, call]),
);
callIndexes.set(filePath, index);
}
return index.get(`${javaOverloadRangeKey(range)}:${name}`);
}
export function javaRangeContains(outer: Range, inner: Range): boolean {
return (
(outer.startLine < inner.startLine ||
(outer.startLine === inner.startLine && outer.startCol <= inner.startCol)) &&
(outer.endLine > inner.endLine ||
(outer.endLine === inner.endLine && outer.endCol >= inner.endCol))
);
}
function rangeOf(node: SyntaxNode): Range {
return {
startLine: node.startPosition.row + 1,
startCol: node.startPosition.column,
endLine: node.endPosition.row + 1,
endCol: node.endPosition.column,
};
}
function enclosing(
node: SyntaxNode,
predicate: (node: SyntaxNode) => boolean,
): SyntaxNode | undefined {
let parent = node.parent;
while (parent !== null) {
if (predicate(parent)) return parent;
parent = parent.parent;
}
return undefined;
}
function isType(node: SyntaxNode): boolean {
return (
typeNodes.has(node.type) ||
((node.type === 'object_creation_expression' || node.type === 'enum_constant') &&
node.namedChildren.some((child) => child.type === 'class_body'))
);
}
function typeIdentity(node: SyntaxNode): string | undefined {
return synthesizeJavaTypeIdentity(node)?.name ?? node.childForFieldName('name')?.text;
}
function parameterTypes(node: SyntaxNode): string[] {
const parameters = node.childForFieldName('parameters')?.namedChildren ?? [];
const result: string[] = [];
let variadic = false;
for (const parameter of parameters) {
if (!['formal_parameter', 'spread_parameter'].includes(parameter.type)) continue;
const declarator = parameter.namedChildren.find(
(child) => child.type === 'variable_declarator',
);
const name = parameter.childForFieldName('name') ?? declarator?.childForFieldName('name');
const type =
parameter.childForFieldName('type') ??
parameter.namedChildren.find(
(child) =>
child.type !== 'modifiers' &&
child.type !== 'variable_declarator' &&
child.id !== name?.id,
);
const dimensions =
parameter.childForFieldName('dimensions')?.text ??
declarator?.childForFieldName('dimensions')?.text ??
'';
result.push(type === undefined || type === null ? '?' : type.text + dimensions);
variadic ||= parameter.type === 'spread_parameter';
}
if (variadic) result.push('varargs');
return result;
}
function typeParameters(node: SyntaxNode): string[] {
const parameters = node.childForFieldName('type_parameters');
return (
parameters?.namedChildren.flatMap((parameter) => {
const name = parameter.namedChildren.find((child) => child.type === 'type_identifier');
return name === undefined ? [] : [name.text];
}) ?? []
);
}
function modifiers(node: SyntaxNode): string[] {
return (
node.namedChildren
.find((child) => child.type === 'modifiers')
?.children.filter((child) => !child.isNamed)
.map((child) => child.text) ?? []
);
}
interface Binding {
readonly type?: string;
readonly at: Range;
readonly scope: Range;
readonly depth: number;
readonly available: number;
readonly owner?: SyntaxNode;
readonly inheritable: boolean;
}
/** One AST walk records declarations and lexical bindings. Calls are evaluated
* against those bindings after the walk, so fields are visible before their
* declaration while local declarations retain their source-order boundary. */
export function captureJavaOverloadFacts(
root: SyntaxNode,
accessorCaptures: readonly CaptureMatch[] = synthesizeJavaRecordComponentAccessorCaptures(root),
): JavaOverloadFileFacts | undefined {
const types: JavaOverloadTypeFact[] = [];
const typeFactsByNode = new Map<number, JavaOverloadTypeFact>();
const typeNodesByFact = new Map<JavaOverloadTypeFact, SyntaxNode>();
const methods: JavaOverloadMethodFact[] = [];
// Reuse the scope synthesizer's validated component names and explicit
// accessor suppression so generated declarations have the same evidence as
// source-written methods without creating a second accessor authority.
const implicitAccessors = new Map(
accessorCaptures.flatMap((capture) => {
const declaration = capture['@declaration.method'];
const name = capture['@declaration.name'];
return declaration === undefined || name === undefined
? []
: [[javaOverloadRangeKey(declaration.range), name.text] as const];
}),
);
const callNodes: SyntaxNode[] = [];
const bindings = new Map<string, Binding[]>();
const localBindings = new Map<number, Map<string, Binding[]>>();
const imports: { name: string; wildcard: boolean; static: boolean }[] = [];
let packageName = '';
const stack = [{ node: root, depth: 0 }];
const addBinding = (
name: string | undefined,
type: string | undefined,
node: SyntaxNode,
scope: SyntaxNode,
depth: number,
available: number,
field = false,
) => {
if (name === undefined) return;
const binding: Binding = {
type,
at: rangeOf(node),
scope: rangeOf(scope),
depth,
available,
owner: enclosing(node, isType),
inheritable: !field || node.parent === null || !modifiers(node.parent).includes('private'),
};
if (field) {
const list = bindings.get(name) ?? [];
list.push(binding);
bindings.set(name, list);
} else {
const scoped = localBindings.get(scope.id) ?? new Map<string, Binding[]>();
const locals = scoped.get(name) ?? [];
locals.push(binding);
scoped.set(name, locals);
localBindings.set(scope.id, scoped);
}
};
while (stack.length > 0) {
const { node, depth } = stack.pop()!;
if (node.type === 'package_declaration') {
packageName =
node.namedChildren.find(
(child) => child.type === 'scoped_identifier' || child.type === 'identifier',
)?.text ?? '';
} else if (node.type === 'import_declaration') {
const name = node.namedChildren.find(
(child) => child.type === 'scoped_identifier' || child.type === 'identifier',
)?.text;
if (name !== undefined)
imports.push({
name,
wildcard: node.namedChildren.some((child) => child.type === 'asterisk'),
static: node.children.some((child) => child.type === 'static'),
});
}
if (isType(node)) {
const identity = typeIdentity(node);
const simpleName =
node.type === 'enum_constant'
? identity
: (node.childForFieldName('name')?.text ?? identity);
if (identity !== undefined && simpleName !== undefined) {
const visibilityScope =
enclosing(
node,
(parent) => lexicalScopes.has(parent.type) || parent.type === 'class_body',
) ?? root;
const visible = lexicalScopes.has(visibilityScope.type)
? {
...rangeOf(visibilityScope),
startLine: node.startPosition.row + 1,
startCol: node.startPosition.column,
}
: rangeOf(visibilityScope);
const bases: string[] = [];
for (const child of node.namedChildren) {
if (child.type === 'superclass')
bases.push(...child.namedChildren.map((part) => part.text));
if (child.type === 'super_interfaces' || child.type === 'extends_interfaces') {
const list = child.namedChildren.find((part) => part.type === 'type_list');
bases.push(...(list?.namedChildren ?? child.namedChildren).map((part) => part.text));
}
}
if (node.type === 'object_creation_expression') {
const base = node.childForFieldName('type');
if (base !== null) bases.push(base.text);
} else if (node.type === 'enum_constant') {
const host = enclosing(node, (parent) => parent.type === 'enum_declaration');
const base = host?.childForFieldName('name');
if (base !== undefined && base !== null) bases.push(base.text);
}
const flags = modifiers(node);
if (
(node.parent?.type === 'interface_body' ||
node.parent?.type === 'annotation_type_body') &&
!flags.some((flag) => ['public', 'protected', 'private'].includes(flag))
)
flags.push('public');
const fact: JavaOverloadTypeFact = {
name: identity,
simpleName,
kind: node.type,
range: rangeOf(node),
visible,
bases,
typeParameters: typeParameters(node),
modifiers: flags,
};
types.push(fact);
typeFactsByNode.set(node.id, fact);
typeNodesByFact.set(fact, node);
}
}
const implicitAccessor =
node.type === 'formal_parameter' || node.type === 'spread_parameter'
? implicitAccessors.get(javaOverloadRangeKey(rangeOf(node)))
: undefined;
if (
(callableNodes.has(node.type) && node.type !== 'lambda_expression') ||
implicitAccessor !== undefined
) {
const owner = enclosing(node, isType);
const ownerIdentity = owner === undefined ? undefined : typeIdentity(owner);
const name = implicitAccessor ?? node.childForFieldName('name')?.text;
if (ownerIdentity !== undefined && name !== undefined) {
const flags = implicitAccessor === undefined ? modifiers(node) : ['public'];
if (
owner?.type === 'interface_declaration' &&
!flags.includes('private') &&
!flags.includes('public')
)
flags.push('public');
methods.push({
name,
owner: ownerIdentity,
range: rangeOf(node),
parameterTypes: implicitAccessor === undefined ? parameterTypes(node) : [],
modifiers: flags,
typeParameters: typeParameters(node),
});
}
}
if (node.type === 'method_invocation') callNodes.push(node);
if (
node.type === 'formal_parameter' ||
node.type === 'spread_parameter' ||
node.type === 'catch_formal_parameter'
) {
const scope = enclosing(
node,
(parent) => callableNodes.has(parent.type) || parent.type === 'catch_clause',
);
const declarator = node.namedChildren.find((child) => child.type === 'variable_declarator');
const name = node.childForFieldName('name') ?? declarator?.childForFieldName('name');
const type =
node.childForFieldName('type') ??
node.namedChildren.find(
(child) =>
child.type !== 'modifiers' &&
child.type !== 'variable_declarator' &&
child.id !== name?.id,
);
const dimensions =
node.childForFieldName('dimensions')?.text ??
declarator?.childForFieldName('dimensions')?.text ??
'';
if (scope !== undefined)
addBinding(
name?.text,
type === undefined || type === null
? undefined
: type.text + dimensions + (node.type === 'spread_parameter' ? '[]' : ''),
node,
scope,
depth,
scope.startIndex,
);
} else if (
node.type === 'variable_declarator' &&
(node.parent?.type === 'field_declaration' ||
node.parent?.type === 'constant_declaration' ||
node.parent?.type === 'local_variable_declaration')
) {
const declaration = node.parent;
const field = declaration.type !== 'local_variable_declaration';
const scope = enclosing(node, (parent) =>
field
? ['class_body', 'interface_body', 'enum_body'].includes(parent.type)
: lexicalScopes.has(parent.type),
);
const type = declaration.childForFieldName('type');
const dimensions = node.childForFieldName('dimensions')?.text ?? '';
if (scope !== undefined)
addBinding(
node.childForFieldName('name')?.text,
type?.text === 'var'
? undefined
: type?.text === undefined
? undefined
: type.text + dimensions,
node,
scope,
depth,
field ? scope.startIndex : node.startIndex,
field,
);
} else if (node.type === 'enhanced_for_statement') {
const type = node.childForFieldName('type')?.text;
const body = node.childForFieldName('body');
if (body !== null)
addBinding(
node.childForFieldName('name')?.text,
type === undefined
? undefined
: type + (node.childForFieldName('dimensions')?.text ?? ''),
node,
body,
depth + 1,
body.startIndex,
);
} else if (node.type === 'lambda_expression') {
const parameters = node.childForFieldName('parameters');
if (parameters?.type === 'identifier')
addBinding(parameters.text, undefined, parameters, node, depth + 1, node.startIndex);
else if (parameters?.type === 'inferred_parameters') {
for (const parameter of parameters.namedChildren)
addBinding(parameter.text, undefined, parameter, node, depth + 1, node.startIndex);
}
}
const children = node.namedChildren;
for (let i = children.length - 1; i >= 0; i--)
stack.push({ node: children[i]!, depth: depth + 1 });
}
if (types.length === 0 && methods.length === 0 && callNodes.length === 0) return undefined;
// Resolve only inheritance visible in this AST. A missing superclass or a
// generic substitution is an uncertainty barrier, never permission to bind
// an enclosing instance's same-named field instead (JLS 6.5.6.1, 8.3).
// Include traversal depth so a cached shallow proof cannot bypass the bound
// on a longer path. Caching unknown results also bounds repeated failures.
const fieldLookupCache = new Map<string, readonly Binding[] | undefined>();
const fieldsFor = (
owner: SyntaxNode,
name: string,
inherited: boolean,
seen = new Set<number>(),
): readonly Binding[] | undefined => {
if (seen.has(owner.id) || seen.size >= 64) return undefined;
const cacheKey = `${owner.id}:${name}:${inherited}:${seen.size}`;
if (fieldLookupCache.has(cacheKey)) return fieldLookupCache.get(cacheKey);
fieldLookupCache.set(cacheKey, undefined);
seen.add(owner.id);
const fact = typeFactsByNode.get(owner.id);
if (fact === undefined) return undefined;
const own = (bindings.get(name) ?? []).filter((binding) => binding.owner?.id === owner.id);
if (own.length > 0) {
if (inherited && fact.typeParameters.length > 0) return undefined;
const result = inherited ? own.filter((binding) => binding.inheritable) : own;
fieldLookupCache.set(cacheKey, result);
return result;
}
if (fact.kind !== 'class_declaration' && fact.kind !== 'interface_declaration')
return undefined;
const found = new Map<string, Binding>();
for (const spelling of fact.bases) {
let nameOfBase = spelling.split('<')[0]!.trim();
if (packageName !== '' && nameOfBase.startsWith(`${packageName}.`))
nameOfBase = nameOfBase.slice(packageName.length + 1);
const visible = types.filter(
(candidate) =>
candidate.simpleName === nameOfBase && javaRangeContains(candidate.visible, fact.range),
);
let nearest: JavaOverloadTypeFact | undefined;
for (const candidate of visible) {
if (nearest === undefined || javaRangeContains(nearest.visible, candidate.visible))
nearest = candidate;
}
if (nearest === undefined) {
if (nameOfBase === 'java.lang.Object') continue;
return undefined;
}
if (
visible.some(
(candidate) =>
candidate !== nearest &&
javaOverloadRangeKey(candidate.visible) === javaOverloadRangeKey(nearest!.visible),
)
)
return undefined;
const base = typeNodesByFact.get(nearest);
const inheritedFields =
base === undefined ? undefined : fieldsFor(base, name, true, new Set(seen));
if (inheritedFields === undefined) return undefined;
for (const binding of inheritedFields) found.set(javaOverloadRangeKey(binding.at), binding);
}
const result = [...found.values()];
fieldLookupCache.set(cacheKey, result);
return result;
};
const bindingAt = (name: string, node: SyntaxNode, fieldOnly = false): Binding | undefined => {
const at = rangeOf(node);
const locals: Binding[] = [];
if (!fieldOnly) {
// Every recorded local scope is an AST ancestor of its uses. Restrict
// candidates before applying the existing shadowing and owner checks.
for (let scope: SyntaxNode | null = node; scope !== null; scope = scope.parent)
locals.push(...(localBindings.get(scope.id)?.get(name) ?? []));
}
let owner = enclosing(node, isType);
while (owner !== undefined) {
if (!fieldOnly) {
let local: Binding | undefined;
for (const binding of locals) {
if (
binding.owner?.id !== owner.id ||
binding.available > node.startIndex ||
!javaRangeContains(binding.scope, at)
)
continue;
if (
local === undefined ||
binding.depth > local.depth ||
(binding.depth === local.depth && binding.available > local.available)
)
local = binding;
}
if (local !== undefined) return local;
}
const fields = fieldsFor(owner, name, false);
if (fields === undefined || fields.length > 1) return undefined;
if (fields.length === 1) return fields[0];
if (fieldOnly || modifiers(owner).includes('static')) return undefined;
owner = enclosing(owner, isType);
}
return undefined;
};
const expression = (node: SyntaxNode, depth = 0): JavaArgumentFact => {
if (depth > 12) return UNKNOWN;
const typed = (spelling: string, at = rangeOf(node)): JavaArgumentFact => ({
kind: 'typed',
spelling,
at,
});
switch (node.type) {
case 'decimal_integer_literal':
case 'hex_integer_literal':
case 'octal_integer_literal':
case 'binary_integer_literal':
return typed(/[lL]$/.test(node.text) ? 'long' : 'int');
case 'decimal_floating_point_literal':
case 'hex_floating_point_literal':
return typed(/[fF]$/.test(node.text) ? 'float' : 'double');
case 'string_literal':
return typed('java.lang.String');
case 'character_literal':
return typed('char');
case 'true':
case 'false':
return typed('boolean');
case 'null_literal':
return { kind: 'null' };
case 'identifier': {
const binding = bindingAt(node.text, node);
return binding?.type === undefined ? UNKNOWN : typed(binding.type, binding.at);
}
case 'field_access': {
if (node.childForFieldName('object')?.type !== 'this') return UNKNOWN;
const name = node.childForFieldName('field')?.text;
const binding = name === undefined ? undefined : bindingAt(name, node, true);
return binding?.type === undefined ? UNKNOWN : typed(binding.type, binding.at);
}
case 'parenthesized_expression':
return node.namedChildren[0] === undefined
? UNKNOWN
: expression(node.namedChildren[0], depth + 1);
case 'cast_expression':
case 'object_creation_expression': {
const type = node.childForFieldName('type');
return type === null ? UNKNOWN : typed(type.text);
}
case 'array_creation_expression': {
const type = node.childForFieldName('type');
const dimensions =
node.namedChildren.filter((child) => child.type === 'dimensions_expr').length +
(node.namedChildren.find((child) => child.type === 'dimensions')?.text.match(/\[/g)
?.length ?? 0);
return type === null || dimensions === 0
? UNKNOWN
: typed(type.text + '[]'.repeat(dimensions));
}
case 'unary_expression': {
const operand = node.childForFieldName('operand') ?? node.namedChildren[0];
if (operand === undefined || operand === null) return UNKNOWN;
const value = expression(operand, depth + 1);
if (
value.kind !== 'typed' ||
!['byte', 'short', 'char', 'int', 'long', 'float', 'double', 'boolean'].includes(
value.spelling,
)
)
return UNKNOWN;
return typed(['byte', 'short', 'char'].includes(value.spelling) ? 'int' : value.spelling);
}
case 'method_invocation': {
const name = node.childForFieldName('name')?.text;
const owner = node.childForFieldName('object');
if ((name !== 'asList' && name !== 'singletonList') || owner === null) return UNKNOWN;
const args =
node
.childForFieldName('arguments')
?.namedChildren.filter(
(child) => child.type !== 'line_comment' && child.type !== 'block_comment',
) ?? [];
const firstName = owner.text.split('.')[0]!;
return {
kind: 'factory',
owner: owner.text,
name,
arguments: args.map((arg) => expression(arg, depth + 1)),
typeArguments:
node.childForFieldName('type_arguments')?.namedChildren.map((type) => type.text) ?? [],
at: rangeOf(node),
shadowed: bindingAt(firstName, owner) !== undefined,
};
}
default:
return UNKNOWN;
}
};
const calls = callNodes.map((node): JavaOverloadCallFact => {
const receiver = node.childForFieldName('object');
let directReceiver = receiver;
while (directReceiver?.type === 'parenthesized_expression')
directReceiver = directReceiver.namedChildren[0] ?? null;
const receiverName =
directReceiver?.type === 'method_invocation'
? directReceiver.childForFieldName('name')?.text
: undefined;
const callable = enclosing(
node,
(parent) => callableNodes.has(parent.type) && parent.type !== 'lambda_expression',
);
return {
name: node.childForFieldName('name')?.text ?? '',
range: rangeOf(node),
arguments: (node.childForFieldName('arguments')?.namedChildren ?? [])
.filter((arg) => arg.type !== 'line_comment' && arg.type !== 'block_comment')
.map((arg) => expression(arg)),
...(receiver === null
? {}
: { receiver: expression(receiver), receiverSpelling: receiver.text }),
...(receiverName === undefined || directReceiver === null
? {}
: { receiverCall: { name: receiverName, range: rangeOf(directReceiver) } }),
receiverIsValue:
receiver !== null &&
(receiver.type !== 'identifier' || bindingAt(receiver.text, receiver) !== undefined),
staticContext: callable !== undefined && modifiers(callable).includes('static'),
};
});
return { version: 1, packageName, imports, types, methods, calls };
}
/** Side-channel revival is fail-closed: malformed data never revives stale
* facts from a preceding file generation in a long-lived process. */
export function isJavaOverloadFileFacts(value: unknown): value is JavaOverloadFileFacts {
if (value === null || typeof value !== 'object') return false;
const facts = value as JavaOverloadFileFacts;
const range = (r: Range): boolean =>
r !== null &&
typeof r === 'object' &&
[r.startLine, r.startCol, r.endLine, r.endCol].every(Number.isSafeInteger);
const strings = (items: readonly string[]): boolean =>
Array.isArray(items) && items.every((item) => typeof item === 'string');
const argument = (arg: JavaArgumentFact, depth = 0): boolean => {
if (depth > 12 || arg === null || typeof arg !== 'object') return false;
if (arg.kind === 'null' || arg.kind === 'unknown') return true;
if (arg.kind === 'typed') return typeof arg.spelling === 'string' && range(arg.at);
return (
arg.kind === 'factory' &&
typeof arg.owner === 'string' &&
['asList', 'singletonList'].includes(arg.name) &&
typeof arg.shadowed === 'boolean' &&
range(arg.at) &&
strings(arg.typeArguments) &&
Array.isArray(arg.arguments) &&
arg.arguments.every((item) => argument(item, depth + 1))
);
};
return (
facts.version === 1 &&
typeof facts.packageName === 'string' &&
Array.isArray(facts.imports) &&
facts.imports.every(
(item) =>
item !== null &&
typeof item === 'object' &&
typeof item.name === 'string' &&
typeof item.wildcard === 'boolean' &&
typeof item.static === 'boolean',
) &&
Array.isArray(facts.types) &&
facts.types.every(
(type) =>
type !== null &&
typeof type === 'object' &&
typeof type.name === 'string' &&
typeof type.simpleName === 'string' &&
typeof type.kind === 'string' &&
range(type.range) &&
range(type.visible) &&
strings(type.bases) &&
strings(type.typeParameters) &&
(type.modifiers === undefined || strings(type.modifiers)),
) &&
Array.isArray(facts.methods) &&
facts.methods.every(
(method) =>
method !== null &&
typeof method === 'object' &&
typeof method.name === 'string' &&
typeof method.owner === 'string' &&
range(method.range) &&
strings(method.parameterTypes) &&
strings(method.modifiers) &&
strings(method.typeParameters),
) &&
Array.isArray(facts.calls) &&
facts.calls.every(
(call) =>
call !== null &&
typeof call === 'object' &&
typeof call.name === 'string' &&
range(call.range) &&
typeof call.receiverIsValue === 'boolean' &&
typeof call.staticContext === 'boolean' &&
Array.isArray(call.arguments) &&
call.arguments.every((arg) => argument(arg)) &&
(call.receiver === undefined || argument(call.receiver)) &&
(call.receiverSpelling === undefined || typeof call.receiverSpelling === 'string') &&
(call.receiverCall === undefined ||
(call.receiverCall !== null &&
typeof call.receiverCall === 'object' &&
typeof call.receiverCall.name === 'string' &&
range(call.receiverCall.range))),
)
);
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,345 @@
/** Java method-invocation conversions (JLS 5.3) and the phase ordering and
* specificity rules of JLS 15.12.2. Unknown evidence is never a conversion. */
export type JavaVerdict = 'yes' | 'no' | 'unknown';
export type JavaPrimitive =
| 'boolean'
| 'byte'
| 'short'
| 'char'
| 'int'
| 'long'
| 'float'
| 'double';
export type JavaType =
| { readonly kind: 'primitive'; readonly name: JavaPrimitive }
| {
readonly kind: 'reference';
readonly name: string;
readonly arguments: readonly JavaType[];
readonly sourceId?: string;
}
| { readonly kind: 'array'; readonly component: JavaType }
| { readonly kind: 'null' }
| { readonly kind: 'unknown' };
export type JavaReferenceType = Extract<JavaType, { kind: 'reference' }>;
export type JavaInvocationPhase = 'strict' | 'loose' | 'variable';
export interface JavaTypeRelations {
readonly sourceSubtype: (from: JavaReferenceType, to: JavaReferenceType) => JavaVerdict;
}
export interface JavaOverloadCandidate<T> {
readonly definition: T;
/** A variadic declaration's final entry is its ARRAY type. */
readonly parameters: readonly JavaType[];
readonly variadic: boolean;
readonly unsupportedGeneric?: boolean;
}
export type JavaOverloadDecision<T> =
| { readonly kind: 'resolved'; readonly definition: T; readonly phase: JavaInvocationPhase }
| { readonly kind: 'ambiguous' | 'unknown' | 'inapplicable'; readonly candidates: readonly T[] };
export const UNKNOWN_JAVA_TYPE: JavaType = { kind: 'unknown' };
export const MAX_JAVA_TYPE_DEPTH = 32;
export const JAVA_PRIMITIVES = new Set<JavaPrimitive>([
'boolean',
'byte',
'short',
'char',
'int',
'long',
'float',
'double',
]);
const widening: Readonly<Record<JavaPrimitive, readonly JavaPrimitive[]>> = {
boolean: [],
byte: ['short', 'int', 'long', 'float', 'double'],
short: ['int', 'long', 'float', 'double'],
char: ['int', 'long', 'float', 'double'],
int: ['long', 'float', 'double'],
long: ['float', 'double'],
float: ['double'],
double: [],
};
const boxes: Readonly<Record<JavaPrimitive, string>> = {
boolean: 'java.lang.Boolean',
byte: 'java.lang.Byte',
short: 'java.lang.Short',
char: 'java.lang.Character',
int: 'java.lang.Integer',
long: 'java.lang.Long',
float: 'java.lang.Float',
double: 'java.lang.Double',
};
const unboxes = new Map(
Object.entries(boxes).map(([primitive, box]) => [box, primitive as JavaPrimitive]),
);
/** These summaries are only used after a provider proves qualified JDK type
* identity. Collection edges preserve their invariant element arguments. */
const jdkParents: Readonly<Record<string, readonly string[]>> = {
'java.lang.Object': [],
'java.lang.String': ['java.lang.Object', 'java.lang.CharSequence', 'java.io.Serializable'],
'java.lang.Number': ['java.lang.Object', 'java.io.Serializable'],
'java.lang.Byte': ['java.lang.Number'],
'java.lang.Short': ['java.lang.Number'],
'java.lang.Integer': ['java.lang.Number'],
'java.lang.Long': ['java.lang.Number'],
'java.lang.Float': ['java.lang.Number'],
'java.lang.Double': ['java.lang.Number'],
'java.lang.Boolean': ['java.lang.Object', 'java.io.Serializable'],
'java.lang.Character': ['java.lang.Object', 'java.io.Serializable'],
'java.lang.CharSequence': ['java.lang.Object'],
'java.lang.Cloneable': ['java.lang.Object'],
'java.io.Serializable': ['java.lang.Object'],
'java.lang.Iterable': ['java.lang.Object'],
'java.lang.Runnable': ['java.lang.Object'],
'java.util.Collection': ['java.lang.Iterable'],
'java.util.List': ['java.util.Collection'],
'java.util.Set': ['java.util.Collection'],
'java.util.ArrayList': ['java.util.List', 'java.lang.Cloneable', 'java.io.Serializable'],
'java.util.LinkedList': ['java.util.List', 'java.lang.Cloneable', 'java.io.Serializable'],
'java.util.HashSet': ['java.util.Set', 'java.lang.Cloneable', 'java.io.Serializable'],
'java.util.Map': ['java.lang.Object'],
'java.util.HashMap': ['java.util.Map', 'java.lang.Cloneable', 'java.io.Serializable'],
};
const parameterizedJdk = new Set([
'java.lang.Iterable',
'java.util.Collection',
'java.util.List',
'java.util.Set',
'java.util.ArrayList',
'java.util.LinkedList',
'java.util.HashSet',
'java.util.Map',
'java.util.HashMap',
]);
const finalJdk = new Set(['java.lang.String', ...Object.values(boxes)]);
export const KNOWN_JAVA_PLATFORM_TYPES = new Set([
...Object.keys(jdkParents),
'java.util.Arrays',
'java.util.Collections',
]);
function all(verdicts: readonly JavaVerdict[]): JavaVerdict {
return verdicts.includes('no') ? 'no' : verdicts.includes('unknown') ? 'unknown' : 'yes';
}
export function sameJavaType(left: JavaType, right: JavaType, depth = 0): JavaVerdict {
if (depth >= MAX_JAVA_TYPE_DEPTH) return 'unknown';
if (left.kind === 'unknown' || right.kind === 'unknown') return 'unknown';
if (left.kind !== right.kind) return 'no';
if (left.kind === 'null') return 'yes';
if (left.kind === 'primitive' && right.kind === 'primitive')
return left.name === right.name ? 'yes' : 'no';
if (left.kind === 'array' && right.kind === 'array')
return sameJavaType(left.component, right.component, depth + 1);
if (left.kind === 'reference' && right.kind === 'reference') {
if (left.name !== right.name || left.sourceId !== right.sourceId) return 'no';
if (left.arguments.length !== right.arguments.length) return 'unknown';
return all(
left.arguments.map((argument, i) => sameJavaType(argument, right.arguments[i]!, depth + 1)),
);
}
return 'no';
}
export function javaSubtype(
from: JavaType,
to: JavaType,
relations: JavaTypeRelations,
depth = 0,
): JavaVerdict {
if (depth >= MAX_JAVA_TYPE_DEPTH) return 'unknown';
if (from.kind === 'unknown' || to.kind === 'unknown') return 'unknown';
if (sameJavaType(from, to, depth) === 'yes') return 'yes';
if (from.kind === 'null') return to.kind === 'reference' || to.kind === 'array' ? 'yes' : 'no';
if (from.kind === 'primitive' || to.kind === 'primitive') {
return from.kind === 'primitive' &&
to.kind === 'primitive' &&
widening[from.name].includes(to.name)
? 'yes'
: 'no';
}
if (from.kind === 'array') {
if (to.kind === 'reference')
return ['java.lang.Object', 'java.lang.Cloneable', 'java.io.Serializable'].includes(
to.name,
) && to.sourceId === undefined
? 'yes'
: 'no';
if (to.kind !== 'array') return 'no';
if (from.component.kind === 'primitive' || to.component.kind === 'primitive')
return sameJavaType(from.component, to.component, depth + 1);
return javaSubtype(from.component, to.component, relations, depth + 1);
}
if (from.kind !== 'reference' || to.kind !== 'reference') return 'no';
if (to.name === 'java.lang.Object' && to.sourceId === undefined) return 'yes';
if (from.name === to.name && from.sourceId === to.sourceId) {
// Raw/unchecked conversions are deliberately not approximated by erasure.
if (from.arguments.length !== to.arguments.length) return 'unknown';
return all(
from.arguments.map((argument, i) => sameJavaType(argument, to.arguments[i]!, depth + 1)),
);
}
if (to.sourceId === undefined && finalJdk.has(to.name)) return 'no';
if (from.sourceId !== undefined || to.sourceId !== undefined)
return relations.sourceSubtype(from, to);
const parents = Object.hasOwn(jdkParents, from.name) ? jdkParents[from.name] : undefined;
if (parents === undefined) return 'unknown';
let uncertain = false;
for (const parent of parents) {
const result = javaSubtype(
{
kind: 'reference',
name: parent,
arguments: parameterizedJdk.has(parent) ? from.arguments : [],
},
to,
relations,
depth + 1,
);
if (result === 'yes') return 'yes';
if (result === 'unknown') uncertain = true;
}
return uncertain || !Object.hasOwn(jdkParents, to.name) ? 'unknown' : 'no';
}
export function javaInvocationConversion(
from: JavaType,
to: JavaType,
phase: JavaInvocationPhase,
relations: JavaTypeRelations,
): JavaVerdict {
const strict = javaSubtype(from, to, relations);
if (phase === 'strict' || strict !== 'no') return strict;
if (from.kind === 'primitive' && to.kind === 'reference') {
return javaSubtype({ kind: 'reference', name: boxes[from.name], arguments: [] }, to, relations);
}
if (from.kind === 'reference' && to.kind === 'primitive') {
const primitive = from.sourceId === undefined ? unboxes.get(from.name) : undefined;
return primitive === undefined
? 'no'
: javaSubtype({ kind: 'primitive', name: primitive }, to, relations);
}
return 'no';
}
export function boxedJavaType(type: JavaType): JavaType {
return type.kind === 'primitive'
? { kind: 'reference', name: boxes[type.name], arguments: [] }
: type;
}
function effectiveParameters<T>(
candidate: JavaOverloadCandidate<T>,
phase: JavaInvocationPhase,
arity: number,
): readonly JavaType[] | undefined {
if (phase !== 'variable')
return candidate.parameters.length === arity ? candidate.parameters : undefined;
if (!candidate.variadic || arity < candidate.parameters.length - 1) return undefined;
const last = candidate.parameters.at(-1);
if (last?.kind !== 'array') return undefined;
return [
...candidate.parameters.slice(0, -1),
...Array.from({ length: arity - candidate.parameters.length + 1 }, () => last.component),
];
}
function moreSpecific<T>(
left: JavaOverloadCandidate<T>,
right: JavaOverloadCandidate<T>,
phase: JavaInvocationPhase,
arity: number,
relations: JavaTypeRelations,
): JavaVerdict {
const compareArity =
phase === 'variable' ? Math.max(arity, left.parameters.length, right.parameters.length) : arity;
const a = effectiveParameters(left, phase, compareArity);
const b = effectiveParameters(right, phase, compareArity);
if (a === undefined || b === undefined) return 'unknown';
return all(a.map((type, i) => javaSubtype(type, b[i]!, relations)));
}
/** The provider evaluates poly expressions against the candidate's parameter;
* the selector must not replace that callback with one frozen argument type. */
export function selectJavaOverload<T>(
candidates: readonly JavaOverloadCandidate<T>[],
arity: number,
argumentConversion: (index: number, target: JavaType, phase: JavaInvocationPhase) => JavaVerdict,
relations: JavaTypeRelations,
): JavaOverloadDecision<T> {
for (const phase of ['strict', 'loose', 'variable'] as const) {
const applicable: JavaOverloadCandidate<T>[] = [];
const uncertain: JavaOverloadCandidate<T>[] = [];
for (const candidate of candidates) {
const parameters = effectiveParameters(candidate, phase, arity);
if (parameters === undefined) continue;
const verdict = all(
parameters.map((parameter, i) => argumentConversion(i, parameter, phase)),
);
if (verdict === 'no') continue;
if (verdict === 'unknown' || candidate.unsupportedGeneric === true) uncertain.push(candidate);
else applicable.push(candidate);
}
// An unknown candidate in an earlier phase can beat any later-phase
// declaration. Even a currently unique applicable method is not a proof.
if (uncertain.length > 0)
return {
kind: 'unknown',
candidates: [...applicable, ...uncertain].map((candidate) => candidate.definition),
};
if (applicable.length === 0) continue;
if (applicable.length === 1)
return { kind: 'resolved', definition: applicable[0]!.definition, phase };
const winners = applicable.filter((candidate) =>
applicable.every(
(other) =>
candidate === other || moreSpecific(candidate, other, phase, arity, relations) === 'yes',
),
);
if (winners.length === 1)
return { kind: 'resolved', definition: winners[0]!.definition, phase };
if (winners.length > 1) {
// JLS 15.12.2.5 can choose a preferred declaration among inherited
// override-equivalent methods. Abstract/default and return-type facts
// are outside this bounded selector; this is not proven ambiguity.
return { kind: 'unknown', candidates: applicable.map((candidate) => candidate.definition) };
}
const unknown = applicable.some((candidate) =>
applicable.some(
(other) =>
candidate !== other &&
moreSpecific(candidate, other, phase, arity, relations) === 'unknown',
),
);
return {
kind: unknown ? 'unknown' : 'ambiguous',
candidates: applicable.map((candidate) => candidate.definition),
};
}
return { kind: 'inapplicable', candidates: candidates.map((candidate) => candidate.definition) };
}
/** Split only balanced top-level arguments; unsupported wildcards/type uses
* remain unknown in the provider rather than losing generic information. */
export function splitJavaTypeArguments(text: string): string[] | undefined {
const parts: string[] = [];
let depth = 0;
let start = 0;
for (let i = 0; i < text.length; i++) {
if (text[i] === '<') depth++;
else if (text[i] === '>') {
if (--depth < 0) return undefined;
} else if (text[i] === ',' && depth === 0) {
parts.push(text.slice(start, i).trim());
start = i + 1;
}
}
if (depth !== 0) return undefined;
parts.push(text.slice(start).trim());
return parts.every(Boolean) ? parts : undefined;
}

View file

@ -42,6 +42,12 @@ import {
clearJavaClassAnnotationFacts,
} from './capture-side-channel.js';
import { clearJavaPackageFacts } from './package-facts.js';
import {
resolveJavaReceiverMember,
resolveJavaSelectedMethodImplementation,
resolveJavaUnqualifiedCall,
resolveJavaReceiverCallSite,
} from './overload-resolution.js';
const javaScopeResolver: ScopeResolver = {
language: SupportedLanguages.Java,
@ -75,6 +81,11 @@ const javaScopeResolver: ScopeResolver = {
populateOwners: (parsed: ParsedFile) => populateClassOwnedMembers(parsed),
applyCaptureSideChannel: applyJavaCaptureSideChannel,
resolveReceiverMember: resolveJavaReceiverMember,
resolveUnqualifiedCall: resolveJavaUnqualifiedCall,
resolveSelectedMethodImplementation: resolveJavaSelectedMethodImplementation,
receiverMemberLookupCoversAllCalls: true,
receiverCallSite: resolveJavaReceiverCallSite,
isSuperReceiver: (text) => text.trim() === 'super',

View file

@ -47,6 +47,7 @@ import { pythonDecoratorRouteHandlerName } from '../route-extractors/python-deco
import { assertCloneable } from '../workers/clone-safety.js';
import { collectPythonCaptureSideChannel } from './python/module-accessors.js';
import { pythonRenameProvider } from './python/rename.js';
import { synthesizePythonInstanceFields } from './python/instance-fields.js';
const BUILT_INS: ReadonlySet<string> = new Set([
'print',
@ -242,6 +243,7 @@ export const pythonProvider = defineLanguage({
mroStrategy: 'c3',
callExtractor: createCallExtractor(pythonCallConfig),
fieldExtractor: createFieldExtractor(pythonFieldConfig),
synthesizeStructureMembers: synthesizePythonInstanceFields,
methodExtractor: createMethodExtractor(pythonMethodConfig),
variableExtractor: createVariableExtractor(pythonVariableConfig),
classExtractor: createClassExtractor(pythonClassConfig),

View file

@ -0,0 +1,48 @@
import type { SyntaxNode } from '../../utils/ast-helpers.js';
const TARGET_WRAPPERS = new Set([
'parenthesized_expression',
'pattern_list',
'tuple_pattern',
'list_pattern',
'list_splat_pattern',
'as_pattern_target',
'expression_list',
'tuple',
'list',
]);
/** Python Language Reference §§6.3.1, 7.2 and 7.5: a target is not a read,
* except for augmented assignment. Reading the receiver of a target still is. */
export function pythonAttributeAccessKinds(node: SyntaxNode): readonly ('read' | 'write')[] {
// An annotation names a type; it does not read that instance's field.
for (let ancestor = node.parent; ancestor !== null; ancestor = ancestor.parent) {
if (ancestor.type === 'type') return [];
if (ancestor.type === 'block' || ancestor.type === 'module') break;
}
let target = node;
while (target.parent !== null && TARGET_WRAPPERS.has(target.parent.type)) {
target = target.parent;
}
const parent = target.parent;
if (parent === null) return ['read'];
if (parent.type === 'call' && parent.childForFieldName('function')?.id === target.id) {
// The member-call capture owns this site.
return [];
}
if (parent.type === 'decorator') return [];
if (parent.type === 'delete_statement') return ['write'];
if (parent.type === 'as_pattern' && parent.childForFieldName('alias')?.id === target.id) {
return ['write'];
}
if (parent.childForFieldName('left')?.id === target.id) {
if (parent.type === 'assignment') {
// `obj.x: T` evaluates obj but neither reads nor assigns x.
return parent.childForFieldName('right') === null ? [] : ['write'];
}
if (parent.type === 'augmented_assignment') return ['read', 'write'];
if (parent.type === 'for_statement' || parent.type === 'for_in_clause') return ['write'];
}
return ['read'];
}

View file

@ -156,6 +156,22 @@ function bindingOf(identifier: SyntaxNode): Omit<NameBinding, 'redirect'> | null
}
}
/** Value bindings (including deletion) owned by this scope, in source order.
* Reuse the descriptor classifier so constructor ownership observes the same
* imports, pattern captures, and nested-scope boundaries as decorator lookup. */
export function pythonValueBindingsInScope(scope: SyntaxNode, name: string): SyntaxNode[] {
return scope.descendantsOfType('identifier').filter((node) => {
if (node.text !== name || scopeOf(node)?.id !== scope.id || bindingOf(node) === null) {
return false;
}
// A bare annotation does not replace the current value.
if (node.parent?.type === 'assignment' && node.parent.childForFieldName('right') === null) {
return false;
}
return name !== '_' || !inCasePattern(node);
});
}
const bindingsByTree = new WeakMap<object, ReadonlyMap<string, readonly NameBinding[]>>();
/** Every binding of a builtin descriptor name in the file, in source order. */

View file

@ -43,6 +43,7 @@ import { isPythonGlobalDeclaration, pythonFunctionDefinitionLabel } from './simp
import { synthesizeCallableFlowCaptures } from '../../utils/callable-flow-captures.js';
import { synthesizeReceiverChainCapture } from '../../utils/receiver-chain-captures.js';
import { applyPythonLexicalBindings } from './lexical-bindings.js';
import { pythonAttributeAccessKinds } from './attribute-references.js';
import {
beginPythonSubtypeDispatchCapture,
recordPythonSimplePositionalCall,
@ -53,6 +54,7 @@ import {
recordPythonModuleAccessor,
synthesizePythonCallResultAssignment,
} from './module-accessors.js';
import { synthesizePythonInstanceFieldCaptures } from './instance-fields.js';
const PYTHON_CALLABLE_CAPTURE_OPTIONS = {
functionNodeTypes: new Set(['function_definition', 'lambda']),
@ -158,6 +160,20 @@ export function emitPythonScopeCaptures(
}
if (Object.keys(grouped).length === 0) continue;
const attribute = nodeMap['@reference.read.member'];
if (attribute !== undefined) {
const kinds = pythonAttributeAccessKinds(attribute);
for (const kind of kinds) {
const reference = { ...grouped };
delete reference['@reference.read.member'];
const tag = `@reference.${kind}.member`;
reference[tag] = nodeToCapture(tag, attribute);
synthesizeReceiverChainCapture(reference, nodeMap['@reference.receiver']);
out.push(reference);
}
continue;
}
recordPythonSubtypeCallShape(grouped, nodeMap, filePath, subtypeLineMapper);
const assignedNode = nodeMap['@declaration.variable'];
@ -208,6 +224,14 @@ export function emitPythonScopeCaptures(
continue;
}
if (grouped['@scope.class'] !== undefined) {
out.push(
grouped,
...synthesizePythonInstanceFieldCaptures(nodeMap['@scope.class']!, filePath),
);
continue;
}
if (grouped['@scope.function'] !== undefined) {
// Structural receiver chain for a call whose receiver is itself an
// expression, so resolution can type it by folding over structure

View file

@ -0,0 +1,18 @@
import type { FieldVisibility } from '../../field-types.js';
import type { SyntaxNode } from '../../utils/ast-helpers.js';
/** Keep class and constructor field metadata on the same naming convention. */
export function pythonFieldVisibility(name: string | undefined): FieldVisibility {
if (!name) return 'public';
if (name.startsWith('__') && !name.endsWith('__')) return 'private';
return name.startsWith('_') ? 'protected' : 'public';
}
/** Match a direct receiver member; chained and foreign objects are excluded. */
export function pythonReceiverField(target: SyntaxNode, receiverName: string): SyntaxNode | null {
if (target.type !== 'attribute') return null;
const object = target.childForFieldName('object');
return object?.type === 'identifier' && object.text === receiverName
? target.childForFieldName('attribute')
: null;
}

Some files were not shown because too many files have changed in this diff Show more