bench(receiver-resolution): gate the drop counts and shape states in CI

U13. Turns the U1 instrument into a regression gate: `--check` asserts the
measured projection against a committed baseline, `--update-baseline` rewrites
it, and the benchmarks job runs the check.

Gated exactly, not budgeted. Two consecutive runs are byte-identical, so a
tolerance band would only hide real movement. Adding fixtures moves these
numbers and forces a rebaseline; that treadmill is the accepted cost of the
guard and the same trade `bench/scope-capture` already makes.

Both arms are gated, because they fail differently:

  countArm  call-only drop totals, split by site kind and by extension. The
            per-extension split makes a single-language regression legible
            instead of arriving as a moved total.

  shapeArm  the state of each receiver spelling by edge presence. This is the
            only arm that can see `?.`, explicit type arguments and
            `repos[0].save()` — they emit no edge AND no drop, so a count-only
            gate is blind to them by construction.

Verified in both directions. At baseline, `--check` passes. With a real drop
injected into the corpus — `f!.make().persist()`, not a hand-edited baseline —
it fails and localizes:

    countArm.callDrops: 99 -> 100
    countArm.bySiteKind.call: 99 -> 100
    countArm.callDropsByExtension..ts: 7 -> 8

Unlike its neighbours in that job this bench is NOT build-free: it runs the real
pipeline, so it needs dist/, which the setup action already builds. One run is
~2m15s against a 25-minute job budget that already carries eight steps — worth
watching on the first CI run rather than pre-emptively raising the timeout on a
guess.
This commit is contained in:
Gergo Magyar 2026-07-30 07:35:17 +00:00
parent bc12ebe95d
commit caf6b6a321
3 changed files with 129 additions and 4 deletions

View file

@ -500,6 +500,21 @@ jobs:
run: node --import tsx bench/callable-value-flow/measure.mjs --check
working-directory: gitnexus
- name: Receiver-resolution drop guards
# NOT build-free: this one runs the real pipeline, so it needs dist/
# (the setup action above builds). ~2m15s.
#
# Two arms, because neither gates alone. The count arm asserts the
# call-only drop count per language — call-only because Case 0's
# recorder gates on the receiver's punctuation, not on what the
# reference is, so property reads would inflate it by ~20%. The shape
# arm asserts the state of each receiver spelling by EDGE PRESENCE,
# which is the only arm that can see the shapes the recorder is blind
# to (`?.`, explicit type args, `repos[0]`): those emit no edge AND no
# drop, so fixing them moves the count by zero.
run: node --import tsx bench/receiver-resolution/measure.mjs --check
working-directory: gitnexus
- name: Scope-emission guards (#2699)
# Build-free: asserts the JS/TS scope set is unchanged. Block scopes are
# what make `let`/`const` in sibling blocks distinct bindings, but a

View file

@ -0,0 +1,44 @@
{
"shapeArm": {
"typescript": {
"plainChain": "RESOLVES",
"plainDeepChain": "RESOLVES",
"optionalChain": "INVISIBLE-GAP",
"nonNullAssert": "VISIBLE-GAP",
"awaitParen": "VISIBLE-GAP",
"explicitTypeArgs": "INVISIBLE-GAP",
"indexElement": "INVISIBLE-GAP"
},
"php": {
"arrowCallChain": "VISIBLE-GAP",
"arrowPropertyPath": "RESOLVES"
},
"cpp": {
"pointerArrowChain": "INVISIBLE-GAP",
"valueDotChain": "RESOLVES"
}
},
"countArm": {
"callDrops": 99,
"totalDropsAllKinds": 124,
"bySiteKind": {
"call": 99,
"read": 25
},
"callDropsByExtension": {
".java": 49,
".cs": 8,
".ts": 7,
".tsx": 6,
".py": 5,
".go": 5,
".cpp": 5,
".php": 4,
".rs": 3,
".kt": 3,
".rb": 2,
".js": 1,
".swift": 1
}
}
}

View file

@ -59,9 +59,16 @@
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.ts';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const BASELINE_PATH = path.resolve(__dirname, 'baseline.json');
/** The `--check` corpus. Committed and multi-language, so the gate is
* deterministic and does not depend on anything outside the repo. */
const DEFAULT_CORPUS = path.resolve(__dirname, '..', '..', 'test', 'fixtures', 'lang-resolution');
// ---------------------------------------------------------------------------
// Shape corpus
// ---------------------------------------------------------------------------
@ -321,15 +328,74 @@ const KNOWN_BLIND = [
'Every count is therefore a LOWER BOUND on a known-biased population. A later delta must be read against the same bias.',
];
/**
* The gated projection: shape states per language, plus the call-drop counts.
* Deliberately EXACT rather than budgeted — two consecutive runs are
* byte-identical, so a range would only hide real movement. Adding fixtures
* moves these numbers and requires a rebaseline; that treadmill is the accepted
* cost of the guard, the same trade the scope-capture bench already makes.
*/
function projection(output) {
return {
shapeArm: Object.fromEntries(
output.shapeArm.map((corpus) => [
corpus.language,
Object.fromEntries(corpus.shapes.map((shape) => [shape.id, shape.state])),
]),
),
countArm: {
callDrops: output.countArm.callDrops,
totalDropsAllKinds: output.countArm.totalDropsAllKinds,
bySiteKind: output.countArm.bySiteKind,
callDropsByExtension: output.countArm.callDropsByExtension,
},
};
}
/** Every leaf whose value differs, as `dotted.path: expected -> actual`. */
function drift(expected, actual, prefix = '') {
const out = [];
const keys = new Set([...Object.keys(expected ?? {}), ...Object.keys(actual ?? {})]);
for (const key of keys) {
const want = expected?.[key];
const got = actual?.[key];
const at = prefix === '' ? key : `${prefix}.${key}`;
if (want !== null && typeof want === 'object') out.push(...drift(want, got ?? {}, at));
else if (want !== got) out.push(`${at}: ${JSON.stringify(want)} -> ${JSON.stringify(got)}`);
}
return out;
}
const args = process.argv.slice(2);
const corpusIndex = args.indexOf('--corpus');
const corpusPath = corpusIndex === -1 ? undefined : args[corpusIndex + 1];
const check = args.includes('--check');
const corpusPath =
corpusIndex === -1 ? (check ? DEFAULT_CORPUS : undefined) : path.resolve(args[corpusIndex + 1]);
const output = { knownBlind: KNOWN_BLIND };
if (corpusPath === undefined || args.includes('--shapes')) {
if (corpusPath === undefined || check || args.includes('--shapes')) {
output.shapeArm = await runShapeArm();
}
if (corpusPath !== undefined) {
output.countArm = await runCountArm(path.resolve(corpusPath));
output.countArm = await runCountArm(corpusPath);
}
if (args.includes('--update-baseline')) {
fs.writeFileSync(BASELINE_PATH, `${JSON.stringify(projection(output), null, 2)}\n`, 'utf8');
console.error(`[receiver-resolution] wrote ${BASELINE_PATH}`);
} else if (check) {
const expected = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8'));
const diffs = drift(expected, projection(output));
if (diffs.length > 0) {
console.error('[receiver-resolution] FAIL — drift against the committed baseline:');
for (const line of diffs) console.error(` ${line}`);
console.error(
'\nIf this is intended (a fixture was added, or a shape genuinely changed state),' +
'\nre-run with --update-baseline and explain the movement in the commit message.',
);
process.exit(1);
}
console.error('[receiver-resolution] OK — shape states and call-drop counts match baseline.');
} else {
console.log(JSON.stringify(output, null, 2));
}
console.log(JSON.stringify(output, null, 2));