From ea757f8b5cc0d2cc1c8fd9b42040bd1b04f7f738 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 18 Jun 2026 16:42:12 +0000 Subject: [PATCH] test(impact): add a source-only (non-reconciled) ground-truth fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The accuracy corpus documents an annotation-circularity threat (KTD9): most fixtures' intra_AIS was reconciled against the live traversal, so F1=1.0 proves self-consistency, not independent correctness. Add one fixture (intra-control-gate) whose intra_AIS is written PURELY from language semantics and is deliberately NOT reconciled — each guarded arm is a single-statement block so there is no coalescing to surprise the annotation. The traversal scores it F1=1.0 (lines {11,13}, FPIS=FNIS=0), so it is a genuine independent confirmation — the one corpus data point that breaks the circularity threat. Re-baseline the annotation fingerprint and bump the corpus counts (intra 6→7, 13 measurable). Addresses PR #2227 tri-review finding (testing). Co-Authored-By: Claude Opus 4.8 (1M context) --- gitnexus/bench/impact-pdg/README.md | 6 ++-- gitnexus/bench/impact-pdg/baselines.json | 2 +- .../intra-control-gate/ground-truth.json | 30 +++++++++++++++++++ .../fixtures/intra-control-gate/src/gate.ts | 14 +++++++++ 4 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 gitnexus/bench/impact-pdg/fixtures/intra-control-gate/ground-truth.json create mode 100644 gitnexus/bench/impact-pdg/fixtures/intra-control-gate/src/gate.ts diff --git a/gitnexus/bench/impact-pdg/README.md b/gitnexus/bench/impact-pdg/README.md index 535208908..5d957a85f 100644 --- a/gitnexus/bench/impact-pdg/README.md +++ b/gitnexus/bench/impact-pdg/README.md @@ -117,7 +117,7 @@ mature CFG/PDG support in this codebase. | `nobody-interface-excluded` | n/a | — | no-body symbols (KTD6); **excluded** from PDG scoring | **Minimum corpus floor (KTD9/F3):** ≥ 3 cases per locus stratum, ≥ 12 total -measurable cases. Current: intra = 6, inter = 3, mixed = 3 → 12 measurable +measurable cases. Current: intra = 7, inter = 3, mixed = 3 → 13 measurable (+1 excluded no-body case). Below this floor the U7 harness must print "underpowered — directional only" instead of a verdict. @@ -276,7 +276,7 @@ After reconciliation, the line-seeded slice reproduces each corrected `intra_AIS exactly (FPIS = FNIS = 0 on all 6 intra and all 3 mixed fixtures). Call-graph gets no such home-field annotation, so the comparison is not rigged toward PDG. -## Measured results (analyzer 1.6.7, 12 measurable + 1 excluded) +## Measured results (analyzer 1.6.7, 13 measurable + 1 excluded) Each engine scored at its **native granularity** against its **native ground truth** — PDG at line vs `intra_AIS`, call-graph at symbol vs `inter_AIS`: @@ -361,7 +361,7 @@ Read it honestly: ## Underpowered-corpus rule (F3) **Minimum corpus floor: ≥ 3 measurable cases per locus stratum, ≥ 12 total.** -Current corpus is exactly at the floor (intra 6, inter 3, mixed 3 = 12 +Current corpus is above the floor (intra 7, inter 3, mixed 3 = 13 measurable; +1 excluded no-body) — so the harness prints headline decimals. When the measurable count after exclusions drops below the floor, it instead prints **"underpowered — directional only"** and reports the DIRECTION ("PDG exact at diff --git a/gitnexus/bench/impact-pdg/baselines.json b/gitnexus/bench/impact-pdg/baselines.json index 80a3d32b0..5e6cc5e74 100644 --- a/gitnexus/bench/impact-pdg/baselines.json +++ b/gitnexus/bench/impact-pdg/baselines.json @@ -2,7 +2,7 @@ "_doc": "U7 impact-PDG accuracy baselines. Two NON-byte-identity gates (KTD10): (1) annotationFingerprint — an order-independent digest over the curated ground-truth set (now INCLUDING criterion.line, the PDG slice seed); any unreviewed edit to a ground-truth.json trips it (re-baseline deliberately after review). (2) f1Bands — a ONE-SIDED F1 regression band per mode per scope: a DROP below (band - epsilon) fails; improvements pass freely. A `null` band means F1 is genuinely undefined for that (scope,mode) on this corpus — the gate skips it (nothing to regress against). measure.mjs --check applies median-of-K across GN_IMPACT_PDG_K runs before comparing, so substrate flakiness (the real-analyze lane, F5) cannot trip the band. Re-baseline: `node --import tsx bench/impact-pdg/measure.mjs --json` → copy annotationFingerprint and strata[scope][mode].f1 here, bump analyzerVersion if the analyzer moved.", "analyzerVersion": "1.6.7", "epsilon": 0.05, - "annotationFingerprint": "3f4a66dc201e5ea7629c0f3ee0117a4a14a5c7e25c27372ee9618e7839e1c433", + "annotationFingerprint": "39ae25c74922932d309d02342224057c782d2cc75d1d5e7a05fb292227c8a88f", "f1Bands": { "intra": { "callgraph": null, diff --git a/gitnexus/bench/impact-pdg/fixtures/intra-control-gate/ground-truth.json b/gitnexus/bench/impact-pdg/fixtures/intra-control-gate/ground-truth.json new file mode 100644 index 000000000..3842fa4d8 --- /dev/null +++ b/gitnexus/bench/impact-pdg/fixtures/intra-control-gate/ground-truth.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "criterion": { + "name": "gate", + "filePath": "src/gate.ts", + "direction": "downstream", + "line": 10, + "marker": "!ok", + "pdgEdgeKinds": ["CDG"] + }, + "locus": "intra", + "provenance": "manual", + "analyzerVersion": "1.6.7", + "intra_AIS": [ + { + "symbol": "gate", + "filePath": "src/gate.ts", + "line": 11, + "note": "`return 0` is control-dependent on the guard (true arm)" + }, + { + "symbol": "gate", + "filePath": "src/gate.ts", + "line": 13, + "note": "`return 1` is control-dependent on the guard (executes only when the guard is false)" + } + ], + "inter_AIS": [], + "rationale": "SOURCE-ONLY fixture (KTD9 annotation-circularity anchor): the intra_AIS below was written PURELY from language semantics and is deliberately NOT reconciled against the live traversal — there is no `CORRECTION (Step-0 reconciliation)` block. criterion.line=10 is the guard predicate `if (!ok)`; downstream CDG controller->dependent edges reach the true-arm `return 0` (line 11) and the fall-through `return 1` (line 13). Each arm is a single statement on its own block, so unlike the coalescing-prone sibling control fixtures there is no consecutive-statement merge to surprise the annotation: the source-derived set {11, 13} is also the measurable set. If the traversal scores this F1=1.0 it is an INDEPENDENT confirmation (the analyzer matched an un-reconciled, semantics-only annotation), not self-consistency — the one corpus data point that breaks the circularity threat the other fixtures document. All dependents are WITHIN `gate`; inter_AIS empty." +} diff --git a/gitnexus/bench/impact-pdg/fixtures/intra-control-gate/src/gate.ts b/gitnexus/bench/impact-pdg/fixtures/intra-control-gate/src/gate.ts new file mode 100644 index 000000000..96da57e30 --- /dev/null +++ b/gitnexus/bench/impact-pdg/fixtures/intra-control-gate/src/gate.ts @@ -0,0 +1,14 @@ +// Pure intra-procedural CONTROL-dependence fixture — SOURCE-ONLY (KTD9 anchor). +// Unlike the sibling control fixtures, this fixture's intra_AIS is NOT reconciled +// against the live traversal: each guarded arm is a SINGLE statement on its own +// block, so there is no consecutive-statement coalescing to surprise the +// source-derived annotation. It exists to give the corpus one independent data +// point — if the traversal matches an annotation written purely from language +// semantics, F1=1.0 is a genuine confirmation, not self-consistency. + +export function gate(ok: boolean): number { + if (!ok) { + return 0; + } + return 1; +}