mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-06 02:49:56 +00:00
test(impact): add a source-only (non-reconciled) ground-truth fixture
The accuracy corpus documents an annotation-circularity threat (KTD9): most
fixtures' intra_AIS was reconciled against the live traversal, so F1=1.0 proves
self-consistency, not independent correctness. Add one fixture
(intra-control-gate) whose intra_AIS is written PURELY from language semantics
and is deliberately NOT reconciled — each guarded arm is a single-statement block
so there is no coalescing to surprise the annotation. The traversal scores it
F1=1.0 (lines {11,13}, FPIS=FNIS=0), so it is a genuine independent confirmation —
the one corpus data point that breaks the circularity threat. Re-baseline the
annotation fingerprint and bump the corpus counts (intra 6→7, 13 measurable).
Addresses PR #2227 tri-review finding (testing).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
b3ef08b149
commit
ea757f8b5c
4 changed files with 48 additions and 4 deletions
|
|
@ -117,7 +117,7 @@ mature CFG/PDG support in this codebase.
|
|||
| `nobody-interface-excluded` | n/a | — | no-body symbols (KTD6); **excluded** from PDG scoring |
|
||||
|
||||
**Minimum corpus floor (KTD9/F3):** ≥ 3 cases per locus stratum, ≥ 12 total
|
||||
measurable cases. Current: intra = 6, inter = 3, mixed = 3 → 12 measurable
|
||||
measurable cases. Current: intra = 7, inter = 3, mixed = 3 → 13 measurable
|
||||
(+1 excluded no-body case). Below this floor the U7 harness must print
|
||||
"underpowered — directional only" instead of a verdict.
|
||||
|
||||
|
|
@ -276,7 +276,7 @@ After reconciliation, the line-seeded slice reproduces each corrected `intra_AIS
|
|||
exactly (FPIS = FNIS = 0 on all 6 intra and all 3 mixed fixtures). Call-graph gets
|
||||
no such home-field annotation, so the comparison is not rigged toward PDG.
|
||||
|
||||
## Measured results (analyzer 1.6.7, 12 measurable + 1 excluded)
|
||||
## Measured results (analyzer 1.6.7, 13 measurable + 1 excluded)
|
||||
|
||||
Each engine scored at its **native granularity** against its **native ground
|
||||
truth** — PDG at line vs `intra_AIS`, call-graph at symbol vs `inter_AIS`:
|
||||
|
|
@ -361,7 +361,7 @@ Read it honestly:
|
|||
## Underpowered-corpus rule (F3)
|
||||
|
||||
**Minimum corpus floor: ≥ 3 measurable cases per locus stratum, ≥ 12 total.**
|
||||
Current corpus is exactly at the floor (intra 6, inter 3, mixed 3 = 12
|
||||
Current corpus is above the floor (intra 7, inter 3, mixed 3 = 13
|
||||
measurable; +1 excluded no-body) — so the harness prints headline decimals. When
|
||||
the measurable count after exclusions drops below the floor, it instead prints
|
||||
**"underpowered — directional only"** and reports the DIRECTION ("PDG exact at
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
"_doc": "U7 impact-PDG accuracy baselines. Two NON-byte-identity gates (KTD10): (1) annotationFingerprint — an order-independent digest over the curated ground-truth set (now INCLUDING criterion.line, the PDG slice seed); any unreviewed edit to a ground-truth.json trips it (re-baseline deliberately after review). (2) f1Bands — a ONE-SIDED F1 regression band per mode per scope: a DROP below (band - epsilon) fails; improvements pass freely. A `null` band means F1 is genuinely undefined for that (scope,mode) on this corpus — the gate skips it (nothing to regress against). measure.mjs --check applies median-of-K across GN_IMPACT_PDG_K runs before comparing, so substrate flakiness (the real-analyze lane, F5) cannot trip the band. Re-baseline: `node --import tsx bench/impact-pdg/measure.mjs --json` → copy annotationFingerprint and strata[scope][mode].f1 here, bump analyzerVersion if the analyzer moved.",
|
||||
"analyzerVersion": "1.6.7",
|
||||
"epsilon": 0.05,
|
||||
"annotationFingerprint": "3f4a66dc201e5ea7629c0f3ee0117a4a14a5c7e25c27372ee9618e7839e1c433",
|
||||
"annotationFingerprint": "39ae25c74922932d309d02342224057c782d2cc75d1d5e7a05fb292227c8a88f",
|
||||
"f1Bands": {
|
||||
"intra": {
|
||||
"callgraph": null,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
{
|
||||
"schemaVersion": 1,
|
||||
"criterion": {
|
||||
"name": "gate",
|
||||
"filePath": "src/gate.ts",
|
||||
"direction": "downstream",
|
||||
"line": 10,
|
||||
"marker": "!ok",
|
||||
"pdgEdgeKinds": ["CDG"]
|
||||
},
|
||||
"locus": "intra",
|
||||
"provenance": "manual",
|
||||
"analyzerVersion": "1.6.7",
|
||||
"intra_AIS": [
|
||||
{
|
||||
"symbol": "gate",
|
||||
"filePath": "src/gate.ts",
|
||||
"line": 11,
|
||||
"note": "`return 0` is control-dependent on the guard (true arm)"
|
||||
},
|
||||
{
|
||||
"symbol": "gate",
|
||||
"filePath": "src/gate.ts",
|
||||
"line": 13,
|
||||
"note": "`return 1` is control-dependent on the guard (executes only when the guard is false)"
|
||||
}
|
||||
],
|
||||
"inter_AIS": [],
|
||||
"rationale": "SOURCE-ONLY fixture (KTD9 annotation-circularity anchor): the intra_AIS below was written PURELY from language semantics and is deliberately NOT reconciled against the live traversal — there is no `CORRECTION (Step-0 reconciliation)` block. criterion.line=10 is the guard predicate `if (!ok)`; downstream CDG controller->dependent edges reach the true-arm `return 0` (line 11) and the fall-through `return 1` (line 13). Each arm is a single statement on its own block, so unlike the coalescing-prone sibling control fixtures there is no consecutive-statement merge to surprise the annotation: the source-derived set {11, 13} is also the measurable set. If the traversal scores this F1=1.0 it is an INDEPENDENT confirmation (the analyzer matched an un-reconciled, semantics-only annotation), not self-consistency — the one corpus data point that breaks the circularity threat the other fixtures document. All dependents are WITHIN `gate`; inter_AIS empty."
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
// Pure intra-procedural CONTROL-dependence fixture — SOURCE-ONLY (KTD9 anchor).
|
||||
// Unlike the sibling control fixtures, this fixture's intra_AIS is NOT reconciled
|
||||
// against the live traversal: each guarded arm is a SINGLE statement on its own
|
||||
// block, so there is no consecutive-statement coalescing to surprise the
|
||||
// source-derived annotation. It exists to give the corpus one independent data
|
||||
// point — if the traversal matches an annotation written purely from language
|
||||
// semantics, F1=1.0 is a genuine confirmation, not self-consistency.
|
||||
|
||||
export function gate(ok: boolean): number {
|
||||
if (!ok) {
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue