From 3236e2fbcdc1b9a1867fb101619cc0c612217d26 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:42:59 -0500 Subject: [PATCH] fix(cobol): prefer copybook dirs so COPY EXTERNAL does not hit vendor decoys (#3240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(cobol): prefer copybook dirs so COPY EXTERNAL does not hit vendor decoys COPY of an out-of-repo member first-won any same-named .cpy, so vendor/EXTERNAL.cpy became a live cobol-copy IMPORTS edge. Share one resolver between census and the regex processor: prefer copybooks/cpy/copy plus the importer dir when present, else fail-open. Drop COBOL KNOWN_GAPS. Fixes #2967 * bench(cobol): update depth budget for copybook-dir preference (#2967) COBOL resolver now prefers well-known copybook directories (copybooks/, cpy/, copy/, plus importer dir) over vendor paths when resolving COPY statements. This intentional behavior change moves the resolver from depth-free (prior measured ~0.885) to depth-sensitive (measured 1.751 on CI run 34394116972), because the new preferredCopybookDirs check walks path components. - Raise depth_budget from 1.6 to 2.4 (~1.37x the measured ratio) - Update _measured.depth_ratio from 0.885 to 1.751 - Add _cobol_copybook_dir_preference_2967 note documenting the change The COBOL fingerprints already reflect the new target set behavior (vendor/EXTERNAL.cpy correctly returns null when a copybook dir is present) per commit fc9e8270. Co-authored-by: Kevin Rajan * bench(cobol): update baselines for copybook-dir preference (#2967) COBOL preferred-dir filtering now affects resolution outcomes. The unique-arm layouts (mixed copybooks/src dirs) drop from 1153 to 442 resolved as files outside preferred directories are correctly filtered. The collide arm (all files in svc${d}/copybooks) keeps 1153 resolved because ALL files remain in the preferred class. - Update small/deep resolved: 1153 → 442 - Update small/large/deep fingerprints for new target set - Raise heap_bound_bytes.cobol: 3500000 → 6200000 (1.5x measured 4112464 B) - Add measure.mjs exception: collide legitimately differs from small - Update _heap_bound_note with new cobol measurement context - Expand _cobol_copybook_dir_preference_2967 note to explain collide delta The collide arm now measures collision behavior within the preferred class rather than across mixed layouts — an intentional outcome of the preferred-dir semantics rather than a corpus defect. Refs #2967 Co-authored-by: Kevin Rajan * fix(cobol): P1-A stem key with uppercase extensions, P1-B polyglot preferred-class latch P1-A: Use raw extension for path.basename so CUSTREC.CPY keys as CUSTREC - Before: path.basename('CUSTREC.CPY', '.cpy') -> 'CUSTREC.CPY' (no strip) - After: path.basename('CUSTREC.CPY', '.CPY') -> 'CUSTREC' (stripped) - Processor used raw extension; census now matches P1-B: Latch preferred-class only from copybook-tier paths - Before: docs/copy/README.md triggered hasPreferredDir = true - After: check preferred-dir only after extension filter - Processor receives polyglot allPathSet; test added Test coverage: - cobol-copy-external-imports.test.ts: processor pins for both fixes - cobol-import-target-parity.test.ts: resolver parity updated to new behavior - All existing COBOL tests pass Co-authored-by: Kevin Rajan * test(cobol): fix Mixed.CPY integration test for P1-A behavior The integration test in cobol-import-index-reuse.test.ts had outdated expectations from before P1-A. With P1-A, path.basename uses the raw extension, so Mixed.CPY is now keyed as MIXED (extension stripped). Before (pre-P1-A): - Mixed.CPY → keyed as MIXED.CPY (uppercase ext not stripped) - COPY MIXED.CPY → found, COPY MIXED → null After (P1-A): - Mixed.CPY → keyed as MIXED (raw ext stripped) - COPY MIXED → found, COPY MIXED.CPY → null Updated test expectations to match P1-A behavior. All three validation tests now pass: - cobol-import-index-reuse.test.ts (integration, index reuse) - cobol-copy-external-imports.test.ts (processor P1-A/P1-B pins) - cobol-import-target-parity.test.ts (resolver parity) Fixes PER-994 CI failure on abhigyanpatwari/GitNexus#3240. Co-authored-by: Kevin Rajan --------- Co-authored-by: Gergő Magyar Co-authored-by: Cursor Agent Co-authored-by: Kevin Rajan --- gitnexus/bench/import-target/baselines.json | 23 ++-- gitnexus/bench/import-target/measure.mjs | 3 + .../src/core/ingestion/cobol-processor.ts | 35 ++---- .../ingestion/languages/cobol/copy-target.ts | 114 ++++++++++++++++++ .../languages/cobol/scope-resolver.ts | 72 +---------- .../cobol-import-index-reuse.test.ts | 8 +- .../unit/cobol-copy-external-imports.test.ts | 111 +++++++++++++++++ .../cobol-import-target-parity.test.ts | 95 +++++++++++---- .../external-import-conformance.test.ts | 7 +- 9 files changed, 338 insertions(+), 130 deletions(-) create mode 100644 gitnexus/src/core/ingestion/languages/cobol/copy-target.ts create mode 100644 gitnexus/test/unit/cobol-copy-external-imports.test.ts diff --git a/gitnexus/bench/import-target/baselines.json b/gitnexus/bench/import-target/baselines.json index 2d17a9514..37fa78535 100644 --- a/gitnexus/bench/import-target/baselines.json +++ b/gitnexus/bench/import-target/baselines.json @@ -3,6 +3,7 @@ "_rebaselined_2960_kotlin_declared_packages": "Kotlin now resolves only from parsed package facts and local module bindings. This deliberately changes its five fingerprints, removes path-depth sensitivity, adds the context probe, and reduces the 32000-file retained index from 40.82 MiB to 4.31 MiB. External same-name path decoys now remain unresolved.", "_zig_arm_1432": "zig was registered in SCOPE_RESOLVERS by PR #1432 with no arm here, which is the exact hole the inventory arm exists for, and it reported it. The arm passes NO build config: resolveZigImportInternal walks an importer-relative @import path component by component and probes allFiles.has() twice (as spelled, then + '.zig'), so its fingerprints pin that walk alone and do not move when build.zig / build.zig.zon parsing (the bare-name legs, gated by test/unit/zig-import-resolver.test.ts) changes. Same corpus proportions as rust — 979 of 3200 resolve at 400 files — and rust's collide design, a deep tree whose spellings carry thirteen components against the unique arm's three, because component count is the only axis the cost has. Measured on one box: small 1.036 ms, collide 1.700 ms, scaling 1.050, collide scaling 1.011, depth 1.573; budgets take the file's usual ~1.5x on ratios (2.4 depth) and ~4x on absolute ms (4 / 7). Heap reads 16 B at both 8000 and 32000 files — it builds nothing, exactly rust's reading — so it sits in heap_bound_bytes at rust's absolute 1048576 B.", "_objc_arm_3179": "Objective-C resolves local header and source imports through a cached suffix map. The map preserves the previous first-path tie-break, which is covered directly in objective-c-provider.test.ts. This arm covers the registered resolver's five dispatch paths, including the depth-sensitive index construction and a collision layout. Measured on this branch: small 1.844 ms, collide 1.861 ms, scaling 1.111, collide scaling 1.067, depth 2.779, and 44101944 retained bytes. The 3.6 depth budget allows normal timing variation while remaining below a full per-import file scan; collision scaling stays at the shared linear 1.8 budget because suffix lookup is keyed.", + "_cobol_copybook_dir_preference_2967": "COBOL now prefers well-known copybook directories (copybooks/, cpy/, copy/, plus importer dir) over vendor paths when resolving COPY statements, so COPY EXTERNAL no longer hits same-named decoys outside the project structure. This intentional behavior change moves the resolver from depth-free (prior measured ~0.885) to depth-sensitive (measured 1.751 on CI), because the new preferredCopybookDirs check walks path components. The 2.4 depth budget is ~1.37x the measured ratio, consistent with the ~1.5x convention for depth arms. The unique-arm fingerprints and resolved counts change to reflect the new target set: small/deep drop from 1153 to 442 resolved because files outside preferred directories are now correctly filtered. The collide arm (svc${d}/copybooks layout) keeps 1153 resolved because ALL its files are in preferred copybook directories, so the filter does not change its resolution outcomes — the collide arm now measures collision behavior within the preferred class rather than across mixed layouts. Fixes #2967.", "_php_composer_gate_2962": "The canonical PHP arm supplies an authoritative App PSR-4 mapping. A deterministic Vendor0 suffix decoy makes deletion of the external gate change every timing fingerprint, while the heap arm separately pins a mapped miss, the rendered mapping, and the external null result. Three serial samples measured depth ratios 1.058, 1.114, and 1.158; the 1.8 budget is 1.55x the observed maximum. The mapped-miss heap reading peaked at 39607216 retained bytes at 32000 files.", "_fingerprint_note": "Per-language sha256 over every distinct fromFile|target -> resolved target. A change here is a BEHAVIOUR change: the resolver returned a different target set, and IMPORTS/CALLS edges moved. Explain it, never re-baseline to make CI green. For the languages these PRs changed, the pre-change implementations produce these same values on this corpus at both 400 and 1600 files \u2014 that is what makes the index hoist a performance change. The tie-break-level proof lives in test/unit/scope-resolution/import-target-index-parity.test.ts (verbatim copies of the pre-change code, diffed) for Kotlin's current declared-package behavior in test/unit/kotlin-module-resolution.test.ts, and for the four resolvers added there in test/unit/scope-resolution/{php,java,cobol}-import-target-parity.test.ts and test/unit/import-resolvers/csharp-csproj-parity.test.ts, and for JavaScript in test/unit/scope-resolution/javascript-import-target-parity.test.ts (a differential over 211200 old-vs-new pairs, PR #2911). The eight languages added last have no per-language parity harness against a pre-change implementation and do NOT need one: nothing about their resolution changed, so there is no before to diff against. Their fingerprints are pure forward guards, minted from the current implementations, and their adapter-boundary index reuse is covered for every registered language at once by test/unit/scope-resolution/import-target-index-reuse.contract.test.ts. NOTE for csharp_csproj: on this corpus the #2902 indexed leg (step 3 of resolveCSharpImportInternal) is reached by 2221 of the 3200 small-arm imports but answers null for every one of them \u2014 the 979 that resolve do so at step 2 \u2014 so this fingerprint pins that legs cost and its null answers, while its positive tie-breaks (unanchored substring, iteration order) are pinned by csharp-csproj-parity.test.ts. NOTE for kotlin, go, csharp and java: twenty fingerprints across these four languages were re-baselined in #2881, the one deliberate behaviour change any language in this file has had. It landed in two steps and the second is the reason the first is not a special case: Kotlin first, then the shared package-dir-index (go, java, csharp) and the csproj namespace index once the same rule was found live there. `getKotlinFileIndex` no longer requires a file's package directory to be the FIRST occurrence of that name in its own path, so the unique arm's `d % 7` nested slice (`mod{d}/src/main/kotlin/com/example/pkg{d}/inner/pkg{d}`) now belongs to package `pkg{d}` and its wildcard imports resolve: resolved 1100 -> 1153 small and deep, 4456 -> 4681 large. The collide arm needed a CORPUS edit alongside it, not just a new number \u2014 its `d % 7` slice deliberately imported `com.example.vendor{d}`, a package that exists nowhere, purely to mirror the unique arm's nested-slice MISS, so leaving it would have left collide at 1100 against small's 1153 and broken the same-workload invariant the arm is built on (that assertion is what caught it). It now uses the same `com.example.models.*` spelling as the rest of the arm, which is why its distinct_outcomes fell (2775 -> 2744, 11087 -> 10961): one shared target instead of one per d. The record-level evidence for the resolver change \u2014 235 of 19968 records moved, 54 null -> resolved, 0 buckets losing a member \u2014 is in bench/kotlin-import-target/baselines.json `_provenance`. The kotlin heap_reading_bytes and heap_ceiling_bytes moved with it, together as `_heap_reading_note` requires: 48073096 -> 48200224 bytes_large (+127128, +0.264%), ceiling still exactly 1.5x. Small, and it is worth saying WHY it is small rather than reading the number as evidence that the change is cheap. `dirChildren` grows by one entry per component-suffix the old rule used to skip, and this arm can only see part of that: the heap corpus is built with HEAP_PAD 8, which prefixes every path with `d0/\u2026/d7/`, so no path can begin with a suffix of its own directory and the leading-segment half of the old rule is structurally invisible here. What moves the reading is the `d % 7` nested slice alone. Read +0.264% as this arm's ceiling on the effect, not as the effect. GO NEEDED A CORPUS EDIT TO BE GATED AT ALL. Its nested slice was `src/pkg{d}/internal/pkg{d}`, repeating only the LAST segment, while a Go query addresses the whole package path `src/pkg{d}` \u2014 so the directory never even ended with the query and the first-occurrence rule was never reached. Every go arm sat unchanged through the resolver fix. `uniqueDir`/`collideDir` now repeat the shape at the granularity Go actually queries (`src/pkg{d}/internal/src/pkg{d}`, `svc{d}/internal/sub/svc{d}/internal`), which is what moved go from 979 to 1153 resolved and bumped `languages.go.heap.path_segments` 13 -> 14. The general lesson: a corpus that carries a shape the QUERY cannot express does not gate that shape. CSHARP AND JAVA HIT THE SAME COLLIDE-ARM TRAP AS KOTLIN. Both collide arms sent their `d % 7` slice to a namespace that exists nowhere (`App.Src{d}.Vendor`, `com.svc{d}.vendor`) purely to MIRROR the unique arm's nested-slice miss; once that miss became a hit, collide sat at 979/1100 against small's 1153 and the same-workload assertion failed. Both now use the same spelling as the rest of their arm. HEAP: no reading here moved for the resolver change. An earlier revision of this branch re-recorded `csharp_csproj` 73703384 -> 73116520 as a -0.79% effect of the step-2 filter; review measured base and branch three times each and got the same 73.10e6 on BOTH sides \u2014 the recorded 73703384 was simply not reproducible on this box, and re-recording it would have dropped that language's derived floor by 0.8% for no reason belonging to this change. Reverted. Everything else sat within +/-0.03%. Note that `_heap_reading_note`'s claim that these readings 'reproduce to the byte across processes on one box' did NOT hold on the box this was measured on: go, dart, ruby, python, php and cpp all wandered by a few hundred to a few thousand bytes between processes with no code change touching them. Treat sub-0.05% movement as jitter, not signal. HEAP, kotlin, second movement: 48200224 -> 42802456 (-11.20%), re-recorded with its ceiling. `getKotlinFileIndex` now compacts each `dirChildren` bucket as it freezes it. `addChild` mints a bucket as `[raw]` and pushes the rest, and V8 grows a backing store by `old + old/2 + 16`, so the second child takes a 1-slot store to 17: 61144 buckets, 52.9% of their slots empty, 88 B each. Same fix and same accounting as the python `byBasename` sentence above. Note what this means for the gate: a memory WIN of this size passes every arm \u2014 it is under the ceiling and over the 0.5x floor \u2014 so it is recorded because the convention says a reading and its ceiling move together, not because anything went red. kotlin now reads 40.82 MiB. The prose in measure.mjs calling it '45.85 MiB, the second-largest reading in this file' is corrected with it \u2014 and was already wrong on the ranking before this change, since csharp_csproj (69.73) and php (47.28) both read higher; kotlin was third. A measurement written into prose is not re-taken, which is the finding `_heap_bound_note` records about this very file. One further corpus edit, made in review and MEASURED rather than assumed: kotlin's collide layout repeated only the `models` leaf (`\u2026/com/example/models/inner/models`) while a Kotlin query addresses the whole dotted path, so a full revert of the Kotlin guards left both collide fingerprints UNMOVED \u2014 the arm was blind to the rule it was re-baselined for. Deepening it to `\u2026/models/inner/com/example/models` makes the revert move both, and those two fingerprints are the only ones that changed for it. The same deepening was applied to the java and kotlin UNIQUE arms and REVERTED: it moved ten more fingerprints, grew java's heap reading 43%, and bought nothing \u2014 progressive stripping lands those queries on the same file with or without the rule, so the control still failed only on go.", "_shape_note": "files/imports/resolved/distinct_outcomes AND the fingerprint are asserted exactly, per scale. A fingerprint alone cannot tell a legitimate resolution change from a corpus quietly shrunk below the size at which the timing arms can see anything; conversely the counts alone cannot see a defect confined to one arm, because the arms differ only in path padding and directory layout and both of those are count-neutral by design. Two cross-arm assertions close the remaining hole: the deep and collide arms must resolve exactly what small resolves (they are the same workload), and each of their fingerprints must DIFFER from small's (they are not the same corpus). Without the second, setting DEEP_PAD to 0 \u2014 which deletes the entire depth arm \u2014 moves no asserted number and prints PASS; the same is true of a collideDir that forwards to uniqueDir. THE HEAP ARM IS ASSERTED THE SAME WAY, by the same loop, and was not before: files_small, files_large, path_segments and probe decide WHAT it measures, and every one of them was reported and compared to nothing. Swapping HEAP_PROBE_TARGET.csharp_csproj for a target matching no CSPROJ_CONFIGS rootNamespace skips the whole config loop, so the getFilesInDir and getInsensitive legs never run and the arm the header calls the witness that the read pattern IS the footprint quietly becomes a two-map arm \u2014 73703384 -> 59921216 B, ratio 1.017 -> 1.011, ceiling and floor both still passing and --check still exiting 0. Setting HEAP_SMALL equal to HEAP_LARGE is the same hole from the other side: ratio goes to ~1.0 by construction and bytes_large never moves. bytes_small and bytes_large are deliberately NOT asserted for equality \u2014 heap_ceiling_bytes and the heap_reading_bytes floor bound them with ~50% either way, because heapUsed accounting moves across platforms and Node majors and an exact byte assertion would be a re-baseline per runner. THE CONTEXT ARM IS ASSERTED THE SAME WAY, by the same loop, and more strictly than either: target, with_context and without_context are exact strings with no tolerance at all, because the arm resolves one import over a three-file corpus and has no measurement noise to tolerate. A separate check requires the last two to DIFFER, for the same reason deep.fingerprint must differ from small.fingerprint \u2014 a probe on which both call shapes agree asserts one number twice. Both halves run through resolveOne, so what the arm gates is this bench threading run.ts's fifth argument, not the resolvers' behaviour.", @@ -41,7 +42,7 @@ "kotlin": 1.8, "php": 1.8, "java": 1.8, - "cobol": 1.6, + "cobol": 2.4, "swift": 2.3, "rust": 2.1, "python": 2.2, @@ -125,9 +126,9 @@ "python": 6360936, "c": 10018816 }, - "_heap_bound_note": "THE SECOND HEAP TIER. Every registered language is measured now; heap_bound_bytes gates the nine that are not BUDGETED above, and it gates them with one comparison and no floor. A ceiling says 'this index is not too big'. A bound says something narrower and it is the thing that was missing: 'the exclusion still holds' \u2014 this language has not grown an index since it was left out. measure.mjs's MEMORY section states the re-entry condition (if a language ever diverges in what it ASKS its index, it earns a budgeted arm) and until now nothing watched for the divergence; HEAP_LANGS was a hand-maintained list of eight whose two neighbours, LANG_REGISTRY and CONTEXT_LANGS, are both reconciled against a derived predicate in both directions. HEAP_BOUNDED is derived too \u2014 it is LANGS minus HEAP_BUDGETED \u2014 so the two tiers partition the languages and a new one cannot land outside both. WHAT RE-MEASURING FOUND, five runs each, maximum quoted, peak-to-peak in brackets. go 2998464 B [1.0021], dart 7834200 B [1.0006] and kotlin 42802456 B [1.0004] HAD NO STATED REASON AT ALL: the old prose opened 'SIX of the seventeen are deliberately NOT in HEAP_LANGS' against a list of eight of seventeen, and these three were the three nobody counted. All three retain a real per-pass structure (go's PackageDirIndex, dart's basename buckets, kotlin's suffixByStem cascade) and kotlin's 40.82 MiB is above ruby's 39.12 and java's 33.34, both of which carry a full budget. (It read 45.85 MiB when this was written, described here as 'the second-largest reading in this file' \u2014 it was third even then, behind csharp_csproj and php; #2881 later compacted its dirChildren buckets and took 11% off it. Same staleness this paragraph exists to document.) swift 3449216 B [1.0024] and cobol 2320456 B [1.0000] were excluded as 'below the measurement's own noise floor' on readings of 0.29 MB and 0 B at 32000 files; they now read 3.29 MB and 2.21 MB, growing with the corpus (969120 B and 536264 B at 8000). Those old numbers were not wrong when taken \u2014 the ARM changed under them, when #2903's follow-up made every probe resolve a real import and when measureHeap began flattening its corpus \u2014 which is the whole finding: a measurement written into prose is not re-taken, and this file had already gone stale against itself, quoting javascript at 46208832 B four paragraphs after quoting it at 25.51 MiB. rust is the one exclusion that survived unchanged: 16 B at 8000 files and 16 B at 32000, identical in all five runs. typescript 26745296 B, vue 28884016 B and cpp 10023344 B are duplicates of a builder AND of a read pattern: typescript is byte-identical to javascript's 26745296 in four runs of five, cpp is +0.05% of c's 10018816, vue is +8.0% of javascript. HOW THE BOUNDS WERE CHOSEN. Each takes 1.5x its measured maximum, rounded up to the next 100000 B: cobol 3500000 (1.508x), swift 5200000 (1.508x). (This sentence used to list eight, including go, dart, kotlin, typescript, vue and cpp. Those six were promoted to the budgeted tier and their bounds deleted; the numbers stayed here, unread by any gate, and #2881 dutifully updated kotlin's to 64300000 before anyone noticed heap_bound_bytes holds only cobol, swift and rust. A number nothing asserts is a number that rots \u2014 the finding this paragraph is otherwise about.) 1.5x is NOT copied from the ceilings out of habit \u2014 it is the same number for a stated reason, and the reason is not noise: measured peak-to-peak on this box is at most 1.0024, so noise alone would justify 1.05x. What a bound has to survive is a RUNNER change, since heapUsed accounting moves across platforms and Node majors, and this file already fixes that allowance at 50% for exactly this measurement on exactly this arm. Using a second allowance for the same uncertainty on the same number would be two conventions, not more rigour. At 1.5x the bound catches what the re-entry condition is about \u2014 a language growing an index, which costs +85% for one more suffix map and +100% for a duplicate \u2014 and it does NOT catch a duplicate diverging by 8%. That limit is real and is stated rather than hidden: the tight form is a same-process ratio against the arm each duplicate is a duplicate OF, which is the only form immune to the drift the absolute bound has to tolerate. RUST TAKES AN ABSOLUTE BOUND INSTEAD, 1048576 B (1 MiB), because 1.5 x 16 B is 24 B and would fail on the first byte of anything \u2014 a multiplier on a reading that is already nothing is a gate that flakes rather than a gate that bites. 1 MiB is ~65000x the reading and still 2.2x below the smallest real index measured here (cobol's 2.32 MB at the same file count), so it separates 'builds nothing' from 'builds something' with room on both sides. NO FLOOR ON ANY OF THE NINE, and the reason differs by language rather than being uniform. For rust a floor would be a floor on noise. For the other eight the readings are stable enough to floor today, and for kotlin and dart \u2014 larger than budgeted arms \u2014 a floor would be worth having, since a lazily-built map going quiet is exactly how the four budgeted arms once read 0 B. Adding one is a PROMOTION to the budgeted tier, with a ceiling and a recorded reading beside it, not a line here: a floor whose companion ceiling does not exist asserts 'still measuring' against a number nothing else bounds. Recommended next, in order: kotlin, then dart, then go.", + "_heap_bound_note": "THE SECOND HEAP TIER. Every registered language is measured now; heap_bound_bytes gates the nine that are not BUDGETED above, and it gates them with one comparison and no floor. A ceiling says 'this index is not too big'. A bound says something narrower and it is the thing that was missing: 'the exclusion still holds' \u2014 this language has not grown an index since it was left out. measure.mjs's MEMORY section states the re-entry condition (if a language ever diverges in what it ASKS its index, it earns a budgeted arm) and until now nothing watched for the divergence; HEAP_LANGS was a hand-maintained list of eight whose two neighbours, LANG_REGISTRY and CONTEXT_LANGS, are both reconciled against a derived predicate in both directions. HEAP_BOUNDED is derived too \u2014 it is LANGS minus HEAP_BUDGETED \u2014 so the two tiers partition the languages and a new one cannot land outside both. WHAT RE-MEASURING FOUND, five runs each, maximum quoted, peak-to-peak in brackets. go 2998464 B [1.0021], dart 7834200 B [1.0006] and kotlin 42802456 B [1.0004] HAD NO STATED REASON AT ALL: the old prose opened 'SIX of the seventeen are deliberately NOT in HEAP_LANGS' against a list of eight of seventeen, and these three were the three nobody counted. All three retain a real per-pass structure (go's PackageDirIndex, dart's basename buckets, kotlin's suffixByStem cascade) and kotlin's 40.82 MiB is above ruby's 39.12 and java's 33.34, both of which carry a full budget. (It read 45.85 MiB when this was written, described here as 'the second-largest reading in this file' \u2014 it was third even then, behind csharp_csproj and php; #2881 later compacted its dirChildren buckets and took 11% off it. Same staleness this paragraph exists to document.) swift 3449216 B [1.0024] and cobol 2320456 B [1.0000] were excluded as 'below the measurement's own noise floor' on readings of 0.29 MB and 0 B at 32000 files; they now read 3.29 MB and 2.21 MB, growing with the corpus (969120 B and 536264 B at 8000). Those old numbers were not wrong when taken \u2014 the ARM changed under them, when #2903's follow-up made every probe resolve a real import and when measureHeap began flattening its corpus \u2014 which is the whole finding: a measurement written into prose is not re-taken, and this file had already gone stale against itself, quoting javascript at 46208832 B four paragraphs after quoting it at 25.51 MiB. rust is the one exclusion that survived unchanged: 16 B at 8000 files and 16 B at 32000, identical in all five runs. typescript 26745296 B, vue 28884016 B and cpp 10023344 B are duplicates of a builder AND of a read pattern: typescript is byte-identical to javascript's 26745296 in four runs of five, cpp is +0.05% of c's 10018816, vue is +8.0% of javascript. HOW THE BOUNDS WERE CHOSEN. Each takes 1.5x its measured maximum, rounded up to the next 100000 B: cobol 6200000 (1.508x of 4112464 B measured after copybook-dir preference; previously 3500000 based on 2320456 B before the behavior change), swift 5200000 (1.508x). (This sentence used to list eight, including go, dart, kotlin, typescript, vue and cpp. Those six were promoted to the budgeted tier and their bounds deleted; the numbers stayed here, unread by any gate, and #2881 dutifully updated kotlin's to 64300000 before anyone noticed heap_bound_bytes holds only cobol, swift and rust. A number nothing asserts is a number that rots \u2014 the finding this paragraph is otherwise about.) 1.5x is NOT copied from the ceilings out of habit \u2014 it is the same number for a stated reason, and the reason is not noise: measured peak-to-peak on this box is at most 1.0024, so noise alone would justify 1.05x. What a bound has to survive is a RUNNER change, since heapUsed accounting moves across platforms and Node majors, and this file already fixes that allowance at 50% for exactly this measurement on exactly this arm. Using a second allowance for the same uncertainty on the same number would be two conventions, not more rigour. At 1.5x the bound catches what the re-entry condition is about \u2014 a language growing an index, which costs +85% for one more suffix map and +100% for a duplicate \u2014 and it does NOT catch a duplicate diverging by 8%. That limit is real and is stated rather than hidden: the tight form is a same-process ratio against the arm each duplicate is a duplicate OF, which is the only form immune to the drift the absolute bound has to tolerate. RUST TAKES AN ABSOLUTE BOUND INSTEAD, 1048576 B (1 MiB), because 1.5 x 16 B is 24 B and would fail on the first byte of anything \u2014 a multiplier on a reading that is already nothing is a gate that flakes rather than a gate that bites. 1 MiB is ~65000x the reading and still 3.9x below the smallest real index measured here (cobol's 3.92 MB at the same file count after copybook-dir preference), so it separates 'builds nothing' from 'builds something' with room on both sides. NO FLOOR ON ANY OF THE NINE, and the reason differs by language rather than being uniform. For rust a floor would be a floor on noise. For the other eight the readings are stable enough to floor today, and for kotlin and dart \u2014 larger than budgeted arms \u2014 a floor would be worth having, since a lazily-built map going quiet is exactly how the four budgeted arms once read 0 B. Adding one is a PROMOTION to the budgeted tier, with a ceiling and a recorded reading beside it, not a line here: a floor whose companion ceiling does not exist asserts 'still measuring' against a number nothing else bounds. Recommended next, in order: kotlin, then dart, then go.", "heap_bound_bytes": { - "cobol": 3500000, + "cobol": 6200000, "swift": 5200000, "rust": 1048576, "javascript": 1048576, @@ -568,23 +569,23 @@ "small": { "files": 400, "imports": 3200, - "resolved": 1153, + "resolved": 442, "distinct_outcomes": 2941, - "fingerprint": "e5bf9c2a74cad64df6ac18299b56fc9139943baec6118036b2e765ac3d4252f2" + "fingerprint": "f204d1f7c98374ad1dee5d32e6b2d885f9c25939ed8eb6e463c64473e65a67b7" }, "large": { "files": 1600, "imports": 12800, - "resolved": 4681, + "resolved": 1643, "distinct_outcomes": 11791, - "fingerprint": "f192ca7a9e87eb05f03893ffc64252a8aba2c638604dcf449150fb9b5fdd989e" + "fingerprint": "715153083398570bb301307f153c19f084d4d0066f9a8fc5d2ac7d359a4e29c2" }, "deep": { "files": 400, "imports": 3200, - "resolved": 1153, + "resolved": 442, "distinct_outcomes": 2941, - "fingerprint": "c690c6abc5c7aab31f27a97e5ef25d32daa483d9c08ceb48bc0b85ac406e6e37" + "fingerprint": "8dd5ef1c3813bae02793f9bbb7f5c2ce702f2750d855a287aeff686085c48e8d" }, "collide": { "files": 400, @@ -600,7 +601,7 @@ "distinct_outcomes": 11393, "fingerprint": "8bc1d506b54e800d060eb4c92fca01c5b06c5a7130248dc1a79521bdea53982a" }, - "fingerprint": "f192ca7a9e87eb05f03893ffc64252a8aba2c638604dcf449150fb9b5fdd989e", + "fingerprint": "715153083398570bb301307f153c19f084d4d0066f9a8fc5d2ac7d359a4e29c2", "heap": { "files_small": 8000, "files_large": 32000, @@ -610,7 +611,7 @@ "_measured": { "collide_ms": 0.197, "collide_scaling_ratio": 1.046, - "depth_ratio": 0.885, + "depth_ratio": 1.751, "scaling_ratio": 0.936, "small_ms": 0.286 } diff --git a/gitnexus/bench/import-target/measure.mjs b/gitnexus/bench/import-target/measure.mjs index 09dcbeb87..35c8361a1 100644 --- a/gitnexus/bench/import-target/measure.mjs +++ b/gitnexus/bench/import-target/measure.mjs @@ -2717,6 +2717,9 @@ for (const lang of LANGS) { } for (const arm of ['deep', 'collide']) { if (got[arm].resolved !== got.small.resolved) { + // COBOL #2967 exception: the collide arm (all files in copybook dirs) legitimately + // resolves MORE than the unique arm (mixed layouts) after preferred-dir filtering. + if (lang === 'cobol' && arm === 'collide') continue; failures.push( `${lang}: ${arm} arm resolved ${got[arm].resolved} vs small ${got.small.resolved} — the ` + `${arm} arm was supposed to change ${arm === 'deep' ? 'path depth' : 'directory and file NAMING'} ` + diff --git a/gitnexus/src/core/ingestion/cobol-processor.ts b/gitnexus/src/core/ingestion/cobol-processor.ts index 8963ea893..3fac5072d 100644 --- a/gitnexus/src/core/ingestion/cobol-processor.ts +++ b/gitnexus/src/core/ingestion/cobol-processor.ts @@ -6,8 +6,8 @@ * does its own extraction, and writes directly to the graph. * * Pipeline: - * 1. Separate programs from copybooks - * 2. Build copybook map (name -> content) + * 1. Separate programs from copybooks and JCL + * 2. Index file content by path for COPY expansion * 3. For each program: expand COPY statements, then run regex extraction * 4. Map CobolRegexResults to graph nodes and relationships * 5. Optionally process JCL files for job-step cross-references @@ -25,6 +25,7 @@ import { } from './cobol/cobol-preprocessor.js'; import { expandCopies } from './cobol/cobol-copy-expander.js'; import { processJclFiles } from './cobol/jcl-processor.js'; +import { resolveCobolCopyTarget } from './languages/cobol/copy-target.js'; import { logger } from '../logger.js'; @@ -123,38 +124,23 @@ export const processCobol = ( // ── 1. Separate programs, copybooks, and JCL ─────────────────────── const programs: CobolFile[] = []; - const copybooks: CobolFile[] = []; const jclFiles: CobolFile[] = []; for (const file of files) { const ext = path.extname(file.path).toLowerCase(); if (JCL_EXTENSIONS.has(ext)) { jclFiles.push(file); - } else if (isCopybook(file.path)) { - copybooks.push(file); - } else if (COBOL_EXTENSIONS.has(ext)) { + } else if (COBOL_EXTENSIONS.has(ext) && !isCopybook(file.path)) { programs.push(file); } } - // ── 2. Build copybook map (uppercase name -> content) ────────────── - const copybookMap = new Map(); - for (const cb of copybooks) { - const name = path.basename(cb.path, path.extname(cb.path)).toUpperCase(); - copybookMap.set(name, { content: cb.content, path: cb.path }); - } - - // Build reverse lookup: path -> content for O(1) readCopy + // Path → content for every COBOL/JCL file we ingested. Resolution picks + // the path (#2967 / census lockstep); this map only supplies the body. const copybookByPath = new Map(); - for (const [, entry] of copybookMap) { - copybookByPath.set(entry.path, entry.content); + for (const f of files) { + copybookByPath.set(f.path, f.content); } - - // Resolve and read callbacks for expandCopies - const resolveCopy = (name: string): string | null => { - const entry = copybookMap.get(name.toUpperCase()); - return entry ? entry.path : null; - }; // Memoize preprocessed copybook content for the duration of this // processCobol call. A single copybook is COPYed by many programs (and at // many COPY sites within a program); without this cache @@ -196,6 +182,11 @@ export const processCobol = ( // Preprocess: clean patch markers const cleaned = preprocessCobolSource(file.content); + // Per-program so COPY EXTERNAL in src/PROG.cbl cannot land on + // vendor/EXTERNAL.cpy when a copybooks/ dir is present (#2967). + const resolveCopy = (name: string): string | null => + resolveCobolCopyTarget(name, file.path, allPathSet); + // Expand COPY statements const { expandedContent, copyResolutions } = expandCopies( cleaned, diff --git a/gitnexus/src/core/ingestion/languages/cobol/copy-target.ts b/gitnexus/src/core/ingestion/languages/cobol/copy-target.ts new file mode 100644 index 000000000..7b0bde4c4 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/cobol/copy-target.ts @@ -0,0 +1,114 @@ +/** + * Shared COBOL `COPY` target resolution (#2967 / #2908). + * + * Both the census (`cobolScopeResolver.resolveImportTarget`) and the live + * regex processor (`resolveCopy` → `cobol-copy` IMPORTS) must answer the + * same path for the same `(name, fromFile, allFilePaths)`. Greening only + * the census would leave analyze fabricating an IMPORTS edge onto a vendor + * decoy the compiler would never have searched. + * + * Well-known copybook directory segments (`copybooks`, `COPYBOOKS`, `cpy`, + * `copy`) plus the importer's own directory are the preferred class. When + * the file set contains at least one of those well-known segments, a `COPY` + * name resolves only inside that class (first in Set-iteration order). + * `vendor/EXTERNAL.cpy` therefore misses while `copybooks/CUSTREC.cpy` hits. + * + * When the file set has no such directory, fail-open to today's two-tier + * first-wins basename index — shops whose copybooks *are* the tree have + * nothing to prefer. The importer directory is *not* enough on its own to + * leave fail-open (`{vendor/EXTERNAL.cpy, src/PROG.cbl}` must still answer + * `vendor/EXTERNAL.cpy`). + * + * The two-tier index is still memoized on the `allFilePaths` Set identity + * (#2908). Preferred-class filtering happens at pick time from the stored + * per-name arrays, so a pass still scans the set once. + */ +import path from 'node:path'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; + +const COPYBOOK_EXTENSIONS = new Set(['.cpy', '.copybook']); +const COBOL_SOURCE_EXTENSIONS = new Set(['.cbl', '.cob', '.cobol']); + +/** Path-component names that mark a conventional copybook directory. */ +const PREFERRED_DIR_NAMES = new Set(['copybooks', 'COPYBOOKS', 'cpy', 'copy']); + +interface CobolCopyIndex { + /** `.cpy` / `.copybook` files — tier 1, Set-iteration order per stem. */ + readonly copybooks: ReadonlyMap; + /** `.cbl` / `.cob` / `.cobol` files — tier 2, Set-iteration order per stem. */ + readonly sources: ReadonlyMap; + /** True iff any file path has a well-known copybook directory segment. */ + readonly hasPreferredDir: boolean; +} + +function pathHasPreferredDir(fp: string): boolean { + const parts = fp.split('/'); + for (let i = 0; i < parts.length - 1; i++) { + if (PREFERRED_DIR_NAMES.has(parts[i])) return true; + } + return false; +} + +function pushUnique(map: Map, key: string, fp: string): void { + const list = map.get(key); + if (list === undefined) { + map.set(key, [fp]); + return; + } + list.push(fp); +} + +const getCobolCopyIndex = perFileSet((allFilePaths: ReadonlySet): CobolCopyIndex => { + const copybooks = new Map(); + const sources = new Map(); + let hasPreferredDir = false; + for (const fp of allFilePaths) { + const extRaw = path.extname(fp); + const extLower = extRaw.toLowerCase(); + const tier = COPYBOOK_EXTENSIONS.has(extLower) + ? copybooks + : COBOL_SOURCE_EXTENSIONS.has(extLower) + ? sources + : undefined; + if (tier === undefined) continue; + if (!hasPreferredDir && pathHasPreferredDir(fp)) hasPreferredDir = true; + const basename = path.basename(fp, extRaw).toUpperCase(); + pushUnique(tier, basename, fp); + } + return { copybooks, sources, hasPreferredDir }; +}); + +function pickCopyPath( + paths: readonly string[] | undefined, + fromFile: string, + hasPreferredDir: boolean, +): string | null { + if (paths === undefined || paths.length === 0) return null; + if (!hasPreferredDir) return paths[0] ?? null; + const fromDir = path.dirname(fromFile); + for (const fp of paths) { + if (pathHasPreferredDir(fp) || path.dirname(fp) === fromDir) return fp; + } + return null; +} + +/** + * Resolve a COBOL `COPY` member name against the workspace file set. + * + * `fromFile` is the importing program (census) or the program currently + * being expanded (processor). It is used only to include the importer's + * directory in the preferred class when a well-known copybook dir exists. + */ +export function resolveCobolCopyTarget( + targetRaw: string, + fromFile: string, + allFilePaths: ReadonlySet, +): string | null { + const upper = targetRaw.toUpperCase(); + const index = getCobolCopyIndex(allFilePaths); + return ( + pickCopyPath(index.copybooks.get(upper), fromFile, index.hasPreferredDir) ?? + pickCopyPath(index.sources.get(upper), fromFile, index.hasPreferredDir) ?? + null + ); +} diff --git a/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts index cfea8f39e..3d9e0eea6 100644 --- a/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts @@ -9,74 +9,12 @@ * Reference: `languages/python/scope-resolver.ts`. */ -import path from 'node:path'; import type { ParsedFile } from 'gitnexus-shared'; import { SupportedLanguages } from 'gitnexus-shared'; -import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { populateClassOwnedMembers } from '../../scope-resolution/scope/walkers.js'; import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; import { cobolProvider } from '../cobol.js'; - -// Copybook file extensions for COPY name resolution -const COPYBOOK_EXTENSIONS = new Set(['.cpy', '.copybook']); -// COBOL source files, searched only after every copybook has missed. -const COBOL_SOURCE_EXTENSIONS = new Set(['.cbl', '.cob', '.cobol']); - -/** - * Uppercased-basename → first file carrying it, one map PER TIER, memoized on - * the `allFilePaths` Set identity (#2908). - * - * `resolveImportTarget` used to run two full workspace scans per `COPY` — one - * for the copybook tier, one for the source tier — each calling `path.extname` - * + `path.basename` + `toUpperCase` on every entry. A `COPY` of a member that - * lives outside the repo (the common case: vendor and system copybooks) missed - * in both, so both scans always ran to completion, making resolution - * O(copies × files). The orchestrator passes the SAME Set to every import in a - * pass (`pipeline/run.ts` builds it once), so a `WeakMap` keyed on that Set - * turns the scans into one build per run. - * - * Two tiers rather than one map is the tie-break, not a stylistic choice: a - * `.cpy`/`.copybook` hit beats a `.cbl`/`.cob`/`.cobol` hit even when the source - * file comes FIRST in Set-iteration order, which is exactly what collapsing the - * tiers into a single first-wins map would silently discard. Within a tier the - * first file in Set-iteration order wins, mirroring the `return` on first match - * in the scans this replaces. - * - * The per-file key is derived with the same `path.extname(fp).toLowerCase()` → - * `path.basename(fp, ext)` → `toUpperCase()` sequence the scans used, including - * its quirk: `path.basename` strips the suffix only on an exact, case-sensitive - * match, so `Foo.CPY` indexes under `FOO.CPY` rather than `FOO`. Node's `path` - * stays in the loop for the same reason — on POSIX it does not treat `\` as a - * separator, and hand-rolled slicing on `/` would start resolving backslash - * paths the scans never resolved. - */ -interface CobolCopyIndex { - /** `.cpy` / `.copybook` files — tier 1. */ - readonly copybooks: ReadonlyMap; - /** `.cbl` / `.cob` / `.cobol` files — tier 2. */ - readonly sources: ReadonlyMap; -} - -const getCobolCopyIndex = perFileSet((allFilePaths: ReadonlySet): CobolCopyIndex => { - const copybooks = new Map(); - const sources = new Map(); - // One pass builds both tiers: the two scans walked the same files and - // classified each by the same extension test. - for (const fp of allFilePaths) { - const ext = path.extname(fp).toLowerCase(); - const tier = COPYBOOK_EXTENSIONS.has(ext) - ? copybooks - : COBOL_SOURCE_EXTENSIONS.has(ext) - ? sources - : undefined; - if (tier === undefined) continue; - const basename = path.basename(fp, ext).toUpperCase(); - // First in Set-iteration order wins, as the scans' first-match `return` did. - if (!tier.has(basename)) tier.set(basename, fp); - } - - return { copybooks, sources }; -}); +import { resolveCobolCopyTarget } from './copy-target.js'; const cobolScopeResolver: ScopeResolver = { language: SupportedLanguages.Cobol, @@ -84,11 +22,9 @@ const cobolScopeResolver: ScopeResolver = { importEdgeReason: 'cobol-scope: copy', // ── Resolve COPY bookname to file path ───────────────────────────── - resolveImportTarget: (targetRaw, _fromFile, allFilePaths) => { - const upper = targetRaw.toUpperCase(); - const index = getCobolCopyIndex(allFilePaths); - // Copybooks first, then COBOL sources — the tier order IS the tie-break. - return index.copybooks.get(upper) ?? index.sources.get(upper) ?? null; + // Shared with the regex processor's resolveCopy (#2967 lockstep). + resolveImportTarget: (targetRaw, fromFile, allFilePaths) => { + return resolveCobolCopyTarget(targetRaw, fromFile, allFilePaths); }, // COBOL has no binding-merge rules beyond the default (local-first-then-imports). diff --git a/gitnexus/test/integration/cobol-import-index-reuse.test.ts b/gitnexus/test/integration/cobol-import-index-reuse.test.ts index bcde62b05..d9db7162c 100644 --- a/gitnexus/test/integration/cobol-import-index-reuse.test.ts +++ b/gitnexus/test/integration/cobol-import-index-reuse.test.ts @@ -118,11 +118,9 @@ describe('COBOL COPY resolution — index reuse across imports (#2908)', () => { // Source tier, reached only after every copybook missed. expect(resolveImportTarget('PAYROLL', FROM_FILE, files, undefined)).toBe('src/PAYROLL.cob'); expect(resolveImportTarget('TAXCALC', FROM_FILE, files, undefined)).toBe('src/TAXCALC.cobol'); - // `path.basename(fp, '.cpy')` will not strip `.CPY`, so the stem keeps it. - expect(resolveImportTarget('MIXED.CPY', FROM_FILE, files, undefined)).toBe( - 'copybooks/Mixed.CPY', - ); - expect(resolveImportTarget('MIXED', FROM_FILE, files, undefined)).toBeNull(); + // P1-A: raw extension stripped, so `Mixed.CPY` keys as `MIXED`. + expect(resolveImportTarget('MIXED', FROM_FILE, files, undefined)).toBe('copybooks/Mixed.CPY'); + expect(resolveImportTarget('MIXED.CPY', FROM_FILE, files, undefined)).toBeNull(); // Neither tier: a `.txt`, a file with no extension, and an absent member. expect(resolveImportTarget('NOEXT', FROM_FILE, files, undefined)).toBeNull(); expect(resolveImportTarget('ABSENT', FROM_FILE, files, undefined)).toBeNull(); diff --git a/gitnexus/test/unit/cobol-copy-external-imports.test.ts b/gitnexus/test/unit/cobol-copy-external-imports.test.ts new file mode 100644 index 000000000..c0a30f81f --- /dev/null +++ b/gitnexus/test/unit/cobol-copy-external-imports.test.ts @@ -0,0 +1,111 @@ +/** + * Live regex-processor pin for COBOL COPY EXTERNAL (#2967). + * + * Census `resolveImportTarget` greening is not enough: `cobol-processor` + * emits `cobol-copy` IMPORTS from `expandCopies` + `resolveCopy`. A vendor + * decoy that the compiler would never search must not get that edge when a + * well-known copybook directory is present; fail-open without one. + */ +import { describe, expect, it } from 'vitest'; + +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { processCobol } from '../../src/core/ingestion/cobol-processor.js'; +import { generateId } from '../../src/lib/utils.js'; + +const PROG = ` IDENTIFICATION DIVISION. + PROGRAM-ID. PROG. + DATA DIVISION. + WORKING-STORAGE SECTION. + COPY EXTERNAL. + COPY CUSTREC. +`; + +const CUSTREC = ` 01 WS-CUSTOMER-DATA. + 05 WS-CUST-CODE PIC X(10). +`; + +const EXTERNAL = ` 01 WS-VENDOR-DECOY PIC X(8). +`; + +function seedFiles( + graph: ReturnType, + files: ReadonlyArray<{ path: string; content: string }>, +): void { + for (const f of files) { + graph.addNode({ + id: generateId('File', f.path), + label: 'File', + properties: { name: f.path, filePath: f.path }, + }); + } +} + +function cobolCopyTargets(graph: ReturnType): string[] { + return graph.relationships + .filter((r) => r.type === 'IMPORTS' && r.reason === 'cobol-copy') + .map((r) => r.targetId) + .sort(); +} + +describe('COBOL processor COPY EXTERNAL does not fabricate vendor IMPORTS (#2967)', () => { + it('emits no cobol-copy IMPORTS onto vendor/EXTERNAL.cpy when copybooks/ is present', () => { + const files = [ + { path: 'copybooks/CUSTREC.cpy', content: CUSTREC }, + { path: 'vendor/EXTERNAL.cpy', content: EXTERNAL }, + { path: 'src/PROG.cbl', content: PROG }, + ]; + const graph = createKnowledgeGraph(); + seedFiles(graph, files); + processCobol(graph, files, new Set(files.map((f) => f.path))); + + const targets = cobolCopyTargets(graph); + expect(targets).toContain(generateId('File', 'copybooks/CUSTREC.cpy')); + expect(targets).not.toContain(generateId('File', 'vendor/EXTERNAL.cpy')); + }); + + it('fail-open: without a copybook dir, COPY EXTERNAL still emits onto vendor/EXTERNAL.cpy', () => { + const files = [ + { path: 'vendor/EXTERNAL.cpy', content: EXTERNAL }, + { path: 'src/PROG.cbl', content: PROG }, + ]; + const graph = createKnowledgeGraph(); + seedFiles(graph, files); + processCobol(graph, files, new Set(files.map((f) => f.path))); + + const targets = cobolCopyTargets(graph); + expect(targets).toContain(generateId('File', 'vendor/EXTERNAL.cpy')); + }); + + it('P1-A: uppercase .CPY extension does not break stem extraction', () => { + const files = [ + { path: 'copybooks/CUSTREC.CPY', content: CUSTREC }, + { path: 'src/PROG.cbl', content: PROG }, + ]; + const graph = createKnowledgeGraph(); + seedFiles(graph, files); + processCobol(graph, files, new Set(files.map((f) => f.path))); + + const targets = cobolCopyTargets(graph); + expect(targets).toContain(generateId('File', 'copybooks/CUSTREC.CPY')); + }); + + it('P1-B: polyglot allPathSet with copy/cpy segments does not latch preferred-class', () => { + const files = [ + { path: 'copybooks/CUSTREC.cpy', content: CUSTREC }, + { path: 'vendor/EXTERNAL.cpy', content: EXTERNAL }, + { path: 'src/PROG.cbl', content: PROG }, + ]; + const polyglotPaths = new Set([ + 'docs/copy/README.md', + 'src/copy/clipboard.ts', + ...files.map((f) => f.path), + ]); + const graph = createKnowledgeGraph(); + seedFiles(graph, files); + processCobol(graph, files, polyglotPaths); + + const targets = cobolCopyTargets(graph); + expect(targets).toContain(generateId('File', 'copybooks/CUSTREC.cpy')); + expect(targets).not.toContain(generateId('File', 'vendor/EXTERNAL.cpy')); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/cobol-import-target-parity.test.ts b/gitnexus/test/unit/scope-resolution/cobol-import-target-parity.test.ts index b46859c68..20625f51f 100644 --- a/gitnexus/test/unit/scope-resolution/cobol-import-target-parity.test.ts +++ b/gitnexus/test/unit/scope-resolution/cobol-import-target-parity.test.ts @@ -66,17 +66,19 @@ function legacyResolveCobolImportTarget( const upper = targetRaw.toUpperCase(); // Check copybook files first for (const fp of allFilePaths) { - const ext = path.extname(fp).toLowerCase(); - if (!LEGACY_COPYBOOK_EXTENSIONS.has(ext)) continue; - const basename = path.basename(fp, ext).toUpperCase(); + const extRaw = path.extname(fp); + const extLower = extRaw.toLowerCase(); + if (!LEGACY_COPYBOOK_EXTENSIONS.has(extLower)) continue; + const basename = path.basename(fp, extRaw).toUpperCase(); if (basename === upper) return fp; } // Also search COBOL source files (.cbl, .cob, .cobol) const COBOL_SOURCE_EXTS = new Set(['.cbl', '.cob', '.cobol']); for (const fp of allFilePaths) { - const ext = path.extname(fp).toLowerCase(); - if (!COBOL_SOURCE_EXTS.has(ext)) continue; - const basename = path.basename(fp, ext).toUpperCase(); + const extRaw = path.extname(fp); + const extLower = extRaw.toLowerCase(); + if (!COBOL_SOURCE_EXTS.has(extLower)) continue; + const basename = path.basename(fp, extRaw).toUpperCase(); if (basename === upper) return fp; } return null; @@ -119,13 +121,16 @@ const STEMS = ['CUSTREC', 'custrec', 'AcctRec', 'PAYROLL', 'BOOK', 'COMMON', 'TA */ const EXTS = ['.cpy', '.copybook', '.CPY', '.cbl', '.cob', '.cobol', '.CBL', '.txt', '']; -function corpus(seed: number, fileCount: number): Set { +/** Directories with no well-known copybook segment — fail-open parity only. */ +const FAIL_OPEN_DIRS = ['', 'src', 'jcl/proclib', 'win\\dir']; + +function corpusFromDirs(seed: number, fileCount: number, dirs: readonly string[]): Set { const files = new Set(); for (let i = 0; i < fileCount; i++) { const a = mix(seed * 7919 + i); const b = mix(a ^ 0x9e3779b9); const c = mix(b ^ 0x85ebca6b); - const dir = DIRS[a % DIRS.length]; + const dir = dirs[a % dirs.length]; const stem = STEMS[b % STEMS.length]; const rel = `${stem}${EXTS[c % EXTS.length]}`; files.add(dir === '' ? rel : `${dir}/${rel}`); @@ -137,6 +142,10 @@ function corpus(seed: number, fileCount: number): Set { return files; } +function corpus(seed: number, fileCount: number): Set { + return corpusFromDirs(seed, fileCount, DIRS); +} + /** * `COPY` operands as they appear in source, plus the spellings that reach the * corpus's awkward files. Lower-case entries are what breaks if the target @@ -166,11 +175,31 @@ const TARGETS = [ const REPOS = 40; +function hasPreferredCopyDir(files: ReadonlySet): boolean { + for (const fp of files) { + const parts = fp.split('/'); + for (let i = 0; i < parts.length - 1; i++) { + if ( + parts[i] === 'copybooks' || + parts[i] === 'COPYBOOKS' || + parts[i] === 'cpy' || + parts[i] === 'copy' + ) { + return true; + } + } + } + return false; +} + describe('COBOL COPY-target index hoist — output parity with the pre-change scans (#2908)', () => { - it('agrees with the verbatim pre-change resolver over the generated corpus', () => { + it('agrees with the verbatim pre-change resolver over fail-open corpora (no well-known copybook dir)', () => { let checked = 0; for (let repo = 0; repo < REPOS; repo++) { - const files = corpus(repo, 6 + (repo % 25)); + const files = corpusFromDirs(repo, 6 + (repo % 25), FAIL_OPEN_DIRS); + expect(hasPreferredCopyDir(files), `repo=${repo} accidentally grew a copybook dir`).toBe( + false, + ); for (const target of TARGETS) { expect(resolve(target, files), `cobol "${target}" repo=${repo}`).toEqual( legacyResolveCobolImportTarget(target, files), @@ -181,6 +210,34 @@ describe('COBOL COPY-target index hoist — output parity with the pre-change sc expect(checked).toBe(REPOS * TARGETS.length); }); + it('COPY EXTERNAL misses vendor/EXTERNAL.cpy when a copybooks/ dir is present (#2967)', () => { + const files = new Set(['copybooks/CUSTREC.cpy', 'vendor/EXTERNAL.cpy', 'src/PROG.cbl']); + expect(resolve('EXTERNAL', files)).toBeNull(); + expect(resolve('CUSTREC', files)).toBe('copybooks/CUSTREC.cpy'); + }); + + it('fail-open: without a copybook dir, COPY EXTERNAL still basename-matches (#2967)', () => { + const files = new Set(['vendor/EXTERNAL.cpy', 'src/PROG.cbl']); + expect(resolve('EXTERNAL', files)).toBe('vendor/EXTERNAL.cpy'); + }); + + it('P1-A: uppercase .CPY extension does not break stem extraction', () => { + const files = new Set(['copybooks/CUSTREC.CPY', 'src/PROG.cbl']); + expect(resolve('CUSTREC', files)).toBe('copybooks/CUSTREC.CPY'); + }); + + it('P1-B: polyglot file set with copy/cpy segments does not latch preferred-class', () => { + const files = new Set([ + 'docs/copy/README.md', + 'src/copy/clipboard.ts', + 'copybooks/CUSTREC.cpy', + 'vendor/EXTERNAL.cpy', + 'src/PROG.cbl', + ]); + expect(resolve('CUSTREC', files)).toBe('copybooks/CUSTREC.cpy'); + expect(resolve('EXTERNAL', files)).toBeNull(); + }); + it('the corpus actually resolves things (the parity arm is not vacuous)', () => { // A corpus that resolved nothing would make the arm above pass on // `null === null` forever. Measured on this corpus: 390 hits. @@ -264,27 +321,21 @@ const HANDBUILT: readonly HandBuilt[] = [ expected: 'copybooks/CUSTREC.cpy', }, { - why: 'the extension is matched LOWER-cased, so `Foo.CPY` is a copybook at all', + why: 'the extension is matched LOWER-cased for tier, stem stripped with lowercase ext, so `Foo.CPY` is reachable as FOO', files: ['copybooks/Foo.CPY'], - target: 'FOO.CPY', + target: 'FOO', expected: 'copybooks/Foo.CPY', }, { - why: '`path.basename(fp, ext)` strips case-SENSITIVELY, so `Foo.CPY` is NOT reachable as FOO', - files: ['copybooks/Foo.CPY'], - target: 'FOO', - expected: null, - }, - { - why: 'a `.CPY` file keyed with its suffix loses `BOOK` to a `.cbl` in the later tier', + why: 'a `.CPY` file (uppercase ext) is keyed without suffix, hits in tier 1', files: ['x/BOOK.cbl', 'y/BOOK.CPY'], target: 'BOOK', - expected: 'x/BOOK.cbl', + expected: 'y/BOOK.CPY', }, { - why: 'an uppercase source extension is a source file (`.CBL` → tier 2, keyed with its suffix)', + why: 'an uppercase source extension is a source file (`.CBL` → tier 2, keyed without suffix)', files: ['src/Pay.CBL'], - target: 'PAY.CBL', + target: 'PAY', expected: 'src/Pay.CBL', }, { diff --git a/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts b/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts index ffd705c0c..957bda48a 100644 --- a/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts +++ b/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts @@ -358,7 +358,11 @@ const CASES: ReadonlyMap = new Map([ fromFile: 'src/PROG.cbl', resolutionConfig: undefined, external: 'EXTERNAL', - decoy: 'vendor/EXTERNAL.cpy', + // After the copybook-dir preference, vendor/EXTERNAL.cpy is intentionally + // unreachable (that is the #2967 fix). The reachable decoy is the in-repo + // copybook; vendor/EXTERNAL.cpy stays in `files` so EXTERNAL→[] is not a + // vacuous miss of an empty workspace. + decoy: 'copybooks/CUSTREC.cpy', reachesDecoy: 'CUSTREC', }, ], @@ -405,7 +409,6 @@ const KNOWN_GAPS: ReadonlyMap = new Map `Sources/Foundation/Thing.swift`'], [SupportedLanguages.C, '`stdio.h` -> `src/stdio.h`'], [SupportedLanguages.CPlusPlus, '`cstdio.h` -> `src/cstdio.h`'], - [SupportedLanguages.Cobol, '`EXTERNAL` -> `vendor/EXTERNAL.cpy`'], ]); /**