diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index b04d9fcee..f211b12c3 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -529,47 +529,101 @@ jobs: run: node --import tsx bench/cpp-qualified-ns/measure.mjs --check working-directory: gitnexus - - name: Import-target resolution guards (#2877/#2878/#2879/#2880/#2872) - # Build-free: runs the Go/C#/Dart/Ruby/Kotlin import-target resolvers - # over ONE shared corpus and asserts each returns an unchanged target - # set (a fingerprint per language AND per arm), that per-import cost - # stays independent of corpus size AND of path depth, that the absolute - # small-arm cost holds — a constant-factor regression that grows both - # scale arms equally passes every ratio — and that the shared - # WorkspaceFileIndex C# and Ruby retain stays within an absolute byte - # ceiling. Each of those resolvers used to scan the whole workspace per - # import (Ruby rebuilt a suffix index per `require`), so resolution was - # O(imports × files); the same corpus shape scores >3.3 against the - # pre-fix implementations. The corpus SHAPE is asserted too — a - # fingerprint alone cannot tell a legitimate resolution change from a - # corpus quietly shrunk below the size the timing arms need. + - name: Import-target resolution guards (every registered language, #2877–#2909, PR #2911) + if: ${{ !cancelled() }} + # Build-free: runs EVERY import-target resolver registered in + # SCOPE_RESOLVERS — plus C# a second time WITH csproj configs, over the + # identical corpus, because the no-csproj arm returns before it can + # reach the leg #2902 indexed. One arm per registered language over ONE + # shared corpus, and no registered language ungated. That is enforced, + # not enumerated: measure.mjs derives its list from a LANG_REGISTRY + # table and its --check inventory arm reconciles that table against + # SCOPE_RESOLVERS in both directions, so a language roster typed out + # here would only be a second copy that can go stale — this one did. + # A C/C++ #include is an import site for this purpose and is gated like + # every other registered language (its headers arrive through + # resolutionConfig rather than allFilePaths, which is the one structural + # difference — see `newPass`). + # + # Asserts each returns an unchanged target set (a fingerprint per + # language AND per arm), that per-import cost stays independent of + # corpus size AND of path depth, that the absolute small-arm cost holds + # — a constant-factor regression that grows both scale arms equally + # passes every ratio — and that the per-pass index eight of them retain + # stays within an absolute byte ceiling. The corpus SHAPE is asserted + # too: a fingerprint alone cannot tell a legitimate resolution change + # from a corpus quietly shrunk below the size the timing arms need. + # + # Several arms exist because an arm that stops MEASURING otherwise + # passes. The heap arms drive real resolvers and carry a FLOOR as well + # as a ceiling: when buildSuffixIndex's suffix maps went lazy, four arms + # that called the builder directly read 0 B, and 0 B is under every + # ceiling. EVERY budget is checked for PRESENCE first, timing and heap + # alike, because `got > undefined` is false and `got < ceiling * + # undefined` is false too, so deleting a budget key deleted its gate — + # and the two heap scalars gate all eight heap arms at once. The heap + # arm's own corpus shape (its two file counts, its path depth and the + # probe it resolves) is asserted by the same loop as the timing arms, + # because those four decide WHAT it measures. And an inventory arm + # reconciles the bench's language table against SCOPE_RESOLVERS itself, + # so a newly registered resolver cannot ship ungated the way JavaScript + # did. + # + # The resolvers gated first were added as their own O(imports × files) + # scans were indexed away (Ruby rebuilt a suffix index per `require`; + # COBOL scanned twice per `COPY`), and the same corpus shape scores >3.3 + # against those pre-fix implementations. The rest were ungated until + # this PR, which is not a theoretical gap: PR #2911 found JavaScript + # reaching suffixResolve with no index at all — 25 972 µs per import at + # 8000 files, protected only by unit tests. This step is what stops the + # next one shipping. # # SCOPE: "independent of corpus size" holds for UNIQUE-LEAF layouts, # where no two directories share a last segment and no two files share a # basename — which is what the small/large/deep arms are, and where # every index bucket holds exactly one entry. The `collide` arm runs the # identical workload on the layout these languages are actually written - # in (svcN/internal, SrcN/Models, a repeated basename per package); - # there the bucket grows with the file count by construction and go, - # csharp and dart legitimately score 2.1–3.9, so that arm carries its - # own per-language budget. It is a scope limit, not a regression — the - # indexed code is still faster on that shape than the pre-change scan. + # in (svcN/internal, SrcN/Models, a repeated basename per package, four + # SPM modules instead of fifty); there the bucket grows with the file + # count by construction and go, csharp, dart, java, swift and c/cpp + # legitimately score 2.1–3.9, so that arm carries its own per-language + # budget. It is a scope limit, not a regression — the indexed code is + # still faster on that shape than the pre-change scan. Rust is the one + # language whose collide arm is NOT a shared-leaf layout: it probes + # candidate paths and is provably flat in the file count, so its arm is + # a deep module tree that varies `::` segment count instead — the axis + # its cost actually has. # # --expose-gc enables the retained-heap arm; --check REFUSES to run # without it rather than passing with the memory gate silently skipped. - # ~14 s. REPS is 15 (matching bench/cfg) rather than a cheaper 5 or 7 - # because depth_ratio divides two sub-3 ms numbers and at those settings - # it tripped its own budget roughly 1 run in 20 — the estimator was - # fixed instead of the budget widened; distributions in _arms_note. + # ~44–45 s, which is essentially unchanged from the ~46 s it cost + # before: the timing phase did fall from 39.8 s to 28.7 s when the + # min-of-N estimator became per-language, but the inventory arm's one + # dynamic import (pipeline/registry.ts pulls in every registered + # provider) costs 6–10 s depending on the box and consumes almost all of + # that. Report mode, which does not load the registry, is the mode that + # got faster: ~33–35 s. Kept as-is because this job runs minutes clear + # of the sharded coverage job that gates the merge, so the seconds buy + # no merge latency — see COST in the bench header. The ts + # family (javascript/typescript/vue) is still the largest block, 8.8 s, + # because suffixResolve probes ~39 extensions per path part on a miss. + # If this ever has to shrink, drop collide/collide_large for typescript + # and vue (−3.9 s) — the only cut that removes near-duplicate work + # rather than coverage. N is 15 (matching bench/cfg) for every language + # whose cheapest arm is under 5 ms, because depth_ratio divides two + # sub-3 ms numbers and at 5 or 7 it tripped its own budget roughly 1 run + # in 20; the six languages whose cheapest arm is 20-28 ms drop to 7-8, + # where the measured overshoot is at most 6.3%. The estimator was fixed + # rather than the budget widened; distributions in _arms_note. # The Kotlin arm here is a second corpus, not a replacement for the # kotlin-import-target bench below, which carries tie-break probes (both # file-set iteration orders, the four-tier cascade) this one does not. - # A failing step aborts every step after it in this job (#2895), which - # cuts both ways: parking a new gate at the end is not safety, it is the - # slot least likely to execute. This one sits with the other - # resolver-index guards; the estimator fix above is what makes that - # safe, and #2899 carries the `if: ${{ !cancelled() }}` that fixes the - # masking for every step at once. + # It sits with the other resolver-index guards rather than at the end of + # the job: parking a new gate last is not safety, it is the slot least + # likely to execute (#2895 measured the last two guards running zero + # times in 13 runs). #2899 landed the `if: ${{ !cancelled() }}` below, + # which is what makes position irrelevant — a failing step no longer + # aborts the ones after it. # Rationale, budgets and the measured blind spot: see the header of # measure.mjs and _blind_spot in baselines.json. run: node --expose-gc --import tsx bench/import-target/measure.mjs --check diff --git a/gitnexus/bench/import-target/baselines.json b/gitnexus/bench/import-target/baselines.json index 18698f35f..c5b57cc66 100644 --- a/gitnexus/bench/import-target/baselines.json +++ b/gitnexus/bench/import-target/baselines.json @@ -1,43 +1,127 @@ { - "_what": "Baselines for bench/import-target/measure.mjs — the Go, C#, Dart, Ruby (#2877/#2878/#2879/#2880) and Kotlin (#2872) import-target resolvers on one shared corpus.", - "_fingerprint_note": "Per-language sha256 over every distinct fromFile|target -> resolved target. A change here is a BEHAVIOUR change: the resolver returned a different target set, and IMPORTS/CALLS edges moved. Explain it, never re-baseline to make CI green. For the four languages this PR changed, the pre-change implementations produce these same values on this corpus at both 400 and 1600 files — that is what makes the index hoist a performance change. The tie-break-level proof lives in test/unit/scope-resolution/import-target-index-parity.test.ts (verbatim copies of the pre-change code, diffed) and, for Kotlin, in test/unit/scope-resolution/kotlin/kotlin-import-target-parity.test.ts.", - "_shape_note": "files/imports/resolved/distinct_outcomes AND the fingerprint are asserted exactly, per scale. A fingerprint alone cannot tell a legitimate resolution change from a corpus quietly shrunk below the size at which the timing arms can see anything; conversely the counts alone cannot see a defect confined to one arm, because the arms differ only in path padding and directory layout and both of those are count-neutral by design. Two cross-arm assertions close the remaining hole: the deep and collide arms must resolve exactly what small resolves (they are the same workload), and each of their fingerprints must DIFFER from small's (they are not the same corpus). Without the second, setting DEEP_PAD to 0 — which deletes the entire depth arm — moves no asserted number and prints PASS; the same is true of a collideDir that forwards to uniqueDir.", - "_arms_note": "Five timing arms, one memory arm and one deterministic arm elsewhere, because none of them gates alone. scaling_ratio (t_large/t_small)/(1600/400) catches cost growing with FILE COUNT — the #2877-#2880 regression itself. depth_ratio (t_deep/t_small at a FIXED file count, ~6x the path components) catches cost growing with path DEPTH, which scaling_ratio divides out and structurally cannot see; buildSuffixIndex (C#, Ruby) and Kotlin suffixByStem emit one entry per component, so they legitimately sit above 1.0 while Go and Dart, whose indexes are depth-free, sit at ~1.0. collide_scaling_ratio is the same measurement on a SHARED-LEAF layout (svcN/internal, SrcN/Models, a repeated mod0.dart/mod0.rb basename in every package) carrying an identical file, import and resolved count: the small/large/deep arms mint one directory name per index, so every index bucket in them holds exactly ONE entry (measured: max last-segment bucket 1 and max matching directories 1 for go and csharp at 400 and 1600 files; max basename bucket 1 for dart and ruby), and bucket cardinality is the only non-constant term the new indexes have. On the shared-leaf shape go, csharp and dart legitimately score 2.1-3.9 because the bucket grows with the file count BY CONSTRUCTION — this is a limit on the SCOPE of the 'independent of corpus size' claim, not a regression (the indexed code is still faster there than the pre-change full scan); their collide budgets say so honestly instead of pretending 1.8. Ruby and Kotlin answer from keyed maps and are collision-immune, so they keep the linear 1.8 budget and that immunity is the assertion. The collide arm is also the only arm that reaches filesDirectlyInPkgDir's dirCount > 1 merge (go: 388 multi-directory calls at 400 files, up to 9 directories; 1517 at 1600 files, up to 34). small_ms_ceiling and collide_ms_ceiling are ABSOLUTE (~4x the measured arm), because a constant-factor regression that grows both scale arms equally passes every ratio. NOISE, measured rather than assumed: depth_ratio divides two sub-3 ms numbers (Dart's are sub-1 ms) and is by far the noisiest arm here, so it set REPS for the whole file. fastest() is a min-of-N estimator, so N is the knob. Over 22 --check runs on an idle box, peak-to-peak: at REPS=5 go ran 0.757-1.748 (2.31x) and tripped its own 1.6 budget about 1 run in 20; at REPS=7 (the kotlin-import-target setting) Dart still ran 0.678-2.043 (3.01x) and tripped once; at REPS=15 (bench/cfg, bench/schema-pairs, bench/callable-value-flow) every language collapsed to a 1.13-1.26x swing with 22/22 passing - go 0.968-1.120, csharp 2.963-3.740, dart 1.031-1.248, ruby 1.406-1.588, kotlin 2.097-2.404, i.e. a maximum sitting at 70-78% of each budget. The budgets were NOT widened; the estimator was fixed instead, which is why the headroom above is real rather than granted. A --check run costs ~14 s. heap_ceiling_bytes is the retained size of the shared WorkspaceFileIndex, the only arm here that can see memory: buildSuffixIndex emits three maps at O(files x depth), the profile package-dir-index.ts cites #2649 to avoid for itself, and C#'s no-csproj leg retained nothing at BASE but now builds it unconditionally. It is measured at 8000 and 32000 files at HEAP_PAD depth rather than at the timing arms' sizes, because the finding is an ABSOLUTE footprint at repository scale. Its ceiling is 1.5x the measured arm, and the DIFFERENCE FROM THE 4x TIMING CONVENTION IS DELIBERATE — do not harmonise it back. 4x exists because runner contention dominates a wall-clock number; this one has essentially no measurement noise (spread 656 B on 77.2 MB across 22 runs, 0.00085%, and identical to the byte across separate processes), so 4x would throw away almost all of the gate's power and sail straight past the regression this arm exists to catch. 1.5x still tolerates ~50% of cross-platform and Node-version drift, which is ~50000x the observed process-to-process spread and far more than a Node major bump plausibly moves heapUsed accounting; it catches a duplicated index (+100%) or a second exactMap-sized suffix map (+~85%). RESIDUAL, stated so nobody assumes otherwise: one additional dirMap-sized map is only +18% of the total (dirMap itself measures 15.7% of the C# index and 20.3% of the Ruby one) and would still pass. heap_ratio_budget, the linear-growth check across the 4x file-count gap, is the orthogonal arm: it sees per-file and per-depth growth but not a constant factor.", - "_triage": "Every ratio and ms ceiling here is a TIMING signal — re-run on an idle machine before investigating; runner contention dominates. depth_ratio is the noisiest of them by a wide margin (it divides two sub-3 ms numbers, and Dart's are sub-1 ms): if exactly one arm fails and it is that one, suspect the machine first. REPS is 15 rather than this bench's original 5 specifically to hold that arm's peak-to-peak swing under 1.26x — see _arms_note for the measured distributions — so a depth_ratio failure that REPRODUCES is a real signal, not noise. The fingerprint, shape and heap arms are the opposite: deterministic (the heap arm reproduces to within 0.001% across processes), a re-run never changes them, and they must never be wished away.", - "_floor": "Measured against the pre-change implementations on THIS corpus at 150/600 files: go 3.36, csharp 4.10, dart 3.32, ruby 3.87. The issues report 4.00 / 3.43 / 4.05 on their own corpora; those are DIFFERENT numbers from different repositories and are not reproduced here — what they and these share is that both independently land in the quadratic band, well clear of the ~1.0 a linear result gives. Note also that this floor was taken at 150/600 while the gate runs at 400/1600, so it is a lower bound on what the pre-change code would score today. Kotlin's own bench measured its pre-index floor at 3.737. The 1.8 budget sits well above the linear result and well below every one of those.", + "_what": "Baselines for bench/import-target/measure.mjs \u2014 EVERY import-target resolver registered in SCOPE_RESOLVERS, on one shared corpus, plus csharp a second time WITH csproj configs. One entry per registered language and one more for the csproj arm, no registered language ungated \u2014 and that is ASSERTED rather than asserted-in-a-comment, which is also why no roster of language names is kept in this prose to go stale: measure.mjs derives its language list from a LANG_REGISTRY table and a --check inventory arm reconciles that table against SCOPE_RESOLVERS in both directions. A C/C++ #include is an import site for this purpose and is gated like every other registered language. csharp and csharp_csproj resolve the IDENTICAL file corpus (buildFiles aliases the two) and differ in exactly one thing: whether csharpConfigs is supplied. Without that second arm the csproj namespace-directory index ships unmeasured, because every C# import in the no-csproj arm returns before reaching it. C and C++ follow that same precedent for a different context \u2014 their HEADERS arrive through resolutionConfig rather than through allFilePaths, and augmentedFilePaths unions the two once per pass, so the corpus is split at newPass rather than pre-merged. The first nine were added as their own O(imports x files) scans were indexed away (#2877/#2878/#2879/#2880, #2872, #2901, #2902, #2908) and this is the forward guard on each; the other eight were ungated until now, and PR #2911 \u2014 JavaScript reaching suffixResolve with no index at all, 25972 us per import at 8000 files \u2014 is what that costs.", + "_fingerprint_note": "Per-language sha256 over every distinct fromFile|target -> resolved target. A change here is a BEHAVIOUR change: the resolver returned a different target set, and IMPORTS/CALLS edges moved. Explain it, never re-baseline to make CI green. For the languages these PRs changed, the pre-change implementations produce these same values on this corpus at both 400 and 1600 files \u2014 that is what makes the index hoist a performance change. The tie-break-level proof lives in test/unit/scope-resolution/import-target-index-parity.test.ts (verbatim copies of the pre-change code, diffed) for Kotlin in test/unit/scope-resolution/kotlin/kotlin-import-target-parity.test.ts, and for the four resolvers added there in test/unit/scope-resolution/{php,java,cobol}-import-target-parity.test.ts and test/unit/import-resolvers/csharp-csproj-parity.test.ts, and for JavaScript in test/unit/scope-resolution/javascript-import-target-parity.test.ts (a differential over 211200 old-vs-new pairs, PR #2911). The eight languages added last have no per-language parity harness against a pre-change implementation and do NOT need one: nothing about their resolution changed, so there is no before to diff against. Their fingerprints are pure forward guards, minted from the current implementations, and their adapter-boundary index reuse is covered for every registered language at once by test/unit/scope-resolution/import-target-index-reuse.contract.test.ts. NOTE for csharp_csproj: on this corpus the #2902 indexed leg (step 3 of resolveCSharpImportInternal) is reached by 2221 of the 3200 small-arm imports but answers null for every one of them \u2014 the 979 that resolve do so at step 2 \u2014 so this fingerprint pins that legs cost and its null answers, while its positive tie-breaks (first-occurrence, unanchored substring, iteration order) are pinned by csharp-csproj-parity.test.ts.", + "_shape_note": "files/imports/resolved/distinct_outcomes AND the fingerprint are asserted exactly, per scale. A fingerprint alone cannot tell a legitimate resolution change from a corpus quietly shrunk below the size at which the timing arms can see anything; conversely the counts alone cannot see a defect confined to one arm, because the arms differ only in path padding and directory layout and both of those are count-neutral by design. Two cross-arm assertions close the remaining hole: the deep and collide arms must resolve exactly what small resolves (they are the same workload), and each of their fingerprints must DIFFER from small's (they are not the same corpus). Without the second, setting DEEP_PAD to 0 \u2014 which deletes the entire depth arm \u2014 moves no asserted number and prints PASS; the same is true of a collideDir that forwards to uniqueDir. THE HEAP ARM IS ASSERTED THE SAME WAY, by the same loop, and was not before: files_small, files_large, path_segments and probe decide WHAT it measures, and every one of them was reported and compared to nothing. Swapping HEAP_PROBE_TARGET.csharp_csproj for a target matching no CSPROJ_CONFIGS rootNamespace skips the whole config loop, so the getFilesInDir and getInsensitive legs never run and the arm the header calls the witness that the read pattern IS the footprint quietly becomes a two-map arm \u2014 73703384 -> 59921216 B, ratio 1.017 -> 1.011, ceiling and floor both still passing and --check still exiting 0. Setting HEAP_SMALL equal to HEAP_LARGE is the same hole from the other side: ratio goes to ~1.0 by construction and bytes_large never moves. bytes_small and bytes_large are deliberately NOT asserted for equality \u2014 heap_ceiling_bytes and the heap_reading_bytes floor bound them with ~50% either way, because heapUsed accounting moves across platforms and Node majors and an exact byte assertion would be a re-baseline per runner. THE CONTEXT ARM IS ASSERTED THE SAME WAY, by the same loop, and more strictly than either: target, with_context and without_context are exact strings with no tolerance at all, because the arm resolves one import over a three-file corpus and has no measurement noise to tolerate. A separate check requires the last two to DIFFER, for the same reason deep.fingerprint must differ from small.fingerprint \u2014 a probe on which both call shapes agree asserts one number twice. Both halves run through resolveOne, so what the arm gates is this bench threading run.ts's fifth argument, not the resolvers' behaviour.", + "_arms_note": "Five timing arms, one memory arm and one deterministic arm elsewhere, because none of them gates alone. scaling_ratio (t_large/t_small)/(1600/400) catches cost growing with FILE COUNT \u2014 the #2877-#2880, #2901, #2902 and #2908 regressions themselves; every one of those legs was Theta(files) per import, so a revert scores ~4 here by construction. depth_ratio (t_deep/t_small at a FIXED file count, ~6x the path components) catches cost growing with path DEPTH, which scaling_ratio divides out and structurally cannot see; buildSuffixIndex (C#, Ruby, PHP, Java) and Kotlin suffixByStem emit one entry per component, so they legitimately sit above 1.0 while Go, Dart and COBOL, whose indexes are depth-free, sit at ~1.0. csharp's depth_budget has now been retightened twice for the same reason, and the second time it did lock the win in. It was 5 against a then-measured 3.318; #2903 made buildSuffixIndex's dirMap lazy and it became 3.5 against 2.31, with the file stating plainly that 3.5 did NOT lock that win in because a revert to an eager dirMap scores 3.318 and passes. Extending the laziness to the two SUFFIX maps drops it again, to 1.438 (java likewise 2.214 -> 1.402), because the deep arm has ~6x the path components and an O(files x depth) build of a map the no-csproj leg never reads is exactly the cost that scales with depth. Both are now 2.2, which is this file's 1.5x convention against measurements whose own peak-to-peak over 4 runs is 1.04x and 1.07x \u2014 and 2.2 DOES lock it in: an eager rebuild scores 2.3+ and fails. The other fifteen depth budgets sit at 1.37-1.75x measured and are unchanged. collide_scaling_ratio is the same measurement on a SHARED-LEAF layout (svcN/internal, SrcN/Models, com/example/model in every service, a repeated mod0.dart/mod0.rb/Mod0.cpy basename) carrying an identical file, import and resolved count: the small/large/deep arms mint one directory name per index, so every index bucket in them holds exactly ONE entry (measured: max last-segment bucket 1 and max matching directories 1 for go and csharp at 400 and 1600 files; max basename bucket 1 for dart and ruby), and bucket cardinality is the only non-constant term the new indexes have. On the shared-leaf shape go, csharp, dart and java legitimately score 2.1-3.9 because the bucket grows with the file count BY CONSTRUCTION \u2014 this is a limit on the SCOPE of the \"independent of corpus size\" claim, not a regression (the indexed code is still faster there than the pre-change full scan); their collide budgets say so honestly instead of pretending 1.8. Ruby, Kotlin, PHP and COBOL answer from keyed maps and are collision-immune, so they keep the linear 1.8 budget and that immunity is the assertion. csharp_csproj is the one arm that runs the other way: its shared leaf collapses dirsByLastSegment to the single key Models, so the slash-free sweep (see CSPROJ_CONFIGS) is CHEAPER on the collide layout than on the unique one and its expensive scale arm is large, not collide_large. Its 1.8 collide budget is therefore the linear one, and the arm that carries its real cost is the unique one. The collide arm is also the only arm that reaches filesDirectlyInPkgDir's dirCount > 1 merge (go: 388 multi-directory calls at 400 files, up to 9 directories; 1517 at 1600 files, up to 34) and the only one that reaches COBOL's copybook-over-source tier tie-break, which needs one bookname to name two files. small_ms_ceiling and collide_ms_ceiling are ABSOLUTE (~4x the measured arm), because a constant-factor regression that grows both scale arms equally passes every ratio. The five arms added here use 4.2x, the middle of the 3.7-4.6x the original five already carry; the two COBOL arms use ~5x, the multiplier dart's sub-1 ms arm has always carried, because a fixed scheduler hiccup is a larger fraction of a smaller number \u2014 measured over 8 runs they sat at 0.25-0.37 ms and 0.18-0.30 ms, and the pre-#2908 two-scans-per-COPY implementation costs ~300 ms on the same arm, so 2.0 and 1.5 still separate fixed from broken by two orders of magnitude. NOISE, measured rather than assumed: depth_ratio divides two sub-3 ms numbers (Dart's are sub-1 ms) and is by far the noisiest arm here, so it set N for the whole file. fastest() is a min-of-N estimator, so N is the knob. Over 22 --check runs on an idle box, peak-to-peak: at N=5 go ran 0.757-1.748 (2.31x) and tripped its own 1.6 budget about 1 run in 20; at N=7 (the kotlin-import-target setting) Dart still ran 0.678-2.043 (3.01x) and tripped once; at N=15 (bench/cfg, bench/schema-pairs, bench/callable-value-flow) every language collapsed to a 1.13-1.26x swing with 22/22 passing. The budgets were NOT widened; the estimator was fixed instead, which is why the headroom above is real rather than granted. N IS NOW PER LANGUAGE, and that is a refinement of the same finding rather than a retreat from it. The overshoot of min-of-K against min-of-15 is a function of the CELL's absolute duration, not of the language: replayed against two independent runs' full sample sets, the worst overshoots at K=7 land on swift.small (0.43 ms, 31.8%) and dart.collide (1.5 ms, 37.6%), while every cell at or above 10 ms overshoots by at most 6.3%. So repsFor() keeps 15 while a language's cheapest arm is under 5 ms and otherwise spends ~150 ms per cell, floored at 7 \u2014 15 for go, csharp, dart, kotlin, java, cobol, swift, rust, python, c and cpp (every language the flakiness above was ever about, cheapest arm 0.19-3.2 ms) and 7-8 for csharp_csproj, ruby, php, javascript, typescript and vue (cheapest arm 20-28 ms). Per LANGUAGE, not per cell, so all five arms of a language share one estimator and the four ratios stay comparisons of like with like. The replay passed all 85 cells on all five gates at 0.4-0.7 of budget and saved 12.8 s and 12.4 s of a 46 s run; min-of-7 also reads slightly HIGHER than min-of-15, so the ceilings get marginally more sensitive rather than less. Confirmed on 4 fresh runs with the adaptive estimator live: every small arm inside 1.12x peak-to-peak and every collide arm inside 1.07x, with the six 7-8 rep languages at 1.008-1.071 \u2014 no worse than the 11 that kept 15. The chosen N is reported per language as `reps`. heap_ceiling_bytes bounds the retained per-pass import index, the only arm here that can see memory: buildSuffixIndex emits maps at O(files x depth), the profile package-dir-index.ts cites #2649 to avoid for itself, and csharp, ruby, php and java all retained NOTHING across imports at BASE (C#'s no-csproj leg and PHP's and Java's every leg re-scanned the raw Set; Ruby rebuilt and discarded a suffix index per require). It is measured at 8000 and 32000 files at HEAP_PAD depth rather than at the timing arms' sizes, because the finding is an ABSOLUTE footprint at repository scale. THE ARM NOW READS WHAT THE LANGUAGE READS, and that change is the whole reason this file was re-baselined. Four of these arms used to call getWorkspaceFileIndex(set) directly and then read index.all.length, which asks no suffix question at all \u2014 harmless only while buildSuffixIndex built both maps eagerly. The moment they went lazy the direct call built NO map, csharp, ruby, php and java each reported 0 B at 32000 files, and 0 B is under every ceiling: --check printed PASS over four gates that had silently become ceilings over nothing, which is precisely the failure this file's own header warns about for rust and cobol. Every arm now resolves a real MISSING import through the real resolver (HEAP_PROBE_TARGET, asserted to miss), so the maps it forces are the maps production forces, and a resolver that starts asking a new question moves the number without anyone editing the bench. That makes the READ PATTERN the dominant term, and the eight numbers say so: java 34958600 B and csharp 29862200 B ask index.get and never getInsensitive; php 37579888 B asks getInsensitive and never get, plus its own first-proper-suffix map; ruby 41025360 B and javascript 26745296 B read get(s) || getInsensitive(s) and pay for both, the second DERIVED from the first; and csharp_csproj 73705944 B additionally asks getFilesInDir. csharp_csproj IS NOW GATED, reversing the earlier decision that it would be 'a ceiling on a duplicate': at +20.8% of the C# index it was one, and at 2.47x of it \u2014 same corpus, same getWorkspaceFileIndex, three maps instead of one \u2014 it is the witness that the read pattern is the footprint. The old RESIDUAL note is superseded by that number: a dirMap-sized addition is no longer +18%, and a consumer that asks all three questions blows csharp's ceiling by 1.64x rather than sliding under it. A SECOND MEASUREMENT BIAS was removed at the same time and it moved every figure here, so do not read these against the old ones as if only the read pattern changed. buildFiles mints paths with template literals, which V8 keeps as ropes; the first traversal that slices one flattens it, allocating the flat string and dropping the rope's pieces, so a build measured over an unflattened corpus reports the index MINUS that net release \u2014 11% low, uniformly. bytes_small was read over a corpus a discarded warm-up pass had already flattened and bytes_large over a fresh one, so every ratio read ~0.85-0.89 for structures that are exactly linear in the file count. measureHeap now flattens each corpus before measuring it; all eight ratios read 0.998-1.017, and the warm-up pass is gone because with the corpus flat a language's first and second reads agree to within 0.3%. python's figure rises from 7624992 to 10362976 for this reason and not because anything regressed, and then to 10543152 (+1.7%) because #2913's nestedDirNames set is retained for the pass, and then FALLS to 6360936 (-39.7%) for a reason worth knowing: byBasename holds roughly one bucket per file, and building each with `[]` followed by `push` made V8 grow the backing store to its 16-slot minimum, so every single-file bucket retained 15 empty pointer slots. Constructing the one-element buckets directly (`set(base, [entry])`) is byte-identical in contents and 3.9 MiB smaller at 32000 paths — 37% of what this arm used to read was empty array slots — the ancestorsByDir memo itself is NOT in this reading, because python's probe target misses at the nested-name rejection and never reaches the walk, so this arm does not bound that memo; measured separately with a probe that does reach it, a 32000-file corpus with every file in its own 10-deep directory retains ~19 MB, which would clear this ceiling, so repointing python's heap probe at a walking spelling means re-recording the ceiling in the same change, and c is unchanged at 10018816 because its basename map does not slice paths. Its ceiling is 1.5x the measured arm, and the DIFFERENCE FROM THE 4x TIMING CONVENTION IS DELIBERATE \u2014 do not harmonise it back. 4x exists because runner contention dominates a wall-clock number; this one has essentially no measurement noise (across 4 runs the widest spread was 0.11% on python, 0.03% on csharp_csproj and 0.00% \u2014 identical to the byte \u2014 on ruby, php, java, javascript and c, and the same holds across separate processes), so 4x would throw away almost all of the gate's power and sail straight past the regression this arm exists to catch. 1.5x still tolerates ~50% of cross-platform and Node-version drift, far more than a Node major bump plausibly moves heapUsed accounting; it catches a duplicated index (+100%) or a second exactMap-sized suffix map (+~85%). heap_floor_fraction is the arm the 0 B incident proved was missing. A ceiling can only say 'not too big'; nothing said 'still measuring something', which is why four dead arms passed. The floor is 0.5 x each language's RECORDED READING (heap_reading_bytes), which is half the measured size and says so. It used to be 0.33 x the CEILING, described the same way \u2014 true only while every ceiling stayed at exactly 1.5x its reading, a convention this file states and nothing enforces, so re-tuning one ceiling upward would have loosened that language's floor by the same factor in the one direction a floor exists to watch. The two forms agree to within 0.8% for all eight today, so this is a correction of derivation, not of strength. It sits ~400x above the readings' own reproducibility and far below any collapse. A genuine 2x memory WIN trips it too, and that is intended: like a fingerprint move, it must be explained and re-baselined rather than absorbed. COBOL is left out for the opposite reason: its index is two Map, O(files) with no depth term, and at 32000 files its retained delta does not clear the noise of the measurement itself. heap_ratio_budget, the linear-growth check across the 4x file-count gap, is the orthogonal arm: it sees per-file and per-depth growth but not a constant factor. ---- THE EIGHT LANGUAGES ADDED LAST (swift, rust, python, javascript, typescript, vue, c, cpp) ---- They carry the SAME five arms and the same gates; what differs is which arm can actually fail for each, because each resolver has a different cost axis, and the budgets below say so instead of copying a number across. Every figure quoted is the MAXIMUM over 5 full runs on an idle box, and the peak-to-peak of every one of these arms stayed inside 1.10x over those runs \u2014 tighter than the 1.13-1.26x the original nine record, because none of these arms divides two sub-1 ms numbers the way dart depth_ratio does. depth_budget is ~1.5x measured throughout: swift 2.3 (1.487), rust 2.1 (1.377), javascript 2.1 (1.376), typescript 2.1 (1.381), vue 2.3 (1.563), c 3.0 (1.990), cpp 3.0 (1.999). PYTHON WAS 11 AGAINST 7.389 AND IS NOW 2.6 AGAINST 1.872, because #2913 fixed the resolver rather than the budget. Its INDEX was always depth-free; hasRepoCandidate and resolveAbsoluteFromFiles each rebuilt one ancestor prefix per directory component of the importer on EVERY import, and the index's own dirPrefixes build inserted one entry per component per file, so the resolver was quadratic in path depth where every other language here is linear or flat. The prefixes are a pure function of the importer's DIRECTORY, so they are now memoized per directory inside getPythonFileIndex (ancestorsByDir), the leading segment is rejected up front against a set of nested directory names, the module and package buckets are consulted before the walk rather than inside it, and the dirPrefixes build stops at the first ancestor already stored. All five fingerprints are byte-identical, so it is a hoist. The budget is 2.2, and BOTH numbers behind it were re-measured on a quiet box AFTER the context leg below started being measured, because that change moved the arm: the work it adds is depth-FLAT, so python's absolute cost more than doubled while depth_ratio FELL to 1.405-1.563 over 5 serial runs (peak-to-peak 1.11x). A budget carried over from before that change would have been slack against a smaller ratio. 2.2 is 1.41x the measured maximum, inside the 1.37-1.75x band the other fifteen sit in, and it LOCKS THE WIN IN: reverting the per-directory ancestor memo alone scores 2.524 and reverting the nested-name rejection alone scores 2.553, both measured under the current call shape, so each fails at 2.2 with 13% to spare. Do not read those two figures as the pre-#2913 cost — 7.239 was that, and the gap closed because the bare-import tier stopped walking at all (see below). The other two parts of the fix are not gated by this arm and are not meant to be: reverting the bucket prune or the dirPrefixes early break lands under any budget this arm's noise supports, so they are gated deterministically instead, by the prefix-parity and package-probe arms of test/unit/scope-resolution/python/python-importer-ancestors.test.ts and python-import-target-parity.test.ts, which go red on exactly those two mutations. A timing budget catches what it can measure; the counts catch the rest. THE BARE-IMPORT TIER (`import os`, single segment, no dot) was a separate O(depth) walk in import-resolvers/python.ts that this bench cannot see at all, because every python arm here spells its imports with a dot and returns at the `pathLike.includes('/')` guard before reaching it. It ran TWICE per `from x import y` — the package probe's recursion re-ran the whole tail on identical inputs — and is now one memoized chain plus an O(1) proof-of-absence against the index's basename buckets: 12/24/72 Set probes at depth 1/4/16 became a flat 2, and 11.615 us/import at 18 path components became 0.740. Gated by probe COUNT in test/unit/scope-resolution/python/python-import-probe-count.test.ts, not here. collide_scaling_budget splits three ways. Three languages scan a bucket that grows with the corpus and get their measured value x1.5: swift 4.9 (3.279 \u2014 its bucket is the module file list it RETURNS, and its collide arm is four modules instead of dirs of them so that bucket is fileCount/4, i.e. 100 files at 400 and 400 at 1600), c 3.8 (2.535) and cpp 4.0 (2.639, the same basename bucket its suffix fallback walks). Four answer from keyed maps and keep the linear 1.8 \u2014 python 1.097, javascript 1.083, typescript 1.053, vue 1.079 \u2014 and that immunity IS the assertion, exactly as for ruby, kotlin, php and cobol. RUST IS THE ONE ARM THAT WAS REDESIGNED RATHER THAN BUDGETED. It resolves by probing candidate paths with allFilePaths.has(...) and never searches, so its cost is O(path segments) and provably flat in the file count (1.095 scaling, 1.061 collide scaling): a shared-leaf collide arm for rust would have asserted nothing, which is worse than no arm. Its collide corpus is instead a deep module tree (src/l0/l1/l2/l3/l4/mod{d}) whose targets carry ~2x the :: segments, so the arm exercises the axis that CAN grow, its 1.8 budget asserts the flatness across file counts, and collide_ms_ceiling 19 bounds the absolute cost of the long-path probe. small_ms_ceiling and collide_ms_ceiling are ~4x measured as everywhere else: rust 10/19 (2.609/4.704), python 7/8 (1.76/1.929, retightened from 12/15 against 3.044/3.771 by #2913), javascript 85/89 (21.254/22.145), typescript 85/86 (21.250/21.464), vue 81/93 (20.164/23.227), c 7/11 (1.620/2.850), cpp 7/12 (1.581/3.009). Swift takes ~5x (2 against 0.421 and 4 against 0.821) \u2014 the multiplier dart and cobol already carry, because a fixed scheduler hiccup is a larger fraction of a sub-1 ms number. ONE CAVEAT ON THE THREE ts-FAMILY MS NUMBERS, stated because nothing else in this file would reveal it: resolveTsTarget carries a per-pass resolveCache keyed currentFile::importPath, which no other resolver here has, and ~10% of this corpus is repeat pairs. Their us/import is therefore a slight underestimate of a cold resolve. It is left in rather than defeated because it is what the real pipeline does, and it is identical across all three so the arms stay comparable. HEAP for the eight: rust, swift, typescript, vue, cpp and cobol are still NOT gated, all of them measured before being left out. rust builds no index on this hook (16 B at 8000 files, 0 B at 32000); swift holds one pointer per file-times-segment and mints no strings, reading 0.98 MB at 8000 files against 0.29 MB at 32000 \u2014 a 4x larger corpus reading 3x SMALLER, which is what a measurement below its own noise floor looks like, and the same reading cobol gives (0.54 MB then 0 B); typescript and vue duplicate javascript through the same builder over the same-shaped corpus, and cpp duplicates c (10021320 against 10016960, 0.04% apart). Those four duplications are the ONLY exclusions that still rest on 'it would be a duplicate', and they are duplicates of a builder AND of a read pattern, which is the pairing csharp_csproj failed once the read pattern started to matter \u2014 if any of the four ever diverges in what it ASKS the index, it earns an arm the same way csharp_csproj just did. All eight gated arms are read the same way now (retainedPassBytes, one real import), so unlike before they are directly comparable to one another. WALL CLOCK \u2014 ~33-35 s in report mode, down from ~46 s, and ~44-45 s for --check, which is essentially UNCHANGED from ~46 s. Only report mode got faster; do not read the pair as 46 -> 42. The breakdown is worth having before anyone trims it. Timing arms: go 2.02, csharp 1.09, csharp_csproj 3.22, dart 0.41, ruby 2.90, kotlin 0.85, php 3.46, java 1.57, cobol 0.09, swift 0.46, rust 0.85, python 1.22, javascript 3.23, typescript 2.72, vue 2.89, c 0.86, cpp 0.91 (28.7 s, from 39.8 s: repsFor() accounts for all of it, and every second of it comes from the six languages whose cheapest cell is 20-28 ms); heap arms 3.43 s for SEVENTEEN languages, from 2.06 s for eight (every registered language is measured now; the nine added cost 1.37 s, of which kotlin alone is 0.57 s \u2014 see _heap_bound_note), and 2.1 s came from 3.0 s for seven when flattening retired the warm-up pass; module load 3.9 s. --check pays one import that report mode does not: the inventory arm loads pipeline/registry.ts, which drags in every registered scope resolver and its providers. Measured in isolation with the bench's own static imports already resident, that import costs 6.3-6.5 s on one box and 9.3-10.0 s on another \u2014 i.e. it consumes almost the whole repsFor win, which is why --check did not get faster. It is loaded dynamically at the point of use rather than at the top of the file, so report mode does not pay it and both modes take their measurements in the same module state. IT WAS WEIGHED AND KEPT, on the number that decides it: the benchmarks job is not CI's critical path. On the last green run of main it took 9 m 23 s against 12 m 58 s for the sharded coverage job that gates the merge, so ~4 m 40 s of slack sits above this bench and those seconds buy zero merge latency. Moving the arm to a vitest file would move the registry load ONTO the critical path, and would weaken it as well: this reconciles LANG_REGISTRY's SupportedLanguages values, which are what the five dispatcher branches key off, whereas a test that cannot import measure.mjs can only reconcile this file's arm NAMES plus a hand-written rule for de-aliasing csharp_csproj. The contract test import-target-index-reuse.contract.test.ts already covers the ADAPTER-boundary contract for every registered resolver; this arm covers a different claim, that the BENCH covers the pipeline. The ts family is still the largest single block of the timing phase (8.8 s) \u2014 its cost is suffixResolve probing ~39 extensions per path part on a miss, which is the real resolver and cannot be tuned away from the bench side. IF IT HAS TO SHRINK, drop collide and collide_large for typescript and vue and nothing else: -3.9 s, and it is the only cut that removes near-duplicate work rather than coverage, because all three run the same resolveTsTarget over the same buildSuffixIndex and javascript keeps the collide arm that covers their shared collision axis. Do NOT reach for REPS_MAX: it is 15 because depth_ratio tripped its own budget about 1 run in 20 at 5 and once at 7, and lowering it would re-open that for the eleven languages whose cheapest cell is sub-5 ms \u2014 which is where every recorded trip happened. The six languages it was safe to lower have already been lowered, per language and from a measurement, by repsFor(). ---- THE FIFTH ARGUMENT (context) AND THE TWO ARMS IT MOVED ---- resolveOne now makes run.ts's five-argument call for the two hooks that declare a fifth parameter, so php and python time the legs behind it. Nothing else moved: the other fifteen arms are handed no context and build no ParsedFile[] at all, and over five runs their five ms numbers and four ratios sit exactly where they did. Both languages' ten fingerprints, resolved counts and distinct_outcomes are IDENTICAL \u2014 the leg AGREES with the cascade on this corpus, which is the whole reason the context arm had to be added rather than leaving the fingerprint to notice. PHP: small_ms 27.762 -> 35.125 (+26.5%) and collide_ms 29.407 -> 36.182 (+23.0%), which is filesByDirectory plus, on every import that resolves, a candidate gather over the resolved file's directory and a localDefs filter; the ms ceilings keep PHP's own 4.21x and 4.26x multipliers (117 -> 148, 125 -> 154). depth_ratio 1.144 -> 1.283 and the 1.9 budget is UNCHANGED, which makes it 1.48x measured rather than 1.66x: directoryAliases emits one entry per path segment, so filesByDirectory is O(files x depth) and the depth arm is the only one that can see it \u2014 that budget got TIGHTER relative to its measurement, not looser, and 1.48x sits inside the 1.37-1.75x band the other sixteen carry. Its heap reading rises 37576816 -> 49574008 (+31.9%) for the same structure, and the reading is the MEMO rather than the workspace it indexes: newPass allocates the ParsedFile objects before retainedPassBytes takes its baseline sample, so they sit outside the delta. PYTHON, WHOSE FIGURES ARE THE LEAST SETTLED THING IN THIS FILE AND ARE RECORDED IN TWO SNAPSHOTS BECAUSE OF IT. A named import is the only spelling that reads context.parsedFiles, and it costs up to three entries into the resolver per import (package probe, exports check, submodule probe) where the synthetic namespace spelling this arm used to pass costs one. Against the resolver as it stood when the call shape changed that read small_ms 1.76 -> 5.751 and collide_ms 1.929 -> 5.894, ~3.1x. Against the resolver a few commits later \u2014 which stopped re-running the whole tail after a null package probe, a double-probe this bench could not previously see because the namespace spelling never entered that branch \u2014 the same arms read 4.404 and 4.505. The ceilings are 18 and 19, chosen to clear BOTH: 4.09x and 4.22x of the current numbers, 3.13x and 3.22x of the higher ones, so neither state is red. Retighten toward 4x once that resolver settles. ITS DEPTH ARM WAS DILUTED AND THE BUDGET IS RETIGHTENED TO MATCH, which is the one thing here worth arguing about: the added work is depth-FLAT, so depth_ratio FALLS 1.872 -> 1.478 while the absolute cost more than doubles, and 2.6 against 1.478 would be 1.76x \u2014 far looser than the 1.39x #2913 chose deliberately to lock its own fix in. 2.1 restores that multiplier (1.42x). THE TWO MUTATION SCORES #2913 RECORDED (3.123 for reverting the per-directory memo, 2.734 for reverting the nested-name rejection) WERE TAKEN AGAINST THE OLD CALL SHAPE AND HAVE NOT BEEN RE-TAKEN. Modelled forward, with the depth-quadratic term reappearing in every resolver entry so its absolute contribution scales with the entry count, they land near 2.8 and 2.4 \u2014 both above 2.1, and the second BELOW 2.6, which is the arithmetic that decided the budget. Re-run the two mutations before trusting the lock-in claim above. python's heap reading is unchanged (10543152 recorded; 10529848-10544616 across eight runs) because its probe misses before the branch that reads parsedFiles \u2014 see _blind_spot for why no probe can reach that memo. Every figure in this section is the MAXIMUM over its snapshot's runs (five, then three), with peak-to-peak 1.031-1.058 on php and 1.019-1.081 on python, taken on a box that was NOT idle and with another change landing in python's resolver mid-measurement. Re-take them serially before merging.", + "_triage": "Every ratio and ms ceiling here is a TIMING signal \u2014 re-run on an idle machine before investigating; runner contention dominates. depth_ratio is the noisiest of them by a wide margin (it divides two sub-3 ms numbers, and Dart's are sub-1 ms): if exactly one arm fails and it is that one, suspect the machine first. N is 15 for every language whose cheapest arm is under 5 ms, rather than this bench's original 5, specifically to hold that arm's peak-to-peak swing under 1.26x \u2014 see _arms_note for the measured distributions and for why the six languages that drop to 7-8 are the ones where cell size makes it safe \u2014 so a depth_ratio failure that REPRODUCES is a real signal, not noise. Each language's chosen N is printed as `reps`; read it before blaming the estimator. The fingerprint, shape and heap arms are the opposite: deterministic (over 4 runs the heap arm's widest spread was 0.11% on python and 0.00% on java, javascript and c), a re-run never changes them, and they must never be wished away. TWO heap failures mean the arm STOPPED MEASURING rather than that memory grew, and both are deterministic: a heap floor failure says the probe no longer forces the index it used to (this is how four arms read 0 B when buildSuffixIndex went lazy, and 0 B passes every ceiling), and a `heap probe ... resolved` throw says a probe target that must MISS now hits, so the reading is a materialized answer and the legs past it were never reached. A heap BOUND failure is deterministic in the same way and means one specific thing: a language excluded from the budgeted tier has grown a structure, or started asking its index a question it did not ask when the exclusion was recorded \u2014 never a timing signal, never a re-run, and never fixed by raising the bound without saying what grew. The context arm is deterministic too, and a failure there means one specific thing rather than a range of them: run.ts's fifth argument is not reaching that resolver from this bench, or the leg behind it stopped running. Never a timing signal, never a re-run.", + "_floor": "Measured against the pre-change implementations on THIS corpus at 150/600 files: go 3.36, csharp 4.10, dart 3.32, ruby 3.87. The issues report 4.00 / 3.43 / 4.05 on their own corpora; those are DIFFERENT numbers from different repositories and are not reproduced here \u2014 what they and these share is that both independently land in the quadratic band, well clear of the ~1.0 a linear result gives. Note also that this floor was taken at 150/600 while the gate runs at 400/1600, so it is a lower bound on what the pre-change code would score today. Kotlin's own bench measured its pre-index floor at 3.737. The four resolvers added later were NOT re-floored on this corpus, and the reason is that they do not need to be: every one of their pre-change legs walked the whole file set per import (PHP one findIndex per path part per extension, Java one scan per stripped prefix, COBOL two full scans per COPY, C# csproj one normalizedFileList pass per import per matching config), so their scaling_ratio is ~4 by construction rather than by measurement. Their per-import costs were measured on their own issue corpora instead: PHP 96.40 ms -> 0.036 ms, Java 8.05 ms -> 0.62 ms, COBOL 3879 us -> 10.5 us, C# csproj 1103 us -> 7.6 us. The 1.8 budget sits well above the linear result and well below every one of those. The eight languages added last were NOT floored either, and for a different reason again: they are not fixes, so there is no pre-change implementation to floor against. Their scaling budgets are the global linear 1.8 and the point of the arms is to hold the current numbers (measured 1.01-1.13) rather than to separate a fix from a break. The one exception is javascript, which IS a fix and does have a floor: 6448.9 us per import at 2000 files and 25972.6 us at 8000 \u2014 4.12x the per-import cost for 4x the files, i.e. O(imports x files) \u2014 against 28.5 / 27.4 us with the index PR #2911 gave it, and 25.0 / 27.0 us for TypeScript over the identical corpus.", "scaling_budget": 1.8, "collide_scaling_budget": { "go": 5.5, "csharp": 3.4, + "csharp_csproj": 1.8, "dart": 3.3, "ruby": 1.8, - "kotlin": 1.8 + "kotlin": 1.8, + "php": 1.8, + "java": 3.4, + "cobol": 1.8, + "swift": 4.9, + "rust": 1.8, + "python": 1.8, + "javascript": 1.8, + "typescript": 1.8, + "vue": 1.8, + "c": 3.8, + "cpp": 4 }, "depth_budget": { "go": 1.6, - "csharp": 5, + "csharp": 2.2, + "csharp_csproj": 2.3, "dart": 1.6, "ruby": 2.2, - "kotlin": 3.4 + "kotlin": 3.4, + "php": 1.9, + "java": 2.2, + "cobol": 1.6, + "swift": 2.3, + "rust": 2.1, + "python": 2.2, + "javascript": 2.1, + "typescript": 2.1, + "vue": 2.3, + "c": 3, + "cpp": 3 }, "small_ms_ceiling": { "go": 7, "csharp": 11, + "csharp_csproj": 97, "dart": 3, "ruby": 77, - "kotlin": 12 + "kotlin": 12, + "php": 148, + "java": 17, + "cobol": 2, + "swift": 2, + "rust": 10, + "python": 18, + "javascript": 85, + "typescript": 85, + "vue": 81, + "c": 7, + "cpp": 7 }, "collide_ms_ceiling": { "go": 28, "csharp": 22, + "csharp_csproj": 105, "dart": 6, "ruby": 95, - "kotlin": 12 + "kotlin": 12, + "php": 154, + "java": 26, + "cobol": 1.5, + "swift": 4, + "rust": 19, + "python": 19, + "javascript": 89, + "typescript": 86, + "vue": 93, + "c": 11, + "cpp": 12 }, "heap_ceiling_bytes": { - "csharp": 116000000, - "ruby": 87000000 + "vue": 43326024, + "typescript": 40117944, + "kotlin": 72109644, + "go": 4497696, + "dart": 11751300, + "cpp": 15035016, + "csharp": 44900000, + "csharp_csproj": 110600000, + "ruby": 61600000, + "php": 74400000, + "java": 52500000, + "python": 9541404, + "javascript": 40200000, + "c": 15000000 }, + "_heap_reading_note": "The measured bytes_large each heap_ceiling_bytes entry above is 1.5x, recorded so the FLOOR can be derived from the reading instead of from the ceiling. It used to be 0.33 x the ceiling, described as 'half the measured size' — which held only while every ceiling stayed at exactly 1.5x its reading, a convention this file states and nothing enforces, so re-tuning one ceiling upward would have loosened that language's floor by the same factor in the one direction a floor exists to watch. 0.5 x the reading is the same effective floor to within 0.8% for all eight and says what it means. These are NOT asserted for equality: they reproduce to the byte across processes on one box, but a Node major or a different platform moves heapUsed accounting, and the ceiling/floor pair is what tolerates that (+50%/-50%). Re-baseline a ceiling and re-baseline the reading with it — they are two views of one measurement.", + "heap_reading_bytes": { + "vue": 28884016, + "typescript": 26745296, + "kotlin": 48073096, + "go": 2998464, + "dart": 7834200, + "cpp": 10023344, + "csharp": 29869080, + "csharp_csproj": 73703384, + "ruby": 41020808, + "php": 49574008, + "java": 34958600, + "python": 6360936, + "javascript": 26745296, + "c": 10018816 + }, + "_heap_bound_note": "THE SECOND HEAP TIER. Every registered language is measured now; heap_bound_bytes gates the nine that are not BUDGETED above, and it gates them with one comparison and no floor. A ceiling says 'this index is not too big'. A bound says something narrower and it is the thing that was missing: 'the exclusion still holds' — this language has not grown an index since it was left out. measure.mjs's MEMORY section states the re-entry condition (if a language ever diverges in what it ASKS its index, it earns a budgeted arm) and until now nothing watched for the divergence; HEAP_LANGS was a hand-maintained list of eight whose two neighbours, LANG_REGISTRY and CONTEXT_LANGS, are both reconciled against a derived predicate in both directions. HEAP_BOUNDED is derived too — it is LANGS minus HEAP_BUDGETED — so the two tiers partition the languages and a new one cannot land outside both. WHAT RE-MEASURING FOUND, five runs each, maximum quoted, peak-to-peak in brackets. go 2998464 B [1.0021], dart 7834200 B [1.0006] and kotlin 48073096 B [1.0004] HAD NO STATED REASON AT ALL: the old prose opened 'SIX of the seventeen are deliberately NOT in HEAP_LANGS' against a list of eight of seventeen, and these three were the three nobody counted. All three retain a real per-pass structure (go's PackageDirIndex, dart's basename buckets, kotlin's suffixByStem cascade) and kotlin's 45.85 MiB is the second-largest reading in this file, above ruby's 39.12 and java's 33.34, both of which carry a full budget. swift 3449216 B [1.0024] and cobol 2320456 B [1.0000] were excluded as 'below the measurement's own noise floor' on readings of 0.29 MB and 0 B at 32000 files; they now read 3.29 MB and 2.21 MB, growing with the corpus (969120 B and 536264 B at 8000). Those old numbers were not wrong when taken — the ARM changed under them, when #2903's follow-up made every probe resolve a real import and when measureHeap began flattening its corpus — which is the whole finding: a measurement written into prose is not re-taken, and this file had already gone stale against itself, quoting javascript at 46208832 B four paragraphs after quoting it at 25.51 MiB. rust is the one exclusion that survived unchanged: 16 B at 8000 files and 16 B at 32000, identical in all five runs. typescript 26745296 B, vue 28884016 B and cpp 10023344 B are duplicates of a builder AND of a read pattern: typescript is byte-identical to javascript's 26745296 in four runs of five, cpp is +0.05% of c's 10018816, vue is +8.0% of javascript. HOW THE BOUNDS WERE CHOSEN. Eight of the nine take 1.5x their measured maximum, rounded up to the next 100000 B: go 4500000 (1.501x), dart 11800000 (1.506x), kotlin 72200000 (1.502x), cobol 3500000 (1.508x), swift 5200000 (1.508x), typescript 40200000 (1.503x), vue 43400000 (1.503x), cpp 15100000 (1.507x). 1.5x is NOT copied from the ceilings out of habit — it is the same number for a stated reason, and the reason is not noise: measured peak-to-peak on this box is at most 1.0024, so noise alone would justify 1.05x. What a bound has to survive is a RUNNER change, since heapUsed accounting moves across platforms and Node majors, and this file already fixes that allowance at 50% for exactly this measurement on exactly this arm. Using a second allowance for the same uncertainty on the same number would be two conventions, not more rigour. At 1.5x the bound catches what the re-entry condition is about — a language growing an index, which costs +85% for one more suffix map and +100% for a duplicate — and it does NOT catch a duplicate diverging by 8%. That limit is real and is stated rather than hidden: the tight form is a same-process ratio against the arm each duplicate is a duplicate OF, which is the only form immune to the drift the absolute bound has to tolerate. RUST TAKES AN ABSOLUTE BOUND INSTEAD, 1048576 B (1 MiB), because 1.5 x 16 B is 24 B and would fail on the first byte of anything — a multiplier on a reading that is already nothing is a gate that flakes rather than a gate that bites. 1 MiB is ~65000x the reading and still 2.2x below the smallest real index measured here (cobol's 2.32 MB at the same file count), so it separates 'builds nothing' from 'builds something' with room on both sides. NO FLOOR ON ANY OF THE NINE, and the reason differs by language rather than being uniform. For rust a floor would be a floor on noise. For the other eight the readings are stable enough to floor today, and for kotlin and dart — larger than budgeted arms — a floor would be worth having, since a lazily-built map going quiet is exactly how the four budgeted arms once read 0 B. Adding one is a PROMOTION to the budgeted tier, with a ceiling and a recorded reading beside it, not a line here: a floor whose companion ceiling does not exist asserts 'still measuring' against a number nothing else bounds. Recommended next, in order: kotlin, then dart, then go.", + "heap_bound_bytes": { + "cobol": 3500000, + "swift": 5200000, + "rust": 1048576 + }, + "heap_floor_fraction": 0.5, "heap_ratio_budget": 1.25, "languages": { "go": { @@ -77,12 +161,18 @@ "fingerprint": "6d844763547b5cad54f41cfa0c0d618b098b214cb58654375fe7d74d229bee98" }, "fingerprint": "ec4bb401b3465713ad6dedc4f9aa774e586b319f21d406fd79eaad29f4e98861", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 13, + "probe": "github.com/org/repo0/pkg/util" + }, "_measured": { - "collide_ms": 6.676, - "collide_scaling_ratio": 3.958, - "depth_ratio": 1.032, - "scaling_ratio": 1.042, - "small_ms": 1.698 + "collide_ms": 5.964, + "collide_scaling_ratio": 3.763, + "depth_ratio": 1.169, + "scaling_ratio": 1.045, + "small_ms": 1.6 } }, "csharp": { @@ -122,16 +212,69 @@ "fingerprint": "6d3a964bdb4ae4a0c64a1e31023fb736c42643f5aeeef704d8e00c31ae12c3af" }, "fingerprint": "0b46146f5213fea8c07f1f90305a14da5ce31c865c495180f3f3903ddc6b8117", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 13, + "probe": "Ghost0.Deep.Missing" + }, "_measured": { - "collide_ms": 4.755, - "collide_scaling_ratio": 2.265, - "depth_ratio": 3.318, - "heap_bytes_large": 77200616, - "heap_bytes_small": 19548056, - "heap_mib_large": 73.62, - "heap_ratio": 0.987, - "scaling_ratio": 1.2, - "small_ms": 2.704 + "collide_ms": 4.074, + "collide_scaling_ratio": 2.163, + "depth_ratio": 1.438, + "scaling_ratio": 1.094, + "small_ms": 2.255 + } + }, + "csharp_csproj": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 979, + "distinct_outcomes": 2983, + "fingerprint": "b63d7f2b8078cce64d87a6c93e331db0a6045686cdc0dd70abe3ac2b0bab19d2" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4064, + "distinct_outcomes": 12029, + "fingerprint": "d9f161410c06c0e73e18ca0f27d6e253402dfe06c9918673a52f04daecb23e36" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 979, + "distinct_outcomes": 2983, + "fingerprint": "7ba2a8ff5151911aa556d809219e9ba5b64c7a022bbfa7f225f08e5d60ab2c62" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 979, + "distinct_outcomes": 2983, + "fingerprint": "fb815bbcfeb4f1049d63f38487ca9e3ada2fcc14ba8478bcc2976e2f632697e1" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4064, + "distinct_outcomes": 12029, + "fingerprint": "f06217a605d83cf66076a0d55a202dfa13fd4adb55f5605b6154135d0ff745dc" + }, + "fingerprint": "d9f161410c06c0e73e18ca0f27d6e253402dfe06c9918673a52f04daecb23e36", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 13, + "probe": "App.Missing0" + }, + "_measured": { + "collide_ms": 25.185, + "collide_scaling_ratio": 1.146, + "depth_ratio": 1.394, + "scaling_ratio": 1.182, + "small_ms": 23.634 } }, "dart": { @@ -171,12 +314,18 @@ "fingerprint": "b7e5303220b8fa64e85c7e17622961018316a10c5ef921a02584864309748b52" }, "fingerprint": "5151cd2498bd4b7698dc9309e2539977d306f9ba82a388c630c89b51fc4a3187", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "package:ext0/src/thing.dart" + }, "_measured": { - "collide_ms": 1.495, - "collide_scaling_ratio": 2.316, - "depth_ratio": 1.153, - "scaling_ratio": 1.102, - "small_ms": 0.53 + "collide_ms": 1.511, + "collide_scaling_ratio": 2.319, + "depth_ratio": 1.169, + "scaling_ratio": 1.071, + "small_ms": 0.542 } }, "ruby": { @@ -216,16 +365,18 @@ "fingerprint": "55a3afc06a48334a6dd2f29c730ae0cfd3a6d54f3013c0853a310af2bbcba277" }, "fingerprint": "31804ae9633d51ce7597d886f9c2230fec3448b0aa00d9ab953086e393cd28a9", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "gem0/missing/thing" + }, "_measured": { - "collide_ms": 21.702, - "collide_scaling_ratio": 1.111, - "depth_ratio": 1.517, - "heap_bytes_large": 57939392, - "heap_bytes_small": 14835688, - "heap_mib_large": 55.26, - "heap_ratio": 0.976, - "scaling_ratio": 1.143, - "small_ms": 20.799 + "collide_ms": 20.732, + "collide_scaling_ratio": 1.119, + "depth_ratio": 1.257, + "scaling_ratio": 1.133, + "small_ms": 19.994 } }, "kotlin": { @@ -265,14 +416,591 @@ "fingerprint": "d867aaa39e47ba55df1853c4eaca741977e946a16ad3d99f35c698abe7241ac7" }, "fingerprint": "003bb2fe82972c6bb6b4b4e569fb49dfcda2d7c61922d68c65b04398cbbde50b", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 18, + "probe": "com.ghost0.deep.Missing" + }, "_measured": { - "collide_ms": 2.703, - "collide_scaling_ratio": 1.199, - "depth_ratio": 2.287, - "scaling_ratio": 1.224, - "small_ms": 2.821 + "collide_ms": 2.611, + "collide_scaling_ratio": 1.179, + "depth_ratio": 2.219, + "scaling_ratio": 1.169, + "small_ms": 2.799 + } + }, + "php": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2867, + "fingerprint": "3bb31eb4cd444b240e56b151007004f2f810bb5ee3f111b7b57738ea17c819b2" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11517, + "fingerprint": "1c313a83acf55ec58994fc55016754488ae2d352aefaeb84a2e3ecbb928b3479" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2867, + "fingerprint": "94bdf5cb27b7a1bb0d24e2ba0157ba71dcf61ec726059dd5a0462377a1d0180b" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2695, + "fingerprint": "61038746f1386bfc747784e7ce6bc52522bc4585259668e22e29f93291b0b3a5" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 10845, + "fingerprint": "c41d254ce8703339576e5642f67dfef81c97445c75db184bb65dc26b4d4715ef" + }, + "fingerprint": "1c313a83acf55ec58994fc55016754488ae2d352aefaeb84a2e3ecbb928b3479", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 14, + "probe": "Vendor0\\Ghost\\Missing" + }, + "context": { + "target": "App\\Ns0\\Dup", + "with_context": "src/App/Ns0/Helpers.php", + "without_context": "src/App/Ns0/Dup.php" + }, + "_measured": { + "collide_ms": 35.91, + "collide_scaling_ratio": 1.068, + "depth_ratio": 1.268, + "scaling_ratio": 1.079, + "small_ms": 34.023 + } + }, + "java": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1100, + "distinct_outcomes": 2868, + "fingerprint": "a5e3b2e63b6c06dc1ae3655193c96801f038f9a69d448e399e1865d9f2601844" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4456, + "distinct_outcomes": 11512, + "fingerprint": "7ffdd453170ef36aa66c3de73f45d6ffa0588850b16111a4078f10f41782ee18" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1100, + "distinct_outcomes": 2868, + "fingerprint": "9c1589a04dfe8c70fa5aff57ebb5742eb8999b8979a8baae701da3ef993114ec" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1100, + "distinct_outcomes": 2868, + "fingerprint": "8a08bb2d5919e9739388c0c514ae0162f6a18c10577d1b7c5bad54e2320efea5" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4456, + "distinct_outcomes": 11512, + "fingerprint": "3286317a7e5c2ae70b1c001690980f61633566a30672e949caebcbb065e8c80f" + }, + "fingerprint": "7ffdd453170ef36aa66c3de73f45d6ffa0588850b16111a4078f10f41782ee18", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 18, + "probe": "com.google.common.vendor0.Missing" + }, + "_measured": { + "collide_ms": 5.434, + "collide_scaling_ratio": 2.365, + "depth_ratio": 1.402, + "scaling_ratio": 1.16, + "small_ms": 3.18 + } + }, + "cobol": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2941, + "fingerprint": "e5bf9c2a74cad64df6ac18299b56fc9139943baec6118036b2e765ac3d4252f2" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11791, + "fingerprint": "f192ca7a9e87eb05f03893ffc64252a8aba2c638604dcf449150fb9b5fdd989e" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2941, + "fingerprint": "c690c6abc5c7aab31f27a97e5ef25d32daa483d9c08ceb48bc0b85ac406e6e37" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2827, + "fingerprint": "c487db2efbf7a683674de84430d88e7a4c75e9427a53cccae934fd8440b85d87" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11393, + "fingerprint": "8bc1d506b54e800d060eb4c92fca01c5b06c5a7130248dc1a79521bdea53982a" + }, + "fingerprint": "f192ca7a9e87eb05f03893ffc64252a8aba2c638604dcf449150fb9b5fdd989e", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "VENDOR0" + }, + "_measured": { + "collide_ms": 0.197, + "collide_scaling_ratio": 1.046, + "depth_ratio": 0.885, + "scaling_ratio": 0.936, + "small_ms": 0.286 + } + }, + "swift": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2913, + "fingerprint": "91c5172b994270807f7fdcf80ac545edd50d3dc87c67290c9aabaed8bb65d594" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11709, + "fingerprint": "16f80a95e52ad1057cf369b7816ce704684fc5b5663a39f23c9149e9221c6170" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2913, + "fingerprint": "22ef94a6e087d7ac909733ef32da2ddf292fa8c82b05b70b5910c307fceca1b4" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2606, + "fingerprint": "4d2c41ba5f8230ab6b9faade80f293ddde153f4ff1d5dd4473f1cd699d2808fd" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 10184, + "fingerprint": "d27b6070f5ad93020bf762221e798c382327406f3f2cca2f7aab3e9ac56faef4" + }, + "fingerprint": "16f80a95e52ad1057cf369b7816ce704684fc5b5663a39f23c9149e9221c6170", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 13, + "probe": "ExternalPkg0" + }, + "_measured": { + "collide_ms": 0.819, + "collide_scaling_ratio": 3.454, + "depth_ratio": 1.496, + "scaling_ratio": 1.063, + "small_ms": 0.385 + } + }, + "rust": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 979, + "distinct_outcomes": 2844, + "fingerprint": "6a2435149e055e6903aab2dd3fa2a0986d8d1d7933bccb0b7f311448372f548c" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4064, + "distinct_outcomes": 11440, + "fingerprint": "442f9124ebeb052557413d9dbb7c5e467ffc69da6357d0d8d9bcd2232ba27092" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 979, + "distinct_outcomes": 2844, + "fingerprint": "aa32ea032a548df09554c40a8b0679f11bc4d4cefc1dcad941283036dbae7c8e" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 979, + "distinct_outcomes": 2844, + "fingerprint": "4d1e28e318a04ee0e2065b5f9a2c971765e2f60310b0e612cfd327c76b6344b6" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4064, + "distinct_outcomes": 11440, + "fingerprint": "3de5234747493741abbf170e164ef80188092747150cfb39ef2cacd5658effc0" + }, + "fingerprint": "442f9124ebeb052557413d9dbb7c5e467ffc69da6357d0d8d9bcd2232ba27092", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 12, + "probe": "ghost0::Missing" + }, + "_measured": { + "collide_ms": 4.767, + "collide_scaling_ratio": 1.042, + "depth_ratio": 1.371, + "scaling_ratio": 1.097, + "small_ms": 2.523 + } + }, + "python": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 845, + "distinct_outcomes": 2867, + "fingerprint": "7a458789903c904968af8f9f851656ea33c1c446959ca79a0222ec65d3e809ed" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 3556, + "distinct_outcomes": 11517, + "fingerprint": "98f99b9eaa3fcc3c58c4be0116853789c8e1299c187088a8b04d28f1885c944a" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 845, + "distinct_outcomes": 2867, + "fingerprint": "c099814a70bbb63471fecc6e9527632e83b9954618963e77648f893ddfe65286" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 845, + "distinct_outcomes": 2867, + "fingerprint": "038c097cb628f6c65c1a228a5df3bb29a81eb3d4d7f297cfe86c3c4c6323c7c0" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 3556, + "distinct_outcomes": 11517, + "fingerprint": "94cd4994ce690db215028ff42f06aa1fd142bd26d62a290f4849bbff36c294f5" + }, + "fingerprint": "98f99b9eaa3fcc3c58c4be0116853789c8e1299c187088a8b04d28f1885c944a", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "vendor0.deep.missing" + }, + "context": { + "target": "pkg", + "with_context": "pkg/__init__.py", + "without_context": "pkg/X.py" + }, + "_measured": { + "collide_ms": 4.521, + "collide_scaling_ratio": 1.085, + "depth_ratio": 1.563, + "scaling_ratio": 1.144, + "small_ms": 4.431 + } + }, + "javascript": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2867, + "fingerprint": "4a80c7b940a6c39d0ebd109980469d2398b417a4479f5d03f35abc482fa76122" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11517, + "fingerprint": "827ac421e8958ff686b2877efa60fbaf1a1c661698cde8f0c8c931919e0f35bd" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2867, + "fingerprint": "4d0551e044b19bccd9775879b1425f3adecf0cdf1e90a5d8a9607ef0a51af880" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2871, + "fingerprint": "45b4bb3b2f9797e21029ef1eef7247702813cac39ac430f9a999d3326596a7a1" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11548, + "fingerprint": "304ed93e83b397b4aa9520750739ffdcf3a9b353ae0ad8fe88c0368c63c85533" + }, + "fingerprint": "827ac421e8958ff686b2877efa60fbaf1a1c661698cde8f0c8c931919e0f35bd", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "vendor0/lib/missing" + }, + "_measured": { + "collide_ms": 22.96, + "collide_scaling_ratio": 1.077, + "depth_ratio": 1.213, + "scaling_ratio": 1.093, + "small_ms": 22.762 + } + }, + "typescript": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2867, + "fingerprint": "fe8fcf81efa0fa3a77894edc6bd0b9ec4ff0bf92f24e116ddf108dc70cdcd97e" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11517, + "fingerprint": "24e36ebfc1c482643812f1ef400e8cb387dae11954531407e113d4e6c3fa2a6d" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2867, + "fingerprint": "e9faf31f6b1299394760e27ff9e04af1a8b4ddca0370db62fd2a59af7a4f5d05" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2871, + "fingerprint": "dbb68b66e8d136140f4a7bc024c97f5de02d1c8b6f3a07efa271dcb73366eeb5" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11548, + "fingerprint": "3129a6f1f25bd5568058682f99812ad35e38231b92025184344b74b86fa2e910" + }, + "fingerprint": "24e36ebfc1c482643812f1ef400e8cb387dae11954531407e113d4e6c3fa2a6d", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "vendor0/lib/missing" + }, + "_measured": { + "collide_ms": 22.324, + "collide_scaling_ratio": 1.059, + "depth_ratio": 1.25, + "scaling_ratio": 1.079, + "small_ms": 20.882 + } + }, + "vue": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2867, + "fingerprint": "88e85b85f7158cc87d770f119c992d71801c4c692da13064cbc8b95718517fe4" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11517, + "fingerprint": "4d62ac179e4371d3f41b691cea725b90272f72da625e914d2f16d323a1e940c8" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2867, + "fingerprint": "786c801ad824c3e49f05aaf37a63c3bfe7dbe6dd2fb44cfef180099f4fcdd401" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2871, + "fingerprint": "8a01ee06ddf2eeb0db72dd8b73544180bf48d8cb82b6e73b1969233842553636" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11548, + "fingerprint": "841b48a4c46fc56cd700ff7c07a515da1139cc4528d479a47876cbed291d91a4" + }, + "fingerprint": "4d62ac179e4371d3f41b691cea725b90272f72da625e914d2f16d323a1e940c8", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "vendor0/lib/Missing.vue" + }, + "_measured": { + "collide_ms": 24.453, + "collide_scaling_ratio": 1.095, + "depth_ratio": 1.384, + "scaling_ratio": 1.071, + "small_ms": 21.765 + } + }, + "c": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2863, + "fingerprint": "4dd05ba9a0c6731d449ec555f56d2f7cb05cdcdaa01a8acc242e3709d305184f" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11512, + "fingerprint": "70d6064fdc08e86036ced58393585afc3693ee527f00983847299e390b413d87" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2863, + "fingerprint": "43707e57b1079e7f01cc84ea5ab891cf77c395e2d52e7fbb7eee30c058c1d667" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2695, + "fingerprint": "982b925fdbbc59d05ae52be1f405f3cbb6fd554390ee38eeff869df9316ffaf2" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 10845, + "fingerprint": "70b30b89ae671208bd836693fbc87d6059656cf347b9397d3b905d24e31912cf" + }, + "fingerprint": "70d6064fdc08e86036ced58393585afc3693ee527f00983847299e390b413d87", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "vendor0/missing.h" + }, + "_measured": { + "collide_ms": 2.924, + "collide_scaling_ratio": 2.575, + "depth_ratio": 1.938, + "scaling_ratio": 1.033, + "small_ms": 1.649 + } + }, + "cpp": { + "small": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2863, + "fingerprint": "6c199e829226c1cdd86e74611b159ff2e83d4c17da2a72552503a7c4518182e4" + }, + "large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 11512, + "fingerprint": "191bddd6f77a10ab6bced04e5c5f55e0af4481563ef3cb57e08c5dfa6c86454e" + }, + "deep": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2863, + "fingerprint": "5603c080739321bb6204c153ec6214dc185e436b5aadb5ec13738e2a759a84f3" + }, + "collide": { + "files": 400, + "imports": 3200, + "resolved": 1153, + "distinct_outcomes": 2695, + "fingerprint": "cbf2fece6338725205beaf87058ce32ae1ba0860d14cedd29b7904b2f3a63726" + }, + "collide_large": { + "files": 1600, + "imports": 12800, + "resolved": 4681, + "distinct_outcomes": 10845, + "fingerprint": "094f7fe2aace191d7e53c2d4ecd7e063cf15bd66643e6201fd46e45b6b63ab6e" + }, + "fingerprint": "191bddd6f77a10ab6bced04e5c5f55e0af4481563ef3cb57e08c5dfa6c86454e", + "heap": { + "files_small": 8000, + "files_large": 32000, + "path_segments": 11, + "probe": "vendor0/missing.hpp" + }, + "_measured": { + "collide_ms": 3.035, + "collide_scaling_ratio": 2.566, + "depth_ratio": 2.064, + "scaling_ratio": 1.167, + "small_ms": 1.626 } } }, - "_blind_spot": "MEASURED, so nobody has to rediscover it: a full workspace scan reintroduced on 1-in-32 imports passes EVERY arm here — dart scored 1.458 scaling and 1.736 ms against the 1.8 budget and 4 ms ceiling of an earlier revision. At 1-in-8 the scaling arm catches it (2.414). The gate that NARROWS this is not a timing gate at all: test/unit/scope-resolution/import-target-index-parity.test.ts counts iterations of the file-set Set and reads 14 instead of 1 for that same 1-in-32 mutation, deterministically and for all five languages. It does NOT close it. The counter watches the Set, and the resolvers no longer read the Set — they read materialized copies of the same file list: WorkspaceFileIndex.normalized and .all (C#, Ruby), Dart's byBasename buckets, and PackageDirIndex.filesByDir (Go, C#). A 1-in-32 scan over any of those three touches the Set zero extra times, so it passes the parity test AND passes --check. Closing it would take an iteration counter on the materialized arrays themselves. Read the two gates together; tightening these ceilings toward the noise floor to chase that case would only buy flaky CI." + "_blind_spot": "MEASURED, so nobody has to rediscover it: a full workspace scan reintroduced on 1-in-32 imports passes EVERY arm here \u2014 dart scored 1.458 scaling and 1.736 ms against the 1.8 budget and 4 ms ceiling of an earlier revision. At 1-in-8 the scaling arm catches it (2.414). The gate that NARROWS this is not a timing gate at all: test/unit/scope-resolution/import-target-index-parity.test.ts counts iterations of the file-set Set and reads 14 instead of 1 for that same 1-in-32 mutation, deterministically and for all five languages. It does NOT close it. The counter watches the Set, and the resolvers no longer read the Set \u2014 they read materialized copies of the same file list: WorkspaceFileIndex.normalized and .all (C#, Ruby), Dart's byBasename buckets, and PackageDirIndex.filesByDir (Go, C#). A 1-in-32 scan over any of those three touches the Set zero extra times, so it passes the parity test AND passes --check. Closing it would take an iteration counter on the materialized arrays themselves. Read the two gates together; tightening these ceilings toward the noise floor to chase that case would only buy flaky CI. CONFIRMED THE HARD WAY by PR #2911: JavaScript resolution was scanning ImportPassCache.normalizedFileList on every import \u2014 a materialized array, not the Set \u2014 at 25972 us per import at 8000 files, and no instrument on the #2901-#2909 branch could see it. It took a differential parity test over 211200 old-vs-new pairs to find. The arms added here would have caught THAT one on absolute ms (85 ms budget against a 20 ms arm; the unindexed resolver costs ~83000 ms on the same corpus), which is the argument for gating every registered language rather than only the ones a PR happens to touch. THE SECOND BLIND SPOT IS CLOSED, and this records what closing it changed. This harness used to call the inner resolvers with the NO-CONTEXT shape: run.ts calls provider.resolveImportTarget with five arguments, the fifth being { parsedFiles, parsedImport }, and resolveOne supplied three. resolveOne now makes the production call, newPass mints the ParsedFile[] FIRST and derives the path set from it exactly as run.ts does, and both legs behind the argument run on every import of their arms \u2014 PHP's named/alias function-or-const leg over filesByDirectory(context.parsedFiles), whose memo defeated measures 197.0 us -> 9976.2 us per import (50.6x), and Python's from-import submodule-precedence branch, the only spelling that reads context.parsedFiles at all. Fifteen of the seventeen arms cannot observe a context (their hooks declare three or four parameters) and are handed none, so their numbers did not move; which two CAN is now reconciled against SCOPE_RESOLVERS' hook arity rather than asserted in prose. NOTHING ELSE IN THIS FILE COULD HAVE GATED IT, which is why the context arm exists: on this corpus the leg AGREES with the cascade for every import, so all ten of PHP's and Python's fingerprints, their resolved counts and their distinct_outcomes are unchanged; a dropped context makes the timing arms FASTER and no arm here has a lower bound on ms; and the heap floor (0.5 x 49573840 = 24.8 MB) still passes the 37576816 B a no-context PHP pass reads. The arm is one import per language resolved through resolveOne twice, with and without the pass's parsedFiles, whose two answers must DIFFER and must both match what is recorded. WHAT REMAINS UNMEASURED, narrowed rather than deleted: Python's parsedFileByPath memo is exercised by the five timing arms and cannot be reached by the heap arm at all, because retainedPassBytes requires a probe that MISSES while every path that builds that memo returns a non-null packageTarget \u2014 so no ceiling bounds that Map (one pointer per parsed file, O(files), no depth term) and the contract test's count gate is what holds it to one build per pass. PHP's leg is measured with NO composer.json, so namespaceDirectories only ever returns the directory of an already-resolved file and the PSR-4 mapping branch stays unreached, exactly as csharp cannot reach the csproj leg; closing that is a second PHP arm on the csharp_csproj precedent, not a parameter. And the const tail of PHP's leg is a different ANSWER at the same cost \u2014 it runs the identical candidate gather and localDefs filter and diverges in the last two lines \u2014 so it is gated by count in test/unit/scope-resolution/import-target-index-reuse.contract.test.ts, which stays the gate to read alongside this file." } diff --git a/gitnexus/bench/import-target/measure.mjs b/gitnexus/bench/import-target/measure.mjs index 4ec4a4615..6e914f840 100644 --- a/gitnexus/bench/import-target/measure.mjs +++ b/gitnexus/bench/import-target/measure.mjs @@ -1,15 +1,36 @@ /** - * Build-free scaling + identity bench for the Go, C#, Dart, Ruby (#2877, #2878, - * #2879, #2880) and Kotlin (#2872) import-target resolvers, over ONE shared - * corpus so the five are directly comparable. + * Build-free scaling + identity bench for EVERY import-target resolver in + * `SCOPE_RESOLVERS` — the registry decides which, not a list kept here, and the + * `--check` inventory arm at the foot of this file fails when the two disagree + * — over ONE shared corpus so the arms are directly comparable. One arm per + * registered language, plus a second `csharp` arm carrying csproj configs + * (#2902), so there is one more arm than there are languages. + * + * NO LANGUAGE IS OMITTED, and that is the point of the list rather than an + * accident of it. Nine of these arms (go, csharp, csharp_csproj, dart, ruby, + * kotlin, php, java, cobol) were added as their own O(imports × files) scans + * were indexed away — #2877/#2878/#2879/#2880, #2872, #2901, #2902, #2908 — and + * the bench is the forward guard on each. The eight added alongside them + * (swift, rust, python, javascript, typescript, vue, c, cpp) resolve imports + * through the same registered hook with the same per-run memoized indexes, and + * were ungated: nothing pinned their output and nothing pinned their scaling. + * One of them was not hypothetical — JavaScript reached `suffixResolve` with no + * index at all and measured 25 972 µs per import at 8000 files (PR #2911) — + * which is exactly the class of defect the other seven were one commit away + * from. + * + * A C or C++ `#include` is an import site for this purpose and is gated like + * every other registered language. See `newPass` for the one structural thing + * those two need that no other language does. * * Kotlin also has `bench/kotlin-import-target/`, and this does not replace it: * that bench fingerprints both file-set iteration orders and probes the * four-tier cascade shape by shape, which this corpus does not. What Kotlin * gains here is a second corpus and the arms below that its own bench predates. * - * Before this PR each of the other four answered its lookups with a full - * `allFilePaths` scan per import, so import resolution cost O(imports × files): + * Each of the first nine resolvers answered its lookups with a full + * `allFilePaths` scan per import before its fix, so import resolution cost + * O(imports × files): * * - Go: `findRootPackageFiles` / `findAllFilesInPkgDir`, the latter once per * path segment on the GOPATH fallback — several full scans per import; @@ -17,7 +38,62 @@ * leg was already using — up to eight passes for a four-segment `using`; * - Dart: one full scan per candidate path, and for an external package both * candidates miss, so both always ran to completion; - * - Ruby: a complete `buildSuffixIndex` rebuilt and discarded per `require`. + * - Ruby: a complete `buildSuffixIndex` rebuilt and discarded per `require`; + * - PHP: two materialized arrays per import and then no index at all, which + * dropped `suffixResolve` onto a linear `findIndex` — one full pass per path + * part per extension, and there are ~50 extensions (96.40 ms per import at + * 20k files, now 0.036 ms); + * - Java: one scan for the direct match plus one more per stripped package + * prefix, and a JDK or third-party import runs the loop to the end (8.05 ms + * per import, now 0.62 ms); + * - COBOL: two scans per `COPY` — one per extension tier — each calling + * `extname` + `basename` + `toUpperCase` on every path, both always running + * to completion because vendor copybooks live outside the repo (3879 µs per + * import, now 10.5 µs); + * - C# csproj: the namespace-directory fallback re-scanned + * `normalizedFileList` per import per matching config (1103 µs, now 7.6 µs). + * `csharp` here builds its context with NO `csharpConfigs`, so it can never + * reach that leg — `csharp_csproj` is the same corpus with the configs + * supplied, and it exists because without it #2902 ships unmeasured. + * + * The eight added afterwards are not a second class of arm — they carry the + * same five timing arms, the same per-scale fingerprint and shape gates and the + * same budgets. What differs is what each one's cost is a function of, because + * that decides which arm can actually fail for it: + * + * - swift: `getSwiftModuleIndex` buckets a file under EVERY interior + * directory segment, so `Sources/Models/User.swift` answers to `Sources` + * and to `Models`. A miss is a Map miss and flat; a HIT returns the whole + * module bucket minus the importer, so its cost is the BUCKET size. Nothing + * in the unique layout produces a large bucket, which is why its collide + * arm is four modules instead of `dirs` of them (`SWIFT_COLLIDE_MODULES`); + * measured 3.28 there against 0.90 on file count. + * - rust: probes candidate paths with `allFilePaths.has(...)` and never + * searches, so its cost is O(path SEGMENTS) and is provably flat in the + * file count — measured 1.10 scaling, 1.06 collide scaling. That flatness + * IS the assertion, and it is why its collide arm is a deep module tree + * with ~2x the `::` segments rather than a shared-leaf layout: a collide + * arm built on file count would have been an arm that cannot fail. Note + * that `buildRustModuleIndex` lives on a DIFFERENT hook + * (`qualified-call.ts::moduleIndexFor`) and is not on this path at all. + * - python: `getPythonFileIndex` is keyed and flat on both file count and + * bucket cardinality (1.11 / 1.10), but `hasRepoCandidate` and + * `resolveAbsoluteFromFiles` each rebuild one ancestor prefix per directory + * component of the IMPORTER, so per-import cost is quadratic in path depth: + * measured depth_ratio 7.39, by far the largest here, and the reason its + * depth budget is 11 rather than the ~2 most languages carry. + * - javascript, typescript, vue: one resolver (`resolveTsTarget`) behind + * three adapters, so the three corpora are the same shape and differ only + * in what actually differs — the extension list (`.js` vs `.ts`) and, for + * Vue, the tsconfig alias branch (see `VUE_TSCONFIG`). All three are + * miss-dominated bare specifiers, because a relative import resolves by + * exact `Set.has` and never reaches the leg that had no index. + * - c, cpp: `resolveCppImportTarget` delegates to `resolveCImportTarget`, so + * the two share a resolver and differ in extension set and in which adapter + * builds the augmented set. Cost is a basename bucket walk with a + * depth-then-lexicographic tie-break, so the collide arm (a `mod{n}` header + * in every service's `include/`) is where it grows: 2.54 / 2.64 against + * 1.06 on file count. * * Two properties of the corpus are load-bearing and must not be "simplified": * @@ -38,15 +114,21 @@ * - `depth_ratio` `t_deep/t_small` at a FIXED file count with ~6x the path * components. `scaling_ratio` divides the file count out, so it is * scale-invariant and structurally cannot see a cost that grows with path - * DEPTH instead — and `buildSuffixIndex` (C#, Ruby) and Kotlin's - * `suffixByStem` each emit one entry per component. Go and Dart, whose - * indexes are depth-free, sit at ~0.9; the others sit legitimately above - * 1.0, which is why the budget is per language; + * DEPTH instead — and `buildSuffixIndex` (C#, Ruby, PHP, Java) and Kotlin's + * `suffixByStem` each emit one entry per component. Go, Dart and COBOL, + * whose indexes are depth-free (COBOL's are keyed on the basename and + * nothing else), sit at ~0.9-1.0; the others sit legitimately above 1.0, + * which is why the budget is per language. Python is the extreme and the + * reason the spread is worth a per-language number at all: its index is + * depth-free, but `hasRepoCandidate` and `resolveAbsoluteFromFiles` rebuild + * an ancestor prefix per importer directory component ON EVERY IMPORT, so + * the RESOLVER, not the index, is quadratic in depth — 7.39; * - `collide_scaling_ratio`, the same measurement on a corpus whose * directories SHARE their last segment and whose files share basenames — * see the `collide` section below; - * - `heap` (C# and Ruby): retained bytes of the shared `WorkspaceFileIndex` - * — see the `heap` section below; + * - `heap` (all 17): retained bytes of the per-pass import index, read by + * resolving one real import — see the `heap` section below. Eight carry a + * ceiling, a floor and a ratio; the other nine carry an upper bound only; * - a sha256 over every distinct `fromFile | target → result`, as the * correctness gate. The tie-break-level proof that this PR's index * reproduces the scans lives in @@ -73,7 +155,17 @@ * comparing the arms to `small` is the only thing that notices; * - `small_ms_ceiling` and `collide_ms_ceiling`, ABSOLUTE bounds, because a * constant-factor regression that grows both scale arms equally passes - * every ratio. + * every ratio; + * - a heap FLOOR beside every heap ceiling, and a presence check in front of + * every timing budget. Both exist because the same failure has now happened + * twice in this file's short life: an arm that stops measuring passes. A + * lazy `buildSuffixIndex` made four heap arms read 0 B, and 0 B is under + * every ceiling; a deleted budget key makes `got > undefined` false, which + * is a deleted gate wearing a passing arm's clothes; + * - an INVENTORY arm against `SCOPE_RESOLVERS` itself. `LANG_REGISTRY` claims + * to cover every registered resolver; this is what makes the claim true + * rather than commented, and it is the arm that would have caught PR #2911's + * language shipping unmeasured. * * SCOPE OF THE "independent of corpus size" CLAIM — the `collide` arm. * `small`/`large`/`deep` mint one directory name per index (`src/pkg7`, @@ -86,14 +178,34 @@ * workload — identical file, import and resolved counts — laid out the way * these languages are actually written: `svcN/internal/`, `SrcN/Models/`, a * `mod0.dart`/`mod0.rb` in every package. Measured on that shape the per-import - * cost is NOT corpus-size-independent for the three resolvers that scan a + * cost is NOT corpus-size-independent for the four resolvers that scan a * bucket: * - * - go and csharp walk `PackageDirIndex.dirsByLastSegment[seg]`, which now - * holds every directory; + * - go, csharp and java walk `PackageDirIndex.dirsByLastSegment[seg]`, which + * now holds every directory; * - dart walks its basename bucket, which now holds every same-named file; - * - ruby and kotlin answer from keyed maps and are collision-IMMUNE, so their - * collide budgets are the linear ones — that immunity is the assertion. + * - ruby, kotlin, php and cobol answer from keyed maps and are collision- + * IMMUNE, so their collide budgets are the linear ones — that immunity is + * the assertion, and for cobol the arm is also the only one that reaches + * the copybook-over-source tier tie-break, which needs one bookname to name + * two files; + * - csharp_csproj runs the OTHER way: its shared leaf collapses + * `dirsByLastSegment` to a single key, which makes the slash-free sweep + * (see `CSPROJ_CONFIGS`) cheaper on the collide layout than on the unique + * one, so its expensive scale arm is `large`, not `collide_large`; + * - of the eight added later, swift (3.28) and c/cpp (2.54/2.64) are the two + * that scan a bucket, and they scan DIFFERENT buckets: swift's is the + * module's own file list, which it returns, and C's is the basename bucket + * its suffix fallback walks. python, javascript, typescript and vue answer + * from keyed maps and sit at 1.03-1.10, so they keep the linear budget and + * that immunity is their assertion, exactly as for ruby and kotlin; + * - rust's collide arm is the one that is NOT a shared-leaf layout, and the + * reason is in the list above: file count is not an axis its cost has, so a + * shared-leaf rust arm would have been an arm that cannot fail. Its collide + * corpus is a deep module tree whose targets carry ~2x the `::` segments, + * which is the axis that CAN grow; the ratio across file counts staying at + * 1.06 on it is the assertion, and `collide_ms_ceiling` bounds the absolute + * cost of the long-path probe. * * This is a scope-of-claim limit, not a regression: on the MISS path with a * shared leaf name the bucket grows with the file count BY CONSTRUCTION, and @@ -103,16 +215,101 @@ * structure, which trades against the O(files × depth) memory * `package-dir-index.ts` cites #2649 to avoid — a design change, not a tune. * - * MEMORY — the `heap` arm. C# and Ruby both resolve through the shared - * `WorkspaceFileIndex`, and `buildSuffixIndex` under it emits three maps at + * MEMORY — the `heap` arm. C#, Ruby, PHP and Java all resolve through the + * shared `WorkspaceFileIndex`, and `buildSuffixIndex` under it emits maps at * O(files × depth): exactly the profile `package-dir-index.ts` cites #2649 to - * avoid for itself. C# is the reason this is gated rather than noted: at BASE - * `getWorkspaceFileIndex` was reached only from the csproj branch and the - * no-csproj leg scanned the Set and retained nothing, whereas it is now called - * unconditionally. Every other arm here is time or count, and no ratio can see - * a footprint. Measured in ABSOLUTE bytes, not only as a ratio: the finding is + * avoid for itself. That is why this is gated rather than noted — all four + * retained NOTHING across imports at BASE. C#'s `getWorkspaceFileIndex` was + * reached only from the csproj branch while the no-csproj leg scanned the Set; + * PHP and Java scanned on every leg; Ruby rebuilt and discarded a suffix index + * per `require`. Every other arm here is time or count, and no ratio can see a + * footprint. Measured in ABSOLUTE bytes, not only as a ratio: the finding is * about the footprint itself, and a ratio alone hides a large constant. * + * Four more are gated for the same reason as those four. JavaScript is the + * clearest case in the file: before PR #2911 it retained NOTHING because it + * built no index at all, and it now retains 25.51 MiB at 32 000 files through + * the + * same `buildSuffixIndex`. `csharp_csproj` is the newest and the one that + * proves the arm's design: same corpus and same `getWorkspaceFileIndex` as + * `csharp`, but its csproj leg asks all three questions instead of one, and it + * retains 70.29 MiB against C#'s 28.48. Python's `getPythonFileIndex` + * (9.88 MiB) and C's basename map (9.55 MiB) are an order of magnitude smaller + * but are the only structure either language keeps, and both are one careless + * edit — a stored `split('/')` array instead of a depth NUMBER — away from the + * O(files × depth) shape this arm exists to catch. + * + * WHAT THE ARM MEASURES IS NOW THE READ PATTERN, and that is the correction + * this file most needed. `buildSuffixIndex`'s two suffix maps became lazy + * (#2903 extended past `dirMap`), and the four original arms — which called + * `getWorkspaceFileIndex(set)` directly and read `index.all.length` — stopped + * asking any suffix question, built no map, and reported 0 B at 32 000 files. + * 0 B is under every ceiling, so `--check` PASSED with four gates that had + * become ceilings over nothing. Every arm now resolves one real MISSING import + * through the real resolver, so the maps it forces are the maps production + * forces; `HEAP_PROBE_TARGET` and `retainedPassBytes` carry the details, and + * `heap_floor_fraction` is the gate that would have caught the 0 B. + * + * EVERY LANGUAGE IS MEASURED, and the eight-entry list this arm ran on is now + * the BUDGET tier rather than the measurement tier. That list — `HEAP_LANGS`, + * now `HEAP_BUDGETED` — was reconciled bidirectionally against its two budget + * maps and every entry had to produce a reading, but nothing tied it to the + * property it stood for, "the languages that retain a per-pass index". Its two + * neighbours in this file do not have that gap: `LANG_REGISTRY` is reconciled + * against `SCOPE_RESOLVERS.keys()` and `CONTEXT_LANGS` against hook arity, both + * directions, both derived. Nine languages were excluded on readings taken once + * and written into this prose, and the paragraph below states the re-entry + * condition ("if any of the four ever diverges in what it ASKS, it earns an arm + * the same way") with nothing watching for the divergence. + * + * Re-measured — all seventeen, five runs each, one probe per language through + * the same `retainedPassBytes` — the prose was wrong in three separate ways: + * + * 1. THREE OF THE NINE HAD NO STATED REASON AT ALL. The old paragraph opened + * "SIX of the seventeen are deliberately NOT in HEAP_LANGS" against a list + * of eight, so go, dart and kotlin were excluded silently. All three + * retain a real per-pass structure: go's `PackageDirIndex` reads + * 2 998 464 B, dart's basename buckets 7 834 200 B, and kotlin's + * `suffixByStem` cascade 48 073 096 B (45.85 MiB) — the second-largest + * reading in this file, above ruby's 39.12 and java's 33.34, both of which + * carry a full budget. + * 2. TWO OF THE STATED REASONS NO LONGER HOLD. swift was excluded as "below + * its own noise floor" on 0.98 MB at 8000 files against 0.29 MB at 32 000; + * it now reads 969 120 B and 3 449 216 B, growing the right way. COBOL was + * excluded "for the same reason" on 0.54 MB then 0 B; it now reads + * 536 264 B and 2 320 456 B, ratio 1.082. Neither number moved because + * either index changed — the ARM changed, twice, when it started resolving + * a real import (#2903) and when `measureHeap` began flattening its + * corpus. Both re-measure to within 0.24% peak-to-peak over five runs, + * which is not a noise floor. + * 3. THE PROSE HAD GONE STALE AGAINST ITSELF. It quoted javascript at + * 46 208 832 B four paragraphs after quoting it at 25.51 MiB + * (26 745 296 B), because one number was re-taken with the arm and the + * other was only ever written down. + * + * Only rust's exclusion survived unchanged: 16 B at 8000 files and 16 B at + * 32 000, identical in all five runs, because it probes candidate paths with + * `allFilePaths.has(...)` and builds nothing. + * + * So the nine are still not BUDGETED — their ceilings, floors and ratio arms + * are not this change to write — but they are all measured and all bounded. See + * `HEAP_BOUNDED` for the gate and `_heap_bound_note` in baselines.json for each + * language's reading and its own reason, which are not one reason: rust builds + * nothing; go, dart, kotlin, swift and cobol build something this file has + * never bounded; and typescript, vue and cpp are duplicates of a BUILDER and of + * a READ PATTERN, both halves of which have to hold — `csharp_csproj` was + * excluded on the first half alone, at +20.8% of the C# index, and reads 2.47x + * of it now that the second half decides the number. Measured here: typescript + * 26 745 296 B against javascript's 26 745 296 B (byte-identical in four runs + * of five), cpp 10 023 344 B against c's 10 018 816 B (+0.05%), vue + * 28 884 016 B (+8.0%, what `.vue` instead of `.ts` buys on two thirds of the + * paths). The bound is what watches for the divergence the re-entry condition + * names — and it watches at 1.5x, so it catches a language GROWING an index, + * not a duplicate drifting by 8%. That limit is stated rather than papered + * over: the tight form is a same-process ratio against the arm each one + * duplicates, which is the only form immune to the cross-runner heapUsed drift + * an absolute bound has to tolerate. + * * KNOWN BLIND SPOT, measured: a full workspace scan reintroduced on 1-in-32 * imports passes every arm here (dart scored 1.458 scaling, 1.736 ms). The gate * that NARROWS it is not a timing gate — the parity test above counts @@ -120,10 +317,114 @@ * mutation. It does not CLOSE it: the counter watches the Set, while the * resolvers hold materialized arrays of the same file list * (`WorkspaceFileIndex.normalized`/`.all`, Dart's basename buckets, - * `PackageDirIndex.filesByDir`), and a 1-in-32 scan over one of THOSE passes + * `PackageDirIndex.filesByDir`, PHP's `filesByRawDirectory`, COBOL's two tier + * maps), and a 1-in-32 scan over one of THOSE passes * both the parity test and `--check`. Chasing it by tightening these ceilings * toward the noise floor would only buy flaky CI; see `_blind_spot` in - * baselines.json. + * baselines.json. PR #2911 is the proof that this blind spot is real rather + * than theoretical: JavaScript's missing index was a scan of + * `ImportPassCache.normalizedFileList` on EVERY import, which the Set counter + * could not see, and it took a differential parity test over 211 200 pairs plus + * this bench's arrival to pin it. + * + * THE FIFTH ARGUMENT — `context`, and exactly how much of it is measured. This + * harness used to call the inner resolvers with THREE arguments while `run.ts` + * calls `provider.resolveImportTarget` with FIVE, the fifth being + * `{ parsedFiles, parsedImport }`. Every arm was therefore a measurement of a + * call shape production never makes, and that is not a cheap thing to get + * wrong: defeating the `perFileSet` memo behind PHP's `filesByDirectory` + * measures 197.0 µs -> 9976.2 µs per import (50.6x) with every test still + * green, and nothing here could see it. + * + * `resolveOne` now makes the production call. Only TWO of the seventeen arms + * can observe it — PHP and Python are the only registered hooks that declare a + * fifth parameter — and that is ASSERTED rather than asserted-in-a-comment: the + * inventory arm at the foot of the file reads + * `SCOPE_RESOLVERS.get(language).resolveImportTarget.length` and reconciles it + * against `CONTEXT_LANGS` in both directions, so a language that grows a + * context leg cannot ship with the leg unmeasured. The other fifteen are handed + * nothing and build no `ParsedFile[]` at all, so their numbers are unmoved. + * + * `newPass` mints the `ParsedFile[]` FIRST and derives the path set from it + * (`new Set(parsedFiles.map(f => f.filePath))`), because that is what `run.ts` + * does — two independently built lists are a shape the pipeline cannot produce + * and would let the two memos disagree about which files exist. Both are fresh + * per pass for the reason the Set always was: `filesByDirectory` (PHP) and + * `parsedFileByPath` (Python) are `perFileSet` memos keyed on the ARRAY's + * identity, so a reused array would hide their build from rep 2 onward and + * `fastest()` takes the minimum. + * + * THE LEGS ACTUALLY RUN, which is what a "context is threaded" claim is worth + * nothing without — a leg that returns early measures nothing, the exact + * failure the four 0 B heap arms already demonstrated in this file. PHP's needs + * `parsedImport.kind` to be `named` or `alias` AND `importedSymbolKind` to be + * `function` or `const`; Python's needs a `named`/`alias` import too, because + * the synthetic `namespace` spelling this file used to pass makes + * `pythonImportedSubmoduleTarget` return null and `context.parsedFiles` is then + * never read at all. A deterministic `context` arm pins both per language: a + * three-file corpus resolved through `resolveOne` twice, once with the pass's + * `parsedFiles` and once without, whose two answers must DIFFER and must both + * equal what baselines.json records. Dropping the fifth argument, dropping + * `importedSymbolKind`, or reverting Python to `namespace` collapses the two + * onto one value and fails — and no other arm here would: on the main corpus + * the leg agrees with the cascade, so both languages' fingerprints are + * UNCHANGED by this (measured, all ten). + * + * WHAT IS STILL NOT MEASURED, narrowed rather than deleted: + * + * - Python's `parsedFileByPath` memo is exercised by the five timing arms and + * NOT by the heap arm, and structurally cannot be. `retainedPassBytes` + * requires its probe to MISS, while every path that builds that memo runs + * through a non-null `packageTarget` which `resolvePythonImportTarget` then + * returns. So nothing here bounds that Map's footprint; it is one pointer + * per parsed file, O(files) with no depth term, and the count gate in + * import-target-index-reuse.contract.test.ts is what holds it to one build + * per pass; + * - PHP's leg is measured with NO composer.json — `resolutionConfig` is + * undefined here, as it always has been — so `namespaceDirectories` only + * ever returns the directory of an already-resolved file and the PSR-4 + * mapping branch stays unreached, exactly as `csharp` cannot reach the + * csproj leg. Closing that is a second PHP arm on the `csharp_csproj` + * precedent, not a parameter; + * - the `const` tail of PHP's leg (`candidateFiles.length === 1`) is a + * different ANSWER, not a different cost: `function` runs the identical + * candidate gather and `localDefs` filter and diverges only in the last two + * lines. It is gated by count in the contract test above. + * + * COST, and the honest version of it. REPORT mode is ~33-35 s, down from ~46 s: + * the timing phase fell from 39.8 s to 28.7 s when `REPS` became per-language + * (see `repsFor`), and that win is real. `--check` is ~44-45 s, which is + * ESSENTIALLY UNCHANGED from the ~46 s it cost before, because the inventory + * arm added here loads `pipeline/registry.ts` and that one dynamic import + * consumes almost the whole `repsFor` win — measured 6.3-6.5 s on one box and + * 9.3-10.0 s on another, in isolation and after this file's own static imports + * are already resident. Do not read the two modes as "~46 → ~42": only report + * mode got faster. + * + * MEASURING ALL SEVENTEEN HEAP ARMS instead of eight costs 1.37 s, and that is + * a measured number rather than the "seconds are free here" the paragraph below + * would have let it be. Timed per language with the phase instrumented, twice: + * the heap phase goes 2.06 s -> 3.43 s (1.377 s and 1.370 s added over the two + * runs). The nine are kotlin 0.57 s — it retains the largest index of the nine + * and builds all three of its maps eagerly — then vue 0.18, typescript 0.17, + * cpp 0.09, swift 0.08, dart 0.08, go 0.07, cobol 0.07, rust 0.06. End to end + * that is report mode 33.76 s -> 34.93 s (min of three runs each, +1.17 s, + * consistent with the phase measurement inside run-to-run noise). `--check` was + * 41.60 s before and reads 41.48-43.56 s after, i.e. the whole-run difference + * is INSIDE the registry import's own 6.3-10.0 s spread and cannot be resolved + * at that level — the +1.37 s phase number is the one to quote. + * + * That cost was weighed and KEPT, on the one number that decides it: the + * `benchmarks` job is not CI's critical path. On the last green run of main it + * finished in 9 m 23 s against 12 m 58 s for the sharded coverage job that + * gates the merge, so ~4 m 40 s of slack sits above this bench and ten seconds + * of it buys zero merge latency. Moving the arm into a vitest file would move + * the registry load ONTO that critical path, and would weaken it besides: from + * `LANG_REGISTRY`'s `SupportedLanguages` values, which are what the five + * dispatcher branches key off, down to baselines.json's arm NAMES plus a + * hand-written rule for de-aliasing `csharp_csproj`. See the wall-clock note in + * `_arms_note` for the per-language breakdown and for what to drop first if + * that stops fitting the job. * * Run: * node --expose-gc --import tsx bench/import-target/measure.mjs # report @@ -134,12 +435,36 @@ import path from 'node:path'; import crypto from 'node:crypto'; import { fileURLToPath } from 'node:url'; +import { SupportedLanguages } from 'gitnexus-shared'; + import { resolveGoImportTarget } from '../../src/core/ingestion/languages/go/import-target.ts'; import { resolveDartImportTarget } from '../../src/core/ingestion/languages/dart/import-target.ts'; import { resolveRubyImportTarget } from '../../src/core/ingestion/languages/ruby/import-target.ts'; import { resolveCsharpImportTarget } from '../../src/core/ingestion/languages/csharp/import-target.ts'; import { resolveKotlinImportTarget } from '../../src/core/ingestion/languages/kotlin/import-target.ts'; -import { getWorkspaceFileIndex } from '../../src/core/ingestion/import-resolvers/workspace-file-index.ts'; +import { resolvePhpImportTargetInternal } from '../../src/core/ingestion/languages/php/import-target.ts'; +import { resolveJavaImportTarget } from '../../src/core/ingestion/languages/java/import-target.ts'; +import { cobolScopeResolver } from '../../src/core/ingestion/languages/cobol/scope-resolver.ts'; +import { resolveSwiftImportTarget } from '../../src/core/ingestion/languages/swift/import-target.ts'; +import { resolveRustImportTarget } from '../../src/core/ingestion/languages/rust/import-target.ts'; +import { resolvePythonImportTarget } from '../../src/core/ingestion/languages/python/import-target.ts'; +import { makeJsResolveImportTarget } from '../../src/core/ingestion/languages/javascript/import-target.ts'; +import { makeVueResolveImportTarget } from '../../src/core/ingestion/languages/vue/import-target.ts'; +// The two `ScopeResolver`s, not their inner resolvers — see `RESOLVE_HOOK`. +import { typescriptScopeResolver } from '../../src/core/ingestion/languages/typescript/scope-resolver.ts'; +import { cScopeResolver } from '../../src/core/ingestion/languages/c/scope-resolver.ts'; +import { cppScopeResolver } from '../../src/core/ingestion/languages/cpp/scope-resolver.ts'; +// `SCOPE_RESOLVERS` is NOT imported here — see the inventory arm at the bottom, +// which loads it dynamically. Statically it costs 6-10 s of module load +// depending on the box (measured both ways there), because reaching the +// registry pulls in every registered provider and everything under them, and it +// is wanted by one `--check` arm that runs after the last measurement. + +/** The JS and Vue adapter FACTORIES return a closure; the memo they read is + * module-level, so one instance per process is both correct and what the + * registry does (`resolveImportTarget: makeJsResolveImportTarget()`). */ +const jsResolveImportTarget = makeJsResolveImportTarget(); +const vueResolveImportTarget = makeVueResolveImportTarget(); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const BASELINE_PATH = path.resolve(__dirname, 'baselines.json'); @@ -149,18 +474,54 @@ const LARGE = 1600; const IMPORTS_PER_FILE = 8; /** Extra directory components prepended in the `deep` arm — see `depth_ratio`. */ const DEEP_PAD = 16; -/** `fastest()` below is a min-of-N estimator, so N is the noise knob: raising it - * lowers and stabilises the minimum. `depth_ratio` divides two sub-3 ms - * measurements, and Dart's are sub-1 ms, so it is by far the noisiest number - * here and it sets N for the whole file. Measured over 22 `--check` runs on an - * idle box: at N=5 it tripped its own budget ~1 run in 20, at N=7 Dart still - * swung 3.0x peak-to-peak and tripped once. N=15 matches `bench/cfg`, - * `bench/schema-pairs` and `bench/callable-value-flow`; the distributions it - * produces are recorded in `_arms_note`. */ -const REPS = 15; +/** + * `fastest()` below is a min-of-N estimator, so N is the noise knob: raising it + * lowers and stabilises the minimum. `depth_ratio` divides two sub-3 ms + * measurements, and Dart's are sub-1 ms, so it is by far the noisiest number + * here. Measured over 22 `--check` runs on an idle box: at N=5 it tripped its + * own budget ~1 run in 20, at N=7 Dart still swung 3.0x peak-to-peak and + * tripped once. N=15 (which matches `bench/cfg`, `bench/schema-pairs` and + * `bench/callable-value-flow`) collapsed every language to a 1.13-1.26x swing + * with 22/22 passing; the distributions are recorded in `_arms_note`. + * + * N used to be 15 for EVERY arm, set globally by the noisiest cell. That paid + * the noisiest cell's insurance premium on cells a thousand times its size: + * the recorded overshoot of min-of-K against min-of-15 is a function of the + * cell's absolute duration, not of the language — 31.8% on `swift.small` + * (0.43 ms) and 37.6% on `dart.collide` (1.5 ms) at the extreme, but at most + * 6.3% at K=7 for every cell at or above 10 ms. + * + * So N is picked PER LANGUAGE, from the cost of its cheapest arm: 15 while that + * is under `REPS_CHEAP_MS`, and `~REPS_BUDGET_MS` worth of samples above it, + * floored at `REPS_MIN`. Per language rather than per cell so all five arms of + * a language share one estimator and the four ratios stay comparisons of like + * with like. In practice that is still 15 for go, csharp, dart, kotlin, java, + * cobol, swift, rust, python, c and cpp — every language the flakiness above + * was ever about — and 7-8 for php, csharp_csproj, ruby, javascript, typescript + * and vue, whose cheapest cell is 20-28 ms. Replayed against two independent + * runs' sample sets it saved 12.8 s and 12.4 s of a 46 s run with all 85 cells + * passing all five gates at 0.4-0.7 of budget, and min-of-7 reads slightly + * HIGHER than min-of-15, so the gates get marginally more sensitive rather than + * less. The chosen N is reported per language as `reps`. + */ +const REPS_MAX = 15; +const REPS_MIN = 7; +/** Sampling budget per cell for the languages that do not get `REPS_MAX`. */ +const REPS_BUDGET_MS = 150; +/** Below this a cell is small enough for the min-of-N estimator itself to be + * the dominant error, so it gets the full `REPS_MAX` regardless of budget. The + * nearest language on either side of it is 3.2 ms and 20.0 ms, so nothing sits + * near the boundary. */ +const REPS_CHEAP_MS = 5; const WARMUP = 2; -/** Heap arm (C#, Ruby). Far more files than the timing arms because the finding +/** N for one language, from one warmed pass of its cheapest arm. */ +function repsFor(probeMs) { + if (probeMs < REPS_CHEAP_MS) return REPS_MAX; + return Math.min(REPS_MAX, Math.max(REPS_MIN, Math.ceil(REPS_BUDGET_MS / probeMs))); +} + +/** Heap arm. Far more files than the timing arms because the finding * is an ABSOLUTE footprint at repository scale, and 1600 files would report a * fraction of a MiB — a number no ceiling could usefully bound. `HEAP_PAD` * keeps the paths at a plausible monorepo depth: `buildSuffixIndex` is @@ -168,12 +529,79 @@ const WARMUP = 2; const HEAP_SMALL = 8000; const HEAP_LARGE = 32000; const HEAP_PAD = 8; -/** Languages whose resolvers retain the shared `WorkspaceFileIndex`. */ -const HEAP_LANGS = ['csharp', 'ruby']; +/** The languages whose retained per-pass index carries a BUDGET — a ceiling, a + * floor derived from `heap_reading_bytes`, and the linear-growth ratio arm. + * All eight are measured the same way as the other nine (`retainedPassBytes`, + * one real import through the real resolver); what this list decides is which + * GATE a reading gets, not whether it is taken. The first five reach the shared + * `WorkspaceFileIndex` and retained NOTHING at BASE; `csharp_csproj` is the + * same corpus through the same index under the csproj context, and it is here + * rather than excluded as a duplicate because after #2903 its READ PATTERN, + * not its corpus, decides the number. + * + * The remaining three are `HEAP_BOUNDED`, DERIVED from this list rather than + * written beside it, and they carry an upper bound and NO floor. That asymmetry + * is the point: a bound catches "this language grew an index", which is the + * re-entry condition, while a floor over a reading at or below its own noise + * would gate the noise. rust reads 16 B at both scales; swift's ratio is 0.888 + * and cobol's 1.082, both outside the linearity every budgeted arm shows, so a + * floor and a ratio arm would be measuring the measurement. See the MEMORY + * section of the header for what re-measuring all seventeen found. */ +const HEAP_BUDGETED = [ + 'csharp', + 'csharp_csproj', + 'ruby', + 'php', + 'java', + 'javascript', + 'python', + 'c', + // Promoted once every language was actually measured. Each retains a real + // per-pass structure and each grows LINEARLY with the file count (ratio + // 0.996-1.004 against a 1.25 budget over 8000 -> 32000 files), so each can + // carry the full ceiling + floor + ratio set rather than a bound alone. + // kotlin's 45.85 MiB is the second-largest reading in this file — larger than + // ruby's and java's, both of which were budgeted from the start — and it had + // no stated exclusion reason at all. + 'kotlin', + 'dart', + 'go', + 'typescript', + 'vue', + 'cpp', +]; -/** Needs `node --expose-gc` to force collection for a clean delta; without it - * the heap metric is reported as null and its `--check` gate would be skipped, - * which is why `--check` refuses to run without the flag (see below). */ +/** + * The arms handed the fifth `context` argument — `{ parsedFiles, parsedImport }` + * — because their registered hook DECLARES it. Two of seventeen, and the + * inventory arm at the foot of this file reconciles that claim against + * `SCOPE_RESOLVERS` in both directions rather than trusting this line. + * + * These are also the only arms for which `newPass` builds a `ParsedFile[]` at + * all. Building one for the other fifteen would cost their timed loop an + * O(files) allocation per pass that no resolver of theirs can even observe — + * their hooks declare three or four parameters — so their numbers stay exactly + * where they were. + */ +const CONTEXT_LANGS = ['php', 'python']; + +/** + * Needs `node --expose-gc` to force collection for a clean delta; without it + * the heap metric is reported as null and its `--check` gate would be skipped, + * which is why `--check` refuses to run without the flag (see below). + * + * TWO cycles because the value is a `WeakMap`'s: the first clears the entry + * once its key is unreachable, the second collects what the entry held. That is + * not always enough — PHP reaches the shared index through a second per-file- + * set memo of its own (`getPhpWorkspaceIndex` wraps `getWorkspaceFileIndex`, + * both keyed on the same Set) and that chain measured FOUR cycles to release, + * with two leaving 9.3 MB of the previous read still counted live. The answer + * to that is `HEAP_RETAINED`, which removes the need to release anything inside + * a measurement window, plus the deeper drain `measureHeap` runs between + * languages where a late free costs nothing. Cycles are not the knob: with + * `HEAP_RETAINED` in place, two and four produce byte-identical readings, and + * four cost 4.5 s of wall clock over a retained heap this size. + */ const GC = typeof global.gc === 'function' ? () => (global.gc(), global.gc()) : null; /** Deterministic 32-bit avalanche (murmur3 finalizer) — no `Math.random()`, so @@ -186,11 +614,113 @@ function mix(n) { } const GO_MODULE = { modulePath: 'example.com/mod' }; -const EXTENSION = { go: '.go', csharp: '.cs', dart: '.dart', ruby: '.rb', kotlin: '.kt' }; +/** + * The `csharp_csproj` arm's project configs — the whole reason that arm exists. + * + * `csharp` builds its context with NO `csharpConfigs`, so every one of its + * imports takes the no-csproj branch and the csproj leg's namespace-directory + * index (#2902) would ship unmeasured. Two configs rather than one because the + * leg's cost is a function of `dirPrefix`'s SHAPE, and one config cannot + * produce all three: + * - `App` + `projectDir: 'src'` gives `dirPrefix = 'src/'`, which + * CONTAINS a slash, so `candidateDirs` answers from the last-segment bucket; + * - `Lib` + `projectDir: ''` gives `dirPrefix = ''`, slash-FREE, the + * one leg that sweeps the last-segment KEYS and so is not constant-time; + * - `Lib` itself (the import IS the root namespace, no `projectDir` to stand + * in) gives an EMPTY `dirPrefix`, answered from `singleSegmentDirs`. + * All three were a full `normalizedFileList` pass per import before #2902. + */ +const CSPROJ_CONFIGS = [ + { rootNamespace: 'App', projectDir: 'src' }, + { rootNamespace: 'Lib', projectDir: '' }, +]; +/** + * The `tsconfigPaths` the Vue arm threads as `resolutionConfig`. + * + * The Vue adapter is `resolveTsTarget` with `language: TypeScript` and nothing + * else, so with a null config its arm would be a byte-for-byte re-run of the + * TypeScript one over a differently-spelled corpus. The alias branch + * (`standard.ts:57-70`) is the one leg of the shared resolver that neither the + * `javascript` arm (which pins `tsconfigPaths: null`) nor the `typescript` arm + * here reaches, so wiring it is what makes this a third measurement rather than + * a third copy — and every local Vue import below is spelled `@/…`. + */ +const VUE_TSCONFIG = { tsconfigPaths: { aliases: new Map([['@/', 'src/']]), baseUrl: '.' } }; +/** Keyed by LAYOUT name, so there is no `csharp_csproj` row: `buildFiles` + * aliases that arm to `csharp` before this table is read. */ +const EXTENSION = { + go: '.go', + csharp: '.cs', + dart: '.dart', + ruby: '.rb', + kotlin: '.kt', + php: '.php', + java: '.java', + cobol: '.cbl', + swift: '.swift', + rust: '.rs', + python: '.py', + javascript: '.js', + typescript: '.ts', + vue: '.vue', + c: '.c', + cpp: '.cpp', +}; +/** C and C++ resolve `#include` against HEADERS, which reach the resolver + * through `resolutionConfig` rather than through `allFilePaths` — see + * `newPass`. Half of each corpus is headers; this is their extension. */ +const HEADER_EXTENSION = { c: '.h', cpp: '.hpp' }; /** Directory fan-out. Shared because `buildRepo`'s collide targets address * files by `j % dirs` / `Math.floor(j / dirs)` and must agree with the layout * `buildFiles` produced. */ const dirsFor = (fileCount) => Math.max(4, Math.floor(fileCount / 8)); +/** Swift's collide arm, and the ONLY place a bucket size is pinned by a + * constant rather than by `dirsFor`. A module bucket is what Swift returns, so + * its cardinality has to grow with the corpus for the arm to measure anything: + * four modules means fileCount/4 per bucket (100 at `collide`, 400 at + * `collide_large`), which is the shape a small SPM package actually has. */ +const SWIFT_COLLIDE_MODULES = 4; +/** File stems follow each language's own naming convention, because C#'s and + * PHP's suffix maps carry a case-insensitive tier and a lower-cased corpus + * would leave it answering the same question twice. Keyed by LAYOUT name, like + * `EXTENSION` — no `csharp_csproj` row, for the same reason. */ +const PASCAL_CASE_FILES = new Set([ + 'csharp', + 'kotlin', + 'php', + 'java', + 'cobol', + // Swift types and Vue SFCs are PascalCase by universal convention. + 'swift', + 'vue', +]); +/** Rust and Python name a DIRECTORY as a module through a well-known file, so + * the first file minted in each directory is that file rather than a numbered + * one. Every in-repo target below resolves to one of them. */ +const PACKAGE_STEM = { rust: 'mod', python: '__init__' }; + +/** + * The end of a per-language dispatcher, where five of them used to fall through + * to a bare `return`. + * + * Four of those fallthroughs meant "ruby" and the fifth meant "csharp". So + * `ruby` appeared nowhere in this file except `EXTENSION` and the language + * list, and — the part that matters — a language added to the list but missed + * in the dispatchers would have been benchmarked as RUBY'S CORPUS RESOLVED BY + * C#'S RESOLVER: five plausible timings, a stable fingerprint, and a permanent + * pass over a language nobody had measured. Every dispatcher now names its last + * branch and throws here instead, so the missing wiring is a crash on the first + * run rather than a green gate. + */ +function unwiredLanguage(where, lang) { + return new Error( + `bench: ${where} has no branch for '${lang}'. Every language in LANG_REGISTRY needs one in ` + + `uniqueDir, collideDir, uniqueTarget, collideTarget and resolveOne. (uniqueDir and ` + + `collideDir see the LAYOUT name, which is never 'csharp_csproj' — buildFiles aliases it ` + + `to 'csharp'.) Falling through here used to hand the language another one's corpus or ` + + `another one's resolver, and nothing in --check could tell.`, + ); +} /** * UNIQUE-LEAF layout: one directory name per index, so no two directories share @@ -200,7 +730,7 @@ const dirsFor = (fileCount) => Math.max(4, Math.floor(fileCount / 8)); * package-dir-index.ts), and the shape Kotlin's `dirChildren` resolves the same * way. */ -function uniqueDir(lang, d) { +function uniqueDir(lang, d, i) { if (lang === 'go') return d % 7 === 0 ? `src/pkg${d}/internal/pkg${d}` : `src/pkg${d}`; if (lang === 'csharp') return d % 7 === 0 ? `src/Ns${d}/Sub/Ns${d}` : `src/Ns${d}`; if (lang === 'dart') return d % 3 === 0 ? `lib/feature${d}` : `pkg/feature${d}`; @@ -209,7 +739,35 @@ function uniqueDir(lang, d) { ? `mod${d}/src/main/kotlin/com/example/pkg${d}/inner/pkg${d}` : `mod${d}/src/main/kotlin/com/example/pkg${d}`; } - return `lib/mod${d}`; + if (lang === 'php') return d % 7 === 0 ? `src/App/Ns${d}/Sub/Ns${d}` : `src/App/Ns${d}`; + if (lang === 'java') { + return d % 7 === 0 + ? `mod${d}/src/main/java/com/example/pkg${d}/inner/pkg${d}` + : `mod${d}/src/main/java/com/example/pkg${d}`; + } + // COBOL resolves on the BASENAME alone (`path.basename(fp, ext)`), so its + // directories are pure realism — a copybook library beside the programs. + if (lang === 'cobol') return d % 3 === 0 ? `copybooks/grp${d}` : `src/prog${d}`; + // SPM. The nested slice makes one file's interior segments repeat + // (`Sources/Mod7/Internal/Mod7/File7.swift`), and `getSwiftModuleIndex` + // pushes once per segment, so that file appears TWICE in module `Mod7`'s + // returned list. Real layout, real output; the fingerprint pins it. + if (lang === 'swift') { + return d % 7 === 0 ? `Sources/Mod${d}/Internal/Mod${d}` : `Sources/Mod${d}`; + } + // Cargo. The nested slice has NO `mod{d}/mod.rs`, so `crate::mod{d}::thing` + // misses there — the same resolves/misses split every other unique arm has. + if (lang === 'rust') return d % 7 === 0 ? `src/mod${d}/inner` : `src/mod${d}`; + if (lang === 'python') return d % 7 === 0 ? `pkg${d}/inner` : `pkg${d}`; + if (lang === 'javascript' || lang === 'typescript') return `src/mod${d}`; + // Vue's local imports are all `@/…`, which the alias rewrites to `src/…`, so + // the whole corpus must live under `src/` for that branch to hit. + if (lang === 'vue') return `src/mod${d}`; + // C and C++ split headers from sources — the shape that makes + // `resolutionConfig` load-bearing. Odd `i` is the header. + if (lang === 'c' || lang === 'cpp') return i % 2 === 1 ? `include/comp${d}` : `src/comp${d}`; + if (lang === 'ruby') return `lib/mod${d}`; + throw unwiredLanguage('uniqueDir', lang); } /** @@ -222,7 +780,7 @@ function uniqueDir(lang, d) { * Each language's local import spelling is chosen so this arm resolves exactly * as many imports as `small` does (asserted): same workload, different layout. */ -function collideDir(lang, d) { +function collideDir(lang, d, i) { if (lang === 'go') { if (d % 7 === 0) return `svc${d}/internal/sub/internal`; return d % 5 === 1 ? `svc${d}/internal/shared` : `svc${d}/internal`; @@ -234,7 +792,32 @@ function collideDir(lang, d) { ? `mod${d}/src/main/kotlin/com/example/models/inner/models` : `mod${d}/src/main/kotlin/com/example/models`; } - return `svc${d}/lib/models`; + if (lang === 'php') return `svc${d}/src/Models`; + if (lang === 'java') { + return d % 7 === 0 + ? `svc${d}/src/main/java/com/example/model/inner/model` + : `svc${d}/src/main/java/com/example/model`; + } + if (lang === 'cobol') return `svc${d}/copybooks`; + // Swift's collision axis is neither a shared directory name nor a shared + // basename: `byModule` is KEYED on the module name, so what grows a bucket is + // FEWER modules holding MORE files. `SWIFT_COLLIDE_MODULES` of them, so the + // bucket a hit returns is fileCount/4 — 100 entries at 400 files and 400 at + // 1600 — and a hit copies that whole bucket minus the importer. + if (lang === 'swift') return `Sources/Mod${d % SWIFT_COLLIDE_MODULES}`; + // Rust's cost is O(path SEGMENTS), not O(files) — it probes candidate paths + // with `.has()` and never searches. So its collide arm is a deep module tree + // whose targets carry ~2x the `::` segments, which is the axis that CAN grow; + // that the ratio across file counts stays flat on it is the assertion. + if (lang === 'rust') return `src/l0/l1/l2/l3/l4/mod${d}`; + // The `inner` slice mirrors the unique arm's, and for the same reason: it is + // where the in-repo target misses, so both arms resolve the same count. + if (lang === 'python') return d % 7 === 0 ? `svc${d}/models/inner` : `svc${d}/models`; + if (lang === 'javascript' || lang === 'typescript') return `pkg${d}/src`; + if (lang === 'vue') return `src/pkg${d}/components`; + if (lang === 'c' || lang === 'cpp') return i % 2 === 1 ? `svc${d}/include` : `svc${d}/src`; + if (lang === 'ruby') return `svc${d}/lib/models`; + throw unwiredLanguage('collideDir', lang); } /** @@ -255,43 +838,166 @@ function collideDir(lang, d) { function buildFiles(lang, fileCount, pad, shape) { const dirs = dirsFor(fileCount); const files = []; - const ext = EXTENSION[lang]; + // `csharp_csproj` is `csharp` with a different CONTEXT and nothing else. The + // alias is here, in the one place that mints paths, rather than as a second + // copy of the same layout in `uniqueDir`/`collideDir`: it makes the two arms' + // corpora identical by construction, so a later edit to C#'s layout cannot + // silently desynchronize them and turn the comparison into two experiments. + const layout = lang === 'csharp_csproj' ? 'csharp' : lang; + const ext = EXTENSION[layout]; const prefix = pad === 0 ? '' : Array.from({ length: pad }, (_, n) => `d${n}`).join('/') + '/'; for (let i = 0; i < fileCount; i++) { const d = i % dirs; - const dir = shape === 'collide' ? collideDir(lang, d) : uniqueDir(lang, d); - // In the collide shape Dart and Ruby carry a REPEATED basename — the term - // their indexes bucket on. `i / dirs` is unique within a directory (8 files - // land in each) and identical across directories, which is exactly the - // `models.dart` / `models.rb`-in-every-package convention. Go, C# and - // Kotlin bucket on the directory instead, so their stems stay unique. - const collideStem = lang === 'dart' || lang === 'ruby'; - const stem = - shape === 'collide' && collideStem - ? `mod${Math.floor(i / dirs)}` - : lang === 'csharp' || lang === 'kotlin' - ? `File${i}` - : `file${i}`; + const dir = shape === 'collide' ? collideDir(layout, d, i) : uniqueDir(layout, d, i); + // In the collide shape Dart, Ruby, PHP and COBOL carry a REPEATED basename + // — the term their indexes bucket or key on (COBOL's two tier maps are + // keyed on the uppercased basename and NOTHING else). `i / dirs` is unique + // within a directory (8 files land in each) and identical across + // directories, which is exactly the `models.dart` / `models.rb`-in-every- + // package convention. Go, C#, Kotlin and Java bucket on the DIRECTORY + // instead, so their stems stay unique and the shared leaf segment is what + // collides for them. + const collideStem = + layout === 'dart' || + layout === 'ruby' || + layout === 'cobol' || + layout === 'php' || + // The three added later that also bucket or key on the BASENAME: + // JS/TS `buildSuffixIndex` (one entry per path suffix, so the last + // component is the shortest key), Vue through the same index, Python's + // `byBasename`, and C/C++'s basename map — for the last, only the header + // half is addressable, so only it repeats (see below). + layout === 'javascript' || + layout === 'typescript' || + layout === 'vue' || + layout === 'python' || + layout === 'c' || + layout === 'cpp'; + const [fileStem, modStem] = PASCAL_CASE_FILES.has(layout) ? ['File', 'Mod'] : ['file', 'mod']; + let stem = + shape === 'collide' && collideStem ? `${modStem}${Math.floor(i / dirs)}` : `${fileStem}${i}`; + // Rust's `mod.rs` and Python's `__init__.py`: one per directory, and the + // file every in-repo target of theirs resolves to. Minted at the first file + // of each directory (`i < dirs`, so `d === i`), which is why both arms' + // resolved counts are the count of in-repo imports either way. + if (PACKAGE_STEM[layout] !== undefined && i < dirs) stem = PACKAGE_STEM[layout]; + // C and C++ address only HEADERS, so only their stems repeat in the collide + // shape; the sources stay unique and are pure corpus weight, exactly as in a + // real tree where nobody `#include`s a `.c`. + if ((layout === 'c' || layout === 'cpp') && i % 2 === 0) stem = `src${i}`; // Go's package leg must exclude `_test.go`; keep a real share of them. // Kotlin resolves `.kt` and `.kts` through the same stem maps; keep both. + // COBOL's copybook tier (`.cpy`) BEATS its source tier (`.cbl`) on the same + // bookname, so both extensions have to be present for that tie-break to be + // reachable at all — and in the collide shape, where basenames repeat, one + // bookname really does land in both tiers. + // A Vue repo is `.vue` SFCs plus plain `.ts` modules, and only the second + // kind reaches the extension-guessing leg (SFC imports carry `.vue` + // explicitly), so both have to be present for both legs to be measured. + // C/C++ alternate header and source; the header half is the addressable one. const suffix = - lang === 'go' && i % 6 === 0 ? '_test.go' : lang === 'kotlin' && i % 11 === 0 ? '.kts' : ext; + layout === 'go' && i % 6 === 0 + ? '_test.go' + : layout === 'kotlin' && i % 11 === 0 + ? '.kts' + : layout === 'cobol' && i % 3 === 0 + ? '.cpy' + : layout === 'vue' && i % 3 === 0 + ? '.ts' + : HEADER_EXTENSION[layout] !== undefined && i % 2 === 1 + ? HEADER_EXTENSION[layout] + : ext; files.push(`${prefix}${dir}/${stem}${suffix}`); } return files; } +/** + * ONE `ParsedFile`, and the ONE place in this file that spells that shape. + * + * CARRIES THE FIELDS THE RESOLVERS READ AND NOTHING ELSE, deliberately. + * `filesByDirectory` reads `filePath`; PHP's declaring-file filter reads + * `localDefs[].type` and `localDefs[].qualifiedName`; Python's + * `pythonFileExportsName` reads `localDefs[].qualifiedName`. `scopes`, + * `parsedImports` and `referenceSites` are on the real shape and are inert on + * this path, and the timed corpora are rebuilt inside every pass (see + * `newPass`), so filling them would charge the RESOLUTION arms for extraction + * work that happens in another phase entirely. + * + * `nodeId` is inert as well — checked, not assumed: neither + * `php/import-target.ts` nor `python/import-target.ts` mentions it, and they are + * the two modules `resolveOne` enters. It is minted anyway because it is on the + * real shape, and its spelling is therefore free to be uniform. + * + * Both callers come through here — `buildParsedFiles` for the timed and heap + * corpora, `CONTEXT_PROBE` for the `context` arm's hand-built ones. It used to + * be spelled out twice, ~900 lines apart, differing only in that `nodeId`; this + * is an untyped `.mjs`, so nothing would have failed at build if `ParsedFile` + * grew a field and only one of the two copies learned about it. + */ +const probeFile = (filePath, defs) => ({ + filePath, + moduleScope: filePath, + scopes: [], + parsedImports: [], + localDefs: defs.map(([type, qualifiedName], n) => ({ + nodeId: `${filePath}#${n}`, + filePath, + type, + qualifiedName, + })), + referenceSites: [], +}); + +/** + * The `ParsedFile[]` the orchestrator threads beside the path set, for the two + * languages whose hook declares a `context` — see `CONTEXT_LANGS`. + * + * Two defs per file, and both are real shapes rather than padding. PHP keeps + * classes and functions in SEPARATE symbol tables, so `App\Ns7\File7` naming + * both a class and a function is ordinary PHP — and it is what makes the leg's + * two halves reachable on the same corpus: the class def exercises the + * `def.type !== expectedType` reject (which returns before the split) and the + * function def exercises the `split(/[\\.]/).at(-1)` compare that decides the + * match. The qualified name carries two separators because that split's cost is + * a function of how many there are, and a one-segment name would understate it. + * + * The owner segment is the file's own directory name (`Ns7`, `Models`, `pkg7`), + * which is stable across the `small`, `deep` and `collide` arms — so the `deep` + * arm differs from `small` in path DEPTH alone, exactly as it does for the path + * set. That matters here: `directoryAliases` emits one entry per path segment, + * so `filesByDirectory` is O(files × depth) and the depth arm is the only one + * that can see it. + */ +function buildParsedFiles(lang, files) { + const parsedFiles = []; + for (const filePath of files) { + const slash = filePath.lastIndexOf('/'); + const stem = filePath.slice(slash + 1, filePath.lastIndexOf('.')); + const parent = slash < 0 ? '' : filePath.slice(0, slash); + const owner = parent.slice(parent.lastIndexOf('/') + 1); + const qualifiedName = lang === 'php' ? `App\\${owner}\\${stem}` : `${owner}.${stem}`; + parsedFiles.push( + probeFile(filePath, [ + ['Class', qualifiedName], + ['Function', qualifiedName], + ]), + ); + } + return parsedFiles; +} + /** * The import one file issues in the UNIQUE-LEAF layout `uniqueDir` produced. * * The TARGET axis is split from the DIRECTORY axis exactly the way `uniqueDir` * and `collideDir` split it above — two flat functions, selected once — rather - * than a `collide ?` ternary threaded through five languages' `local ? …` + * than a `collide ?` ternary threaded through seventeen languages' `local ? …` * ladders. `local` picks in-repo vs external; the handful of MISS lines that * are identical between the two shapes are duplicated on purpose, because the * alternative is four levels of nesting in a single expression. */ -function uniqueTarget(lang, { local, r, d, j }) { +function uniqueTarget(lang, { local, r, d, j, dirs }) { if (lang === 'go') { return local ? `${GO_MODULE.modulePath}/src/pkg${d}` @@ -306,6 +1012,30 @@ function uniqueTarget(lang, { local, r, d, j }) { ? ['System', 'System.Threading.Tasks', 'System.Collections.Generic'][(r >>> 4) % 3] : `Ghost${(r >>> 4) % 97}.Deep.Missing`; } + if (lang === 'csharp_csproj') { + // The mix is the arm. `System` and `Ghost{n}.Deep.Missing` match NEITHER + // root namespace, so they `continue` straight out of the config loop + // (csharp.ts:231-241) and never reach the indexed leg at all — an arm built + // on the no-csproj arm's spelling mix would measure #2902 not at all. They + // are kept as the fast-`continue` control at 1 slot in 8; the other four + // external slots address a root namespace on purpose. + if (local) return `App.Ns${d}`; + const leg = (r >>> 3) % 5; + // Matches `App`, misses every directory: `dirPrefix = 'src/Missing{n}'`, + // whose last segment buckets to nothing. 2 slots in 8. + if (leg < 2) return `App.Missing${(r >>> 4) % 97}`; + // Matches `Lib`, whose `projectDir` is empty, so `dirPrefix` is slash-FREE + // and `candidateDirs` sweeps the last-segment keys — the one leg of the + // three whose cost is not constant in the corpus. See `_arms_note`. + if (leg === 2) return `Lib.Missing${(r >>> 4) % 97}`; + // The import IS a root namespace with no `projectDir`: `dirPrefix` is + // EMPTY, the query no last-segment bucket expresses, answered from + // `singleSegmentDirs`. + if (leg === 3) return 'Lib'; + return (r >>> 4) % 2 === 0 + ? ['System', 'System.Threading.Tasks', 'System.Collections.Generic'][(r >>> 5) % 3] + : `Ghost${(r >>> 4) % 97}.Deep.Missing`; + } if (lang === 'dart') { return local ? `package:app/feature${d}/file${j}.dart` @@ -326,11 +1056,129 @@ function uniqueTarget(lang, { local, r, d, j }) { ] : `com.ghost${(r >>> 4) % 97}.deep.Missing`; } - return local - ? `mod${d}/file${j}` - : (r >>> 3) % 2 === 0 - ? ['json', 'set', 'net/http', 'digest'][(r >>> 4) % 4] - : `gem${(r >>> 4) % 97}/missing/thing`; + if (lang === 'php') { + // Backslash-separated, the way a `use` statement is actually written; the + // resolver normalizes them. No composer.json is threaded (the adapter's + // `resolutionConfig` is left undefined), so every one of these lands on + // `suffixResolve` — the leg that ran one `findIndex` over every file per + // path part per extension, ~50 of them, and measured 96.40 ms per import at + // 20k files before #2901. + return local + ? `App\\Ns${d}\\File${j}` + : (r >>> 3) % 2 === 0 + ? [ + 'Psr\\Log\\LoggerInterface', + 'Symfony\\Component\\Console\\Command', + 'Doctrine\\ORM\\EntityManager', + ][(r >>> 4) % 3] + : `Vendor${(r >>> 4) % 97}\\Ghost\\Missing`; + } + if (lang === 'java') { + // Java has NO in-repo-namespace gate (#2910 is filed for it), so a JDK + // import genuinely can resolve to a local file — `java.util.List` would + // answer to a `util/List.java` anywhere in the repo, and the progressive + // stripping loop would find it by its bare basename. These spellings are + // chosen to miss on THIS corpus (whose files are all `File{i}.java` under + // `…/pkg{d}/`) and the resolved count is asserted, not assumed. + return local + ? (r >>> 3) % 3 === 0 + ? `com.example.pkg${d}.*` + : `com.example.pkg${d}.File${j}` + : (r >>> 3) % 2 === 0 + ? ['java.util.List', 'java.io.IOException', 'java.util.concurrent.ConcurrentHashMap'][ + (r >>> 4) % 3 + ] + : `com.google.common.vendor${(r >>> 4) % 97}.Missing`; + } + if (lang === 'cobol') { + // `COPY` takes a bare bookname. A share of the local ones is spelled in + // lower case: COBOL is case-insensitive and the resolver upper-cases the + // target, so those must resolve to the same file — free coverage of the + // one transformation on the lookup path. + return local + ? (r >>> 3) % 3 === 0 + ? `file${j}` + : `File${j}` + : (r >>> 3) % 2 === 0 + ? ['DFHAID', 'DFHBMSCA', 'SQLCA', 'CICSDEF'][(r >>> 4) % 4] + : `VENDOR${(r >>> 4) % 97}`; + } + if (lang === 'swift') { + // `import X` names an SPM MODULE, never a file, so there is no `.File{j}` + // spelling to mint: the target is the module and the answer is its whole + // file list. The misses are the frameworks that ship with the platform and + // the SPM packages that live in `.build/`, i.e. outside the corpus. + return local + ? `Mod${d}` + : (r >>> 3) % 2 === 0 + ? ['Foundation', 'UIKit', 'Combine', 'SwiftUI'][(r >>> 4) % 4] + : `ExternalPkg${(r >>> 4) % 97}`; + } + if (lang === 'rust') { + // `crate::mod{d}::thing` resolves by PROBING: `src/mod{d}/thing.rs`, + // `src/mod{d}/thing/mod.rs`, `src/mod{d}.rs`, then `src/mod{d}/mod.rs`, + // which hits. The `d % 7` slice has no `mod.rs` at that path and misses, + // which is where the resolved count comes from. + return local + ? `crate::mod${d}::thing` + : (r >>> 3) % 2 === 0 + ? ['std::collections::HashMap', 'tokio::sync::mpsc', 'serde::Deserialize'][(r >>> 4) % 3] + : `ghost${(r >>> 4) % 97}::Missing`; + } + if (lang === 'python') { + // Dotted absolute imports. The stdlib spellings and the unknown + // distributions both die at `hasRepoCandidate`, which is the gate that + // keeps `django.apps` off a local `accounts/apps.py`. + return local + ? `pkg${d}.file${j}` + : (r >>> 3) % 2 === 0 + ? ['os.path', 'collections.abc', 'django.db.models'][(r >>> 4) % 3] + : `vendor${(r >>> 4) % 97}.deep.missing`; + } + if (lang === 'javascript' || lang === 'typescript') { + // BARE specifiers, not relative ones. A relative import resolves by exact + // `Set.has` and never reaches `suffixResolve` — the leg that had no index + // for JavaScript until PR #2911 and cost 25 972 µs per import at 8000 + // files — so a corpus of `./sibling` imports would measure the wrong one. + return local + ? `src/mod${d}/file${j}` + : (r >>> 3) % 2 === 0 + ? ['react', 'lodash/fp', '@scope/ui/dist/index'][(r >>> 4) % 3] + : `vendor${(r >>> 4) % 97}/lib/missing`; + } + if (lang === 'vue') { + // Every in-repo import is `@/…`, so the alias branch runs on all of them. + // The `.vue` share carries its extension (SFC imports always do) and takes + // the exact-path leg; the `.ts` share omits it and takes the guessing leg. + return local + ? j % 3 === 0 + ? `@/mod${d}/File${j}` + : `@/mod${d}/File${j}.vue` + : (r >>> 3) % 2 === 0 + ? ['vue', 'pinia', '@vueuse/core'][(r >>> 4) % 3] + : `vendor${(r >>> 4) % 97}/lib/Missing.vue`; + } + if (lang === 'c' || lang === 'cpp') { + // `#include "comp{d}/file{j}.h"`. `j | 1` picks the HEADER half of the + // corpus — the even half is `.c`/`.cpp` and nothing includes those. The + // misses are the two kinds a real tree has: a system header that is not in + // the repo at all, and a vendored path that does not exist. + const h = HEADER_EXTENSION[lang]; + const jj = j | 1; + return local + ? `comp${jj % dirs}/file${jj}${h}` + : (r >>> 3) % 2 === 0 + ? ['stdio.h', 'stdlib.h', 'string.h'][(r >>> 4) % 3] + : `vendor${(r >>> 4) % 97}/missing${h}`; + } + if (lang === 'ruby') { + return local + ? `mod${d}/file${j}` + : (r >>> 3) % 2 === 0 + ? ['json', 'set', 'net/http', 'digest'][(r >>> 4) % 4] + : `gem${(r >>> 4) % 97}/missing/thing`; + } + throw unwiredLanguage('uniqueTarget', lang); } /** @@ -364,6 +1212,30 @@ function collideTarget(lang, { local, r, d, j, dirs }) { ? ['System', 'System.Threading.Tasks', 'System.Collections.Generic'][(r >>> 4) % 3] : `Ghost${(r >>> 4) % 97}.Deep.Missing`; } + if (lang === 'csharp_csproj') { + // Same five families as the unique arm, in the same proportions, so the + // resolved count is identical by construction (asserted). Two things change. + // + // The local spelling moves onto the SECOND config: the collide layout puts + // nothing under `src/`, so `projectDir: 'src'` addresses no directory here + // and `App.Src{d}.Models` would resolve nothing. `Lib` (`projectDir: ''`) + // addresses `Src{d}/Models` directly — the same relayout-not-reworkload + // substitution every other language makes in this function. + // + // And `dirsByLastSegment` collapses from one key per directory to the + // single key `Models`, which makes the slash-free SWEEP cheaper here than + // on the unique layout while making the bucket the nested slice walks hold + // every directory — the inverse of the go/csharp/dart collide arms, whose + // every term gets worse. See `_arms_note`. + if (local) return `Lib.Src${d}.Models`; + const leg = (r >>> 3) % 5; + if (leg < 2) return `App.Missing${(r >>> 4) % 97}`; + if (leg === 2) return `Lib.Missing${(r >>> 4) % 97}`; + if (leg === 3) return 'Lib'; + return (r >>> 4) % 2 === 0 + ? ['System', 'System.Threading.Tasks', 'System.Collections.Generic'][(r >>> 5) % 3] + : `Ghost${(r >>> 4) % 97}.Deep.Missing`; + } if (lang === 'dart') { return local ? `package:app/pkg${j % dirs}/lib/src/mod${Math.floor(j / dirs)}.dart` @@ -388,11 +1260,144 @@ function collideTarget(lang, { local, r, d, j, dirs }) { ] : `com.ghost${(r >>> 4) % 97}.deep.Missing`; } - return local - ? `svc${j % dirs}/lib/models/mod${Math.floor(j / dirs)}` - : (r >>> 3) % 2 === 0 - ? ['json', 'set', 'net/http', 'digest'][(r >>> 4) % 4] - : `gem${(r >>> 4) % 97}/missing/thing`; + if (lang === 'php') { + // `Models\Mod{n}` is carried by every service, so the segment-suffix key it + // resolves through holds one entry no matter how many files exist: PHP + // answers from keyed maps and is collision-IMMUNE, which is what this arm + // asserts. The local spelling still always resolves, as it does on the + // unique layout — PHP's cascade strips leading segments, so even the + // nested-same-name slice is reachable by a shorter suffix. + return local + ? `App\\Models\\Mod${Math.floor(j / dirs)}` + : (r >>> 3) % 2 === 0 + ? [ + 'Psr\\Log\\LoggerInterface', + 'Symfony\\Component\\Console\\Command', + 'Doctrine\\ORM\\EntityManager', + ][(r >>> 4) % 3] + : `Vendor${(r >>> 4) % 97}\\Ghost\\Missing`; + } + if (lang === 'java') { + // `com.svc{d}.model` matches no directory, but its LAST segment is the one + // every directory now ends in, so `firstFileDirectlyInPkgDir` walks the + // whole `model` bucket twice — at the direct match and again after the + // first strip — before the third strip finds `model` on its own. That walk + // is the non-constant term this arm exists to measure. `vendor` buckets to + // nothing, mirroring the unique arm's nested slice, which also misses. + return local + ? (r >>> 3) % 3 === 0 + ? d % 7 === 0 + ? `com.svc${d}.vendor.*` + : `com.svc${d}.model.*` + : `com.example.model.File${j}` + : (r >>> 3) % 2 === 0 + ? ['java.util.List', 'java.io.IOException', 'java.util.concurrent.ConcurrentHashMap'][ + (r >>> 4) % 3 + ] + : `com.google.common.vendor${(r >>> 4) % 97}.Missing`; + } + if (lang === 'cobol') { + // The repeated basename is COBOL's ONLY collision axis, and its index is a + // keyed map, so this arm asserts immunity. It also reaches the tier + // tie-break the unique arm cannot: `Mod{n}` now names both a `.cpy` and a + // `.cbl`, and the copybook must win regardless of Set-iteration order. + return local + ? (r >>> 3) % 3 === 0 + ? `mod${Math.floor(j / dirs)}` + : `Mod${Math.floor(j / dirs)}` + : (r >>> 3) % 2 === 0 + ? ['DFHAID', 'DFHBMSCA', 'SQLCA', 'CICSDEF'][(r >>> 4) % 4] + : `VENDOR${(r >>> 4) % 97}`; + } + if (lang === 'swift') { + // Four modules instead of `dirs` of them, so the bucket a hit returns holds + // fileCount/4 files and grows with the corpus. Same in-repo share, same + // resolved count; the only thing that changed is bucket cardinality. + return local + ? `Mod${d % SWIFT_COLLIDE_MODULES}` + : (r >>> 3) % 2 === 0 + ? ['Foundation', 'UIKit', 'Combine', 'SwiftUI'][(r >>> 4) % 4] + : `ExternalPkg${(r >>> 4) % 97}`; + } + if (lang === 'rust') { + // ~2x the `::` segments of the unique arm, in both the hits and the misses, + // because SEGMENT COUNT is the only axis this resolver's cost has. The + // `d % 7` slice names a module that exists nowhere, mirroring the unique + // arm's `inner` slice, so the resolved count is unchanged. The external + // spellings run the prefix-shortening loop in `resolveModulePath` to the + // end — two `.has()` probes per shortened prefix — which is the longest + // path through the function and the one worth an absolute ceiling. + return local + ? d % 7 === 0 + ? `crate::l0::l1::l2::l3::l4::vendor${d}::thing::Inner` + : `crate::l0::l1::l2::l3::l4::mod${d}::thing::Inner` + : (r >>> 3) % 2 === 0 + ? [ + 'std::collections::hash_map::HashMap', + 'tokio::sync::mpsc::channel', + 'serde::de::value::MapDeserializer', + ][(r >>> 4) % 3] + : `ghost${(r >>> 4) % 97}::deep::nested::more::Missing`; + } + if (lang === 'python') { + // A `models` package in every service and a repeated `mod{n}.py` inside it, + // so `byBasename` holds one entry per service for each stem and the + // fewest-segments-then-lexicographic tie-break in `resolveAbsoluteFromFiles` + // actually has something to break. The external spelling shares the + // basename and still misses — `vendor{n}` fails `hasRepoCandidate`. + return local + ? `svc${j % dirs}.models.mod${Math.floor(j / dirs)}` + : (r >>> 3) % 2 === 0 + ? ['os.path', 'collections.abc', 'django.db.models'][(r >>> 4) % 3] + : `vendor${(r >>> 4) % 97}.models.mod0`; + } + if (lang === 'javascript' || lang === 'typescript') { + // `pkg{n}/src/mod{m}` in every package. `buildSuffixIndex` is a KEYED map + // that keeps one path per suffix, so this is the arm that asserts the + // ts-family resolver is collision-immune. The external spelling must not + // share the repeated stem, or it would suffix-match a real file and the + // corpus would stop being miss-heavy (measured: 67% resolved instead of + // 36% when it was `vendor{n}/src/mod{m}`). + return local + ? `pkg${j % dirs}/src/mod${Math.floor(j / dirs)}` + : (r >>> 3) % 2 === 0 + ? ['react', 'lodash/fp', '@scope/ui/dist/index'][(r >>> 4) % 3] + : `vendor${(r >>> 4) % 97}/src/ghost${(r >>> 4) % 8}`; + } + if (lang === 'vue') { + return local + ? j % 3 === 0 + ? `@/pkg${j % dirs}/components/Mod${Math.floor(j / dirs)}` + : `@/pkg${j % dirs}/components/Mod${Math.floor(j / dirs)}.vue` + : (r >>> 3) % 2 === 0 + ? ['vue', 'pinia', '@vueuse/core'][(r >>> 4) % 3] + : `vendor${(r >>> 4) % 97}/components/Ghost${(r >>> 4) % 8}.vue`; + } + if (lang === 'c' || lang === 'cpp') { + // A `mod{n}` header in every service's `include/`, which is what a C tree + // looks like. The basename bucket the suffix fallback walks now holds one + // candidate per service, so the depth-then-lexicographic tie-break decides + // — and the bucket grows with the corpus, which is why this arm carries its + // own scaling budget. + const h = HEADER_EXTENSION[lang]; + const jj = j | 1; + return local + ? `include/mod${Math.floor(jj / dirs)}${h}` + : (r >>> 3) % 2 === 0 + ? ['stdio.h', 'stdlib.h', 'string.h'][(r >>> 4) % 3] + : `vendor${(r >>> 4) % 97}/mod0${h}`; + } + if (lang === 'ruby') { + // `models/mod{n}.rb` in every package. Ruby answers `require` from a keyed + // suffix map, so the repeated basename cannot grow a bucket: this arm + // asserts that immunity, which is why its collide budget is the linear one. + return local + ? `svc${j % dirs}/lib/models/mod${Math.floor(j / dirs)}` + : (r >>> 3) % 2 === 0 + ? ['json', 'set', 'net/http', 'digest'][(r >>> 4) % 4] + : `gem${(r >>> 4) % 97}/missing/thing`; + } + throw unwiredLanguage('collideTarget', lang); } /** @@ -420,20 +1425,86 @@ function buildRepo(lang, fileCount, pad = 0, shape = 'unique') { return { files, imports }; } -/** The timed loop. A FRESH Set per pass, so every pass pays exactly one index - * build — reusing one Set across passes would hide the build after the first - * and let a rebuilt-per-import index look free from rep 2 onward. */ +/** + * The per-pass state one resolver sees: the file set it is handed, and the + * `resolutionConfig` the orchestrator threads beside it. + * + * `allFilePaths` is a FRESH Set per pass on purpose — every per-file-set memo + * in `import-resolvers/per-file-set.ts` is keyed on that object's identity, so + * reusing one Set across passes would hide the index build after the first and + * let a rebuilt-per-import index look free from rep 2 onward. + * + * `config` is why this exists as a function rather than a `new Set(files)` at + * three call sites. Three languages here need one and they need three different + * things: + * + * - C and C++ take their HEADERS through `resolutionConfig`, not through + * `allFilePaths`. The phase hands the C resolver the `.c` files it + * classified and the header scan separately, and + * `augmentedFilePathsFor(allFilePaths)(headerPaths)` unions the two ONCE per + * pass — a two-input memo, so both inputs have to be pass-stable or it + * rebuilds an O(files) Set per include. Splitting the corpus here is what + * makes that union reachable at all; handing the resolver one pre-merged set + * would leave the memo, and the shape it exists for, unmeasured. + * - Vue takes `tsconfigPaths`, and the alias branch is the one leg of the + * shared ts-family resolver its arm covers that the other two do not. + * + * `csharp_csproj` is the precedent and stays where it is: a per-language + * CONTEXT over a corpus aliased to another language's, rather than a new axis. + * + * `parsedFiles` is the third pass-stable object, present for `CONTEXT_LANGS` + * and undefined for everyone else. It is built BEFORE the path set and the path + * set is derived FROM it, which is not a stylistic choice: `run.ts` does + * `new Set(parsedFiles.map((f) => f.filePath))`, so two independently built + * lists would be a shape the pipeline cannot produce. Fresh per pass for + * exactly the reason the Set is — `filesByDirectory` and `parsedFileByPath` are + * `perFileSet` memos keyed on this ARRAY's identity, so reusing one array would + * hide their build from rep 2 onward and `fastest()` reports the minimum. + */ +function newPass(lang, files) { + if (HEADER_EXTENSION[lang] !== undefined) { + const sources = []; + const headers = []; + for (const f of files) (f.endsWith(HEADER_EXTENSION[lang]) ? headers : sources).push(f); + return { allFilePaths: new Set(sources), config: new Set(headers) }; + } + if (lang === 'vue') return { allFilePaths: new Set(files), config: VUE_TSCONFIG }; + if (CONTEXT_LANGS.includes(lang)) { + const parsedFiles = buildParsedFiles(lang, files); + return { + allFilePaths: new Set(parsedFiles.map((f) => f.filePath)), + config: undefined, + parsedFiles, + }; + } + return { allFilePaths: new Set(files), config: undefined }; +} + +/** + * The `{ parsedFiles, parsedImport }` object `run.ts` mints per import — per + * import there too, so this allocation is production's, not the bench's. + * + * `undefined` when the pass carries no parsed workspace, which happens in + * exactly one place: the CONTROL half of the `context` arm, whose whole job is + * to prove the arm can tell the two call shapes apart. + */ +const contextFor = (pass, parsedImport) => + pass.parsedFiles === undefined ? undefined : { parsedFiles: pass.parsedFiles, parsedImport }; + +/** The timed loop. One `newPass` per pass, so every pass pays exactly one index + * build — see `newPass`. */ function resolveAll(lang, files, imports) { - const allFilePaths = new Set(files); + const pass = newPass(lang, files); let sink = 0; for (const [from, target] of imports) { - const hit = resolveOne(lang, from, target, allFilePaths); + const hit = resolveOne(lang, from, target, pass); if (hit !== null) sink++; } return sink; } -function resolveOne(lang, from, target, allFilePaths) { +function resolveOne(lang, from, target, pass) { + const allFilePaths = pass.allFilePaths; if (lang === 'go') return resolveGoImportTarget(target, from, allFilePaths, GO_MODULE); if (lang === 'dart') return resolveDartImportTarget(target, from, allFilePaths); if (lang === 'ruby') return resolveRubyImportTarget(target, from, allFilePaths); @@ -443,10 +1514,107 @@ function resolveOne(lang, from, target, allFilePaths) { { fromFile: from, allFilePaths }, ); } - return resolveCsharpImportTarget( - { kind: 'namespace', localName: '_', importedName: '_', targetRaw: target }, - { fromFile: from, allFilePaths }, - ); + // `pass.config` is undefined for PHP, so no composer.json: the PSR-4 mapping + // legs are skipped and every import lands on the suffix cascade #2901 + // indexed. The FIFTH argument is the production one, and + // `importedSymbolKind: 'function'` is what opens the named/alias leg over + // `filesByDirectory(context.parsedFiles)` — see THE FIFTH ARGUMENT. It runs + // on every import rather than on a share of them because the leg is the point + // of the arm and it costs the cascade nothing: `resolvePhpImportInternal` + // has already returned by the time the leg is consulted, so this arm still + // measures everything it measured before, plus the leg. + // + // `importedName` is inert here and stays 'X' like the java and kotlin arms: + // the leg derives the name it matches on from `targetRaw` itself, so + // computing a real one would be a split per import charged to the timed loop + // for a field nothing reads. + if (lang === 'php') { + return resolvePhpImportTargetInternal( + target, + from, + allFilePaths, + pass.config, + contextFor(pass, { + kind: 'named', + localName: 'X', + importedName: 'X', + targetRaw: target, + importedSymbolKind: 'function', + }), + ); + } + if (lang === 'java') { + return resolveJavaImportTarget( + { kind: 'named', localName: 'X', importedName: 'X', targetRaw: target }, + { fromFile: from, allFilePaths }, + ); + } + // The `ScopeResolver` hook itself — COBOL's copy index has no other export. + if (lang === 'cobol') return cobolScopeResolver.resolveImportTarget(target, from, allFilePaths); + if (lang === 'swift') { + return resolveSwiftImportTarget( + { kind: 'namespace', localName: 'X', importedName: 'X', targetRaw: target }, + { fromFile: from, allFilePaths }, + ); + } + if (lang === 'rust') return resolveRustImportTarget(target, from, allFilePaths, undefined); + if (lang === 'python') { + // `from import X` — the spelling the orchestrator actually hands + // the provider, and the ONLY one that reads `context.parsedFiles`: a + // `namespace` import makes `pythonImportedSubmoduleTarget` return null, the + // submodule-precedence branch never runs and the field is dead. This arm + // used to pass that synthetic namespace spelling and skipped the branch + // for exactly that reason, which is what made the field unmeasurable. + // + // So the arm now pays the branch: a package probe, a `parsedFileByPath` + // lookup over the resolved package's `localDefs`, and a submodule probe — + // up to three entries into the resolver per import, which is why its ms + // numbers are several times what the namespace spelling read. That IS the + // per-import cost of a `from … import …` in production. + // + // 'X' names nothing the corpus declares, on purpose: `pythonFileExportsName` + // then scans the whole `localDefs` list and returns false, so every + // resolving import runs the submodule probe too. That is the expensive + // half of the branch — a name the package DOES export short-circuits at + // the first def — and matches corpus property 1 above. + // + // The `{ fromFile, allFilePaths, parsedFiles }` shape is exactly what + // `pythonScopeResolver` builds from the context before calling this. + return resolvePythonImportTarget( + { kind: 'named', localName: 'X', importedName: 'X', targetRaw: target }, + { fromFile: from, allFilePaths, parsedFiles: pass.parsedFiles }, + ); + } + if (lang === 'javascript') return jsResolveImportTarget(target, from, allFilePaths); + if (lang === 'vue') return vueResolveImportTarget(target, from, allFilePaths, pass.config); + // TypeScript, C and C++ go through the registered `ScopeResolver` hook rather + // than an inner resolver, because for all three the thing under test lives IN + // the adapter: TypeScript's `tsPassCacheFor` memo is private to + // `typescript/scope-resolver.ts`, and C's and C++'s `augmentedFilePathsFor` + // is private to theirs. Calling past it would benchmark a copy of the adapter + // instead of the adapter. + if (lang === 'typescript') { + return typescriptScopeResolver.resolveImportTarget(target, from, allFilePaths, undefined); + } + if (lang === 'c') { + return cScopeResolver.resolveImportTarget(target, from, allFilePaths, pass.config); + } + if (lang === 'cpp') { + return cppScopeResolver.resolveImportTarget(target, from, allFilePaths, pass.config); + } + if (lang === 'csharp' || lang === 'csharp_csproj') { + return resolveCsharpImportTarget( + { kind: 'namespace', localName: '_', importedName: '_', targetRaw: target }, + { + fromFile: from, + allFilePaths, + // The ONLY difference between the two C# arms. Present, the adapter + // takes the csproj branch and never falls through to the no-csproj legs. + ...(lang === 'csharp_csproj' ? { csharpConfigs: CSPROJ_CONFIGS } : {}), + }, + ); + } + throw unwiredLanguage('resolveOne', lang); } /** The single untimed identity pass, producing BOTH non-timing results: the @@ -461,7 +1629,7 @@ function resolveOne(lang, from, target, allFilePaths) { * that makes this pass cheap is exactly what would hide the cost that loop * exists to measure. */ function identityPass(lang, files, imports) { - const allFilePaths = new Set(files); + const pass = newPass(lang, files); const outcomes = new Set(); const wasNullByKey = new Map(); let resolved = 0; @@ -472,16 +1640,26 @@ function identityPass(lang, files, imports) { if (!wasNull) resolved++; continue; } - const hit = resolveOne(lang, from, target, allFilePaths); + const hit = resolveOne(lang, from, target, pass); wasNull = hit === null; wasNullByKey.set(key, wasNull); if (!wasNull) resolved++; - const rendered = wasNull ? '' : Array.isArray(hit) ? hit.join(',') : hit; + const rendered = renderResolved(hit); outcomes.add(`${key}\u0000${rendered}`); } return { outcomes, resolved }; } +/** One resolver answer as a comparable string. Kotlin and Java can return a + * LIST (their wildcard tier), so the array form is part of the shape, and + * `` keeps a miss distinct from a resolver that answered the empty + * string. Shared by the fingerprint above and by the `context` arm below, so + * the two never drift into reporting one answer two ways. */ +function renderResolved(hit) { + if (hit === null) return ''; + return Array.isArray(hit) ? hit.join(',') : hit; +} + /** MIN, not median: both scales are timed in one process and every error source * (GC, scheduler preemption, a noisy CI neighbour) is additive, so the fastest * observed pass is the closest estimate of the uncontended cost. */ @@ -489,10 +1667,10 @@ function fastest(values) { return Math.min(...values); } -function timeResolution(lang, files, imports) { +function timeResolution(lang, files, imports, reps) { for (let w = 0; w < WARMUP; w++) resolveAll(lang, files, imports); const samples = []; - for (let r = 0; r < REPS; r++) { + for (let r = 0; r < reps; r++) { const t0 = performance.now(); resolveAll(lang, files, imports); samples.push(performance.now() - t0); @@ -501,44 +1679,219 @@ function timeResolution(lang, files, imports) { } /** - * Retained JS heap of the `WorkspaceFileIndex` built over `files`. + * One WARMED pass, used only to size `reps` for the language. * - * GROWTH form, not the release form `bench/cfg/measure.mjs` uses: the index is - * memoized in a `WeakMap` keyed on the Set, so releasing it means releasing the - * Set too, which would fold the Set's own cost into the delta. Here the Set is - * live across BOTH reads and the `files` array holds the path strings, so the - * delta is the index's own footprint — the arrays, the three `buildSuffixIndex` - * maps and `normToRaw` — and not the paths they point at. A forced double GC - * before each read makes it robust to pre-existing garbage the same way. + * Run on the `small` arm, and `small` is measurably the cheapest of the five + * for every language where the answer can differ — all six that come out below + * `REPS_MAX` (csharp_csproj, ruby, php, javascript, typescript, vue). Three + * languages do have a cheaper arm — cobol's `collide` by 45%, kotlin's by 7%, + * dart's `deep` by a few percent — and all three sit so far under + * `REPS_CHEAP_MS` that either reading returns 15. `small` is also the arm + * `small_ms_ceiling` bounds, so it is the one number here a reader already has + * an intuition for. + * + * Warmed rather than taken from the WARMUP passes themselves: an unwarmed pass + * reads several times high, which would push the expensive languages to + * `REPS_MIN` for the wrong reason. */ -function retainedIndexBytes(files) { - const set = new Set(files); +function probeMs(lang, files, imports) { + for (let w = 0; w < WARMUP; w++) resolveAll(lang, files, imports); + const t0 = performance.now(); + resolveAll(lang, files, imports); + return performance.now() - t0; +} + +/** + * Retained JS heap of everything one language derives from one file set — + * measured by RESOLVING AN IMPORT through it, never by calling a builder. + * + * THE ARM READS WHAT THE LANGUAGE READS, and that is now the whole design. + * Until #2903 was extended to the two suffix maps, four of these arms called + * `getWorkspaceFileIndex(set)` directly and read `index.all.length`, which asks + * no suffix question at all. That was harmless only while `buildSuffixIndex` + * built both maps eagerly. The moment they went lazy the direct call built NO + * map, all four arms reported 0 B at 32 000 files, and 0 B is under every + * ceiling — four gates silently became ceilings over nothing, which is exactly + * the failure this file's header warns about for rust and cobol. Driving the + * real resolver cannot fail that way: whatever maps the language forces are the + * maps it forces in production, and if a resolver starts asking a new question + * the number moves on its own instead of needing this file edited. + * + * It also happens to be the only form available for half of these languages — + * Swift's `getSwiftModuleIndex`, Python's `getPythonFileIndex`, C's + * `suffixIndex` and the ts-family `passCacheFor` are private to their modules, + * and exporting four builders to feed a bench would widen four module surfaces + * for a measurement's convenience. Now that all eight arms use one form, the + * readings ARE comparable to one another (they were not before). + * + * GROWTH form, not the release form `bench/cfg/measure.mjs` uses: every index + * here is memoized in a `WeakMap` keyed on the Set, so releasing it means + * releasing the Set too, which would fold the Set's own cost into the delta. + * Here the pass is live across BOTH samples and the `files` array holds the + * path strings, so the delta is the derived structures' own footprint and not + * the paths they point at. For C and C++ it legitimately includes the augmented + * Set, which is part of what they hold; for every language it includes the one + * or two resolve-cache entries the probe leaves behind. + */ +function retainedPassBytes(lang, files, probeTarget) { + const pass = newPass(lang, files); + // See `HEAP_RETAINED`: nothing built for this language is released until the + // next one starts, so no deferred collection can land between the two samples + // below and cancel part of the delta. + HEAP_RETAINED.push(pass); GC(); const before = process.memoryUsage().heapUsed; - const index = getWorkspaceFileIndex(set); + const hit = resolveOne(lang, files[0], probeTarget, pass); GC(); const after = process.memoryUsage().heapUsed; - // Keeps both live past the second read, and fails loudly if the corpus ever - // stops being one distinct path per file (which would silently shrink it). - if (index.all.length !== files.length || set.size !== files.length) { - throw new Error(`heap arm corpus is not distinct: ${set.size} of ${files.length}`); + // A HIT would mean the reading is a materialized answer rather than the + // index, and — for the languages whose cascade returns early — that the legs + // past the hit were never reached and their structures never built. + if (hit !== null) { + throw new Error(`heap probe '${probeTarget}' resolved for ${lang}; it must MISS: ${hit}`); + } + // Fails loudly if the corpus ever stops being one distinct path per file, + // which would silently shrink every reading here. + const size = pass.allFilePaths.size + (pass.config instanceof Set ? pass.config.size : 0); + if (size !== files.length) { + throw new Error(`heap arm corpus is not distinct: ${size} of ${files.length}`); } return Math.max(0, after - before); } +/** + * Every pass this arm builds, held alive ON PURPOSE until the next language + * starts. + * + * A `heapUsed` delta is only the new structures if nothing OLD is released + * between its two samples, and that is not a property a forced GC can be + * trusted to establish: measured, the previous read's index survived a + * two-cycle collect at the next read's baseline and was dropped by the collect + * before its second sample, so the two cancelled and the arm reported 249 200 B + * for a 9.3 MB index (PHP) and 329 064 B for a 6.7 MB one (JavaScript, once, + * non-reproducibly — the same defect with a different language's timing). + * + * Holding the passes removes the precondition instead of tuning it: nothing a + * measurement window depends on is ever collectable inside it, so the delta + * cannot absorb a late free no matter how many cycles the collector needs. + * Byte-identical readings at two and at four `gc()` cycles are the evidence + * that it works, where without it the two disagree by 9 MB. + * + * Emptied once per language, in `measureHeap`, which is the one place a late + * free is harmless: it happens before that language's first baseline and + * outside both of its measurement windows, and it is followed by a drain deeper + * than any chain here has needed. Never emptying at all also works and is what + * this was first measured with, but it peaks at ~380 MB and costs 4.5 s, + * because every forced collection from that point on has to mark it. + */ +const HEAP_RETAINED = []; + +/** + * The import each heap language resolves to force its build. A MISS in every + * case (asserted above), so the reading is the index and not a materialized + * answer, and so the cascade runs to completion instead of returning at the + * first leg. + * + * Each spelling is one the language's own corpus already mints in + * `uniqueTarget`, so the arm forces the same read pattern the timing arms do — + * which after #2903 is what decides the number: + * + * - `csharp` and `java` ask `index.get` and never `getInsensitive`, so the + * case-folded map is never built (49.6% of the eager Java index was dead); + * - `php` asks `getInsensitive` and never `get` (49.4% dead), and builds its + * own first-proper-suffix map on top; + * - `ruby` and the ts family read `get(s) || getInsensitive(s)`, so they pay + * for both — the second one DERIVED from the first, which is why they cost + * less than two independent traversals; + * - `csharp_csproj` additionally asks `getFilesInDir`, forcing the `dirMap` + * #2903 made lazy. It is the witness that the read pattern IS the + * footprint: same corpus and same `getWorkspaceFileIndex` as `csharp`, + * three times the retained bytes. + */ +const HEAP_PROBE_TARGET = { + csharp: 'Ghost0.Deep.Missing', + // Matches the `App` root namespace and no directory, so it runs the config + // loop's single-file leg (`get` + `getInsensitive`) AND its directory leg + // (`getFilesInDir`) before answering null — the three-map read pattern. + csharp_csproj: 'App.Missing0', + ruby: 'gem0/missing/thing', + php: 'Vendor0\\Ghost\\Missing', + java: 'com.google.common.vendor0.Missing', + javascript: 'vendor0/lib/missing', + python: 'vendor0.deep.missing', + c: 'vendor0/missing.h', + // The nine below are the BOUNDED tier — see `HEAP_BOUNDED`. Same rule as the + // eight above: a spelling `uniqueTarget` already mints for that language, and + // one that MISSES, so the reading is the index and the cascade runs to the + // end. Chosen from the miss family that reaches furthest into each cascade: + // - `go` takes the GOPATH fallback, one `filesDirectlyInPkgDir` per path + // segment, which is the leg that forces `PackageDirIndex`; + // - `dart` is an external package, so BOTH candidate paths miss and both + // walk the basename bucket to completion; + // - `kotlin` misses in `suffixByStem`, the map its four-tier cascade builds; + // - `cobol` misses in both tier maps, `swift` in `byModule`, and `rust` + // probes candidate paths and builds nothing — that last is the reading + // the exclusion rests on; + // - `typescript`, `vue` and `cpp` carry the same spelling shape as the + // `javascript` and `c` arms they are excluded as duplicates OF, so the + // bound compares like with like. `vue`'s is bare rather than `@/…` + // because the alias branch rewrites to `src/` and would resolve. + go: 'github.com/org/repo0/pkg/util', + dart: 'package:ext0/src/thing.dart', + kotlin: 'com.ghost0.deep.Missing', + cobol: 'VENDOR0', + swift: 'ExternalPkg0', + rust: 'ghost0::Missing', + typescript: 'vendor0/lib/missing', + vue: 'vendor0/lib/Missing.vue', + cpp: 'vendor0/missing.hpp', +}; + +/** + * `buildFiles` mints every path with a template literal, and V8 represents + * those as ROPES — the concatenation is not materialized until something forces + * it. The first traversal that slices a path (`lastIndexOf('/')`, `toLowerCase`, + * every index builder here) flattens it, which allocates the flat string AND + * drops the rope's now-unreachable pieces, so a build measured over an + * unflattened corpus reports the index MINUS that net release: measured 11% + * low, uniformly, on every language whose index slices paths. + * + * It biased the arm in the one direction that matters. `bytes_small` was read + * over a corpus a discarded warm-up pass had already flattened and + * `bytes_large` over a fresh one, so every `ratio` here was ~0.85-0.89 for + * structures that are exactly linear in the file count — the ratio budget was + * bounding an artefact. Flattened first, all eight read 0.99-1.02. + * + * It also retires the warm-up pass, which was never about JIT: with the corpus + * flat, a language's first and second reads of the same file count agree to + * within 0.3%. + */ +function flatten(files) { + for (const file of files) file.lastIndexOf('/'); + return files; +} + function measureHeap(lang) { if (GC === null) return null; - const small = buildFiles(lang, HEAP_SMALL, HEAP_PAD, 'unique'); - // The first build in a fresh process reads a few percent low (lazily grown - // spaces, unJITted build loop); discard it. - retainedIndexBytes(small); - const bytesSmall = retainedIndexBytes(small); - const large = buildFiles(lang, HEAP_LARGE, HEAP_PAD, 'unique'); - const bytesLarge = retainedIndexBytes(large); + // Release the PREVIOUS language's passes here and nowhere else, then drain + // them twice over. This is the one point at which a deferred collection is + // free: it is before this language's first baseline and outside both of its + // measurement windows, so however many cycles the release needs, it cannot + // land between a `before` and an `after`. + HEAP_RETAINED.length = 0; + GC(); + GC(); + const probe = HEAP_PROBE_TARGET[lang]; + const read = (files) => retainedPassBytes(lang, files, probe); + const small = flatten(buildFiles(lang, HEAP_SMALL, HEAP_PAD, 'unique')); + const bytesSmall = read(small); + const large = flatten(buildFiles(lang, HEAP_LARGE, HEAP_PAD, 'unique')); + const bytesLarge = read(large); return { files_small: HEAP_SMALL, files_large: HEAP_LARGE, path_segments: small[0].split('/').length, + probe, bytes_small: bytesSmall, bytes_large: bytesLarge, mib_large: Number((bytesLarge / 1024 / 1024).toFixed(2)), @@ -546,6 +1899,88 @@ function measureHeap(lang) { }; } +/** + * The `context` arm's corpora — one per `CONTEXT_LANGS` entry, each a handful + * of files carrying ONE import whose answer DIFFERS between the production + * five-argument call and the three-argument one this harness used to make. + * + * That difference is the whole arm. The main corpus cannot serve as one: there + * the leg AGREES with the cascade for every import (measured — both languages' + * ten fingerprints are unchanged by threading the context), which is the right + * outcome for a corpus built to measure cost, and useless for proving the + * context arrives. Timing cannot prove it either; a dropped context makes the + * arms FASTER, and nothing here has a lower bound on ms. + * + * Both are resolved THROUGH `resolveOne`, not through the resolvers directly, + * because what is under test is this file's threading rather than the + * resolvers' behaviour. The control differs in exactly one thing: + * `pass.parsedFiles` is undefined, which `contextFor` turns into no fifth + * argument at all. + */ +const CONTEXT_PROBE = { + /** + * `use function App\Ns0\Dup;` where the CLASS `Dup` lives in `Dup.php` and + * the FUNCTION `Dup` lives in `Helpers.php`. PHP keeps the two in separate + * symbol tables and PSR-4 maps only the class, which is the case the leg + * exists for: the suffix cascade answers the file whose NAME matches the last + * segment, the leg answers the file that DECLARES the function. Two distinct + * non-null paths, so neither half of the arm can be mistaken for a miss, and + * `Alpha.php` is a third file in the same directory so the candidate gather + * has something to reject. + */ + php: { + from: 'src/App/Ns0/Alpha.php', + target: 'App\\Ns0\\Dup', + parsedFiles: [ + probeFile('src/App/Ns0/Alpha.php', [['Class', 'App\\Ns0\\Alpha']]), + probeFile('src/App/Ns0/Dup.php', [['Class', 'App\\Ns0\\Dup']]), + probeFile('src/App/Ns0/Helpers.php', [['Function', 'App\\Ns0\\Dup']]), + ], + }, + /** + * `from pkg import X`, with `pkg/__init__.py` exporting `X` AND a same-named + * submodule `pkg/X.py` beside it — the precedence CPython documents and the + * one `pythonFileExportsName` exists to reproduce. With the parsed workspace + * the package's own export wins (`pkg/__init__.py`); without it the export is + * invisible, the submodule probe runs and `pkg/X.py` wins. + * + * `X` rather than a prettier name because `resolveOne` passes `importedName: + * 'X'`: the probe is tied to the spelling the timing arms use, so changing + * one without the other fails here. + * + * This corpus also catches a revert to the synthetic `namespace` spelling, + * which no exact-value assertion could: that spelling never reads + * `parsedFiles`, so BOTH halves answer `pkg/__init__.py` and the + * with/without inequality below is what notices. + */ + python: { + from: 'app/main.py', + target: 'pkg', + parsedFiles: [ + probeFile('pkg/__init__.py', [['Function', 'pkg.X']]), + probeFile('pkg/X.py', [['Function', 'pkg.X.run']]), + probeFile('app/main.py', [['Function', 'app.main.run']]), + ], + }, +}; + +/** Resolve the probe twice through `resolveOne` — once with the pass's parsed + * workspace, once without — and report both answers. Deterministic and + * microseconds, so it runs in report mode too. */ +function measureContext(lang) { + const { from, target, parsedFiles } = CONTEXT_PROBE[lang]; + const allFilePaths = new Set(parsedFiles.map((f) => f.filePath)); + const answer = (files) => + renderResolved( + resolveOne(lang, from, target, { allFilePaths, config: undefined, parsedFiles: files }), + ); + return { + target, + with_context: answer(parsedFiles), + without_context: answer(undefined), + }; +} + function fingerprint(outcomes) { return crypto .createHash('sha256') @@ -566,7 +2001,72 @@ if (CHECK && GC === null) { process.exit(1); } -const LANGS = ['go', 'csharp', 'dart', 'ruby', 'kotlin']; +/** + * Every arm, and the registered language each one exercises. + * + * This used to be a hand-written list of seventeen strings under a comment + * claiming it was "every language in `SCOPE_RESOLVERS`" — a claim nothing in + * the file could check, because the file never imported the registry. Adding a + * resolver to `pipeline/registry.ts` is two lines, neither of which is this + * one, so a seventeenth registered language would have shipped ungated and + * printed PASS. That is not a hypothetical failure mode: JavaScript reached + * `suffixResolve` with no index at all and measured 25 972 µs per import at + * 8000 files (PR #2911) for exactly as long as nothing gated it. + * + * So the list is DERIVED and the claim is ASSERTED. `LANGS` is this table's + * keys, and the `--check` inventory arm below fails when a registered resolver + * has no arm here (or an arm names a language the registry does not have) — + * the same shape `test/unit/scope-resolution/import-target-index-reuse.contract.test.ts` + * uses ten files away, and the same "one row per language" table + * `bench/cfg/measure.mjs` keeps. + * + * The mapping is many-to-one on purpose: `csharp` and `csharp_csproj` are two + * arms over one registered resolver, differing only in whether `csharpConfigs` + * is supplied, because the no-csproj arm returns before it can reach the leg + * #2902 indexed. + */ +const LANG_REGISTRY = { + go: SupportedLanguages.Go, + csharp: SupportedLanguages.CSharp, + csharp_csproj: SupportedLanguages.CSharp, + dart: SupportedLanguages.Dart, + ruby: SupportedLanguages.Ruby, + kotlin: SupportedLanguages.Kotlin, + php: SupportedLanguages.PHP, + java: SupportedLanguages.Java, + cobol: SupportedLanguages.Cobol, + swift: SupportedLanguages.Swift, + rust: SupportedLanguages.Rust, + python: SupportedLanguages.Python, + javascript: SupportedLanguages.JavaScript, + typescript: SupportedLanguages.TypeScript, + vue: SupportedLanguages.Vue, + c: SupportedLanguages.C, + cpp: SupportedLanguages.CPlusPlus, +}; +const LANGS = Object.keys(LANG_REGISTRY); +/** + * The heap arm's SECOND tier: every arm that is not budgeted, and the reason it + * is a `filter` over `LANGS` rather than a second list beside `HEAP_BUDGETED`. + * + * The two tiers partition `LANGS` by construction, so there is no third state a + * language can be in — the state the nine spent this file's whole life in, + * where "not budgeted" and "not measured" were the same thing and neither was + * derived from anything. Adding a registered language now costs a bound whether + * or not anyone thinks about memory: the inventory arm gives it a `LANGS` row, + * this line gives it a tier, and the presence check below fails until it has a + * key. Deriving it also means the two tiers cannot overlap or leave a gap, which + * two hand-written lists could do in either direction. + * + * A bound and NOT a floor, deliberately, and the boundary is the one thing here + * worth re-reading before moving a language across it: a floor asserts "this + * arm is still measuring something", which is a claim about an index the file + * has budgeted, and rust's 16 B cannot carry it. What every one of the nine CAN + * carry is "the exclusion still holds" — that this language has not grown an + * index since it was left out. See the TIER TWO loop at the foot of the file, + * and `_heap_bound_note` in baselines.json for each language's reason. + */ +const HEAP_BOUNDED = LANGS.filter((lang) => !HEAP_BUDGETED.includes(lang)); /** name, file count, depth padding, directory/basename layout. */ const ARMS = [ ['small', SMALL, 0, 'unique'], @@ -582,9 +2082,14 @@ const SCALES = ARMS.map(([name]) => name); const report = {}; for (const lang of LANGS) { const scales = {}; + // Sized once per language, from the FIRST arm — `small`, the cheapest — so + // all five arms share one estimator and the four ratios below stay + // comparisons of like with like. See `repsFor`. + let reps = null; for (const [name, fileCount, pad, shape] of ARMS) { const { files, imports } = buildRepo(lang, fileCount, pad, shape); const { outcomes, resolved } = identityPass(lang, files, imports); + if (reps === null) reps = repsFor(probeMs(lang, files, imports)); scales[name] = { files: files.length, imports: imports.length, @@ -592,17 +2097,22 @@ for (const lang of LANGS) { // resolved share would still produce a "valid" fingerprint over far less. resolved, distinct_outcomes: outcomes.size, - ms: Number(timeResolution(lang, files, imports).toFixed(3)), + ms: Number(timeResolution(lang, files, imports, reps).toFixed(3)), fingerprint: fingerprint(outcomes), }; } report[lang] = { ...scales, + // Reported so a triager can see which estimator produced the five ms + // numbers above; environment-derived, so never asserted. + reps, scaling_ratio: Number((scales.large.ms / scales.small.ms / (LARGE / SMALL)).toFixed(3)), // `scaling_ratio` divides the file count out, so it is scale-invariant and // structurally cannot see a cost that grows with path DEPTH instead — and - // `buildSuffixIndex` (C#, Ruby) and Kotlin's `suffixByStem` both emit one - // entry per '/' in a path. Same file count, ~6x the components. + // `buildSuffixIndex` (C#, Ruby, PHP, Java, and the whole ts family) and + // Kotlin's `suffixByStem` all emit one entry per '/' in a path, and + // Python's ancestor walk rebuilds one prefix per component PER IMPORT. + // Same file count, ~6x the components. depth_ratio: Number((scales.deep.ms / scales.small.ms).toFixed(3)), // Same measurement on the shared-leaf layout. Legitimately above the 1.8 // budget for go/csharp/dart — see the scope-of-claim note in the header. @@ -613,11 +2123,24 @@ for (const lang of LANGS) { }; } -// AFTER every timing arm, never interleaved with them: the heap arm allocates a -// 32k-path corpus and a ~75 MiB index, and leaving that garbage behind for the -// next language's timed loop to collect would tax an arm it has nothing to do -// with. -for (const lang of HEAP_LANGS) report[lang].heap = measureHeap(lang); +// AFTER every timing arm, never interleaved with them, and now for a second +// reason as well as the first. The first: the heap arm allocates a 32k-path +// corpus and a ~70 MiB index per language, and leaving that behind for the next +// language's timed loop to collect would tax an arm it has nothing to do with. +// The second: `HEAP_RETAINED` holds a language's whole corpus and index alive +// across both of its reads — up to ~92 MiB for `csharp_csproj` — and that must +// not overlap a measurement of time. +// +// `LANGS`, not `HEAP_BUDGETED`: which tier a language is in decides its GATE, +// not whether it is read. Measured cost of the nine extra arms is 1.37 s — this +// phase goes 2.06 s -> 3.43 s, of which kotlin alone is 0.57 s. See COST. +for (const lang of LANGS) report[lang].heap = measureHeap(lang); + +// Deterministic and microseconds — it resolves six imports over two three-file +// corpora — so unlike the heap arm it neither needs nor deserves isolation from +// the timing phase. It runs last only because it reads best beside the heap arm +// in the report. +for (const lang of CONTEXT_LANGS) report[lang].context = measureContext(lang); if (!CHECK) { console.log(JSON.stringify(report, null, 2)); @@ -626,6 +2149,111 @@ if (!CHECK) { const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8')); const failures = []; + +/** + * PRESENCE, for one budget, in the one place that spells the reason. + * + * A missing budget is a DELETED GATE, not a passing arm: `got > undefined` is + * `false`, `ceiling * undefined` is `NaN` and `bytes < NaN` is `false`, so every + * comparison in this file answers "within budget" for every possible + * measurement the moment its key stops being a number. Each of the three call + * sites below is one deleted key away from a silent no-op, and the run still + * prints PASS. + * + * `Number.isFinite` rather than `typeof === 'number'`: over JSON input the two + * agree (JSON cannot express NaN or Infinity), and the stricter one is the one + * whose name says what the gate needs. + * + * The two per-site facts stay the caller's, because they are what a triager acts + * on: `reads` is the comparison that silently stopped gating, quoted, and + * `scope` is what deleting this one key actually costs — a single arm, or all + * eight at once. Only the shared framing and the shared trailing sentence live + * here. Returns the message rather than pushing it, so the timing loop can + * `continue` past a budget it must not then compare against. + */ +const requireNumericBudget = ({ key, value, reads, scope }) => + Number.isFinite(value) + ? null + : `no numeric ${key} in baselines.json — a missing budget is a DELETED GATE, not a passing ` + + `arm: the comparison it gates reads \`${reads}\`, which is false for every possible ` + + `measurement. ${scope} Deterministic: a re-run will not change it.`; + +/** + * The REVERSE direction of a reconciliation: every key declared in `label` that + * `codeList` does not name. + * + * The forward direction ("the code has an arm with no budget") is a presence + * check inside whichever loop iterates the code's list. This is the other way + * round — a budget, a baseline block or a registry row for an arm that is never + * measured — and no forward check can see it, because the thing it names is + * exactly the thing nothing iterates. + * + * `codeListName` and `why` stay the caller's: which list is authoritative and + * what the orphan costs are the two facts that differ between the three arms, + * and flattening them would leave a triager with a name and no reading of it. + */ +function expectNoOrphanKeys(label, declaredKeys, codeList, codeListName, why) { + for (const key of declaredKeys) { + if (codeList.includes(key)) continue; + failures.push( + `${label} has an entry for '${key}', which is not in ${codeListName} — ${why} ` + + `Deterministic: a re-run will not change it.`, + ); + } +} + +/** The corpus-shape facts asserted for one timing scale. */ +const SCALE_SHAPE = { + fields: ['files', 'imports', 'resolved', 'distinct_outcomes', 'fingerprint'], + why: + 'the corpus changed shape or the resolver changed its answer for this arm. Every scale is ' + + 'asserted separately: the arms differ only in padding and layout, so a defect that touches ' + + 'one of them alone moves nothing in the others.', +}; +/** The same, for the heap arm — the four inputs that decide what it measures. + * Asserted for all seventeen, budgeted tier and bounded tier alike, and it is + * the bounded tier that needs it most: a bound is a single comparison, so a + * probe swapped for one that reaches less is a bound over a smaller workload + * and there is no floor beside it to notice. + * `bytes_small`/`bytes_large` are deliberately NOT here: they are bounded by + * `heap_ceiling_bytes` and `heap_reading_bytes` with ~50% of slack either way + * (`heap_bound_bytes` with 50% on the one side), because a Node major or a + * different platform moves heapUsed accounting and an exact-equality arm on a + * byte count would be a re-baseline per runner. */ +const HEAP_SHAPE = { + fields: ['files_small', 'files_large', 'path_segments', 'probe'], + why: + 'these four decide WHAT the heap arm measures and nothing else here can see them move — a ' + + 'probe that stops reaching a leg, or two file counts collapsed onto one, leaves every ' + + 'ceiling, floor, bound and ratio passing over an arm that changed workload. Deterministic: ' + + 'a re-run will not change it.', +}; +/** The same, for the `context` arm. All three fields are exact strings, not + * bounds: this arm has no measurement noise at all — it resolves one import + * two ways over a three-file corpus — so anything less than equality would be + * slack for nothing. */ +const CONTEXT_SHAPE = { + fields: ['target', 'with_context', 'without_context'], + why: + 'the fifth `context` argument stopped reaching this resolver, reached it in a different ' + + 'shape, or the resolver changed what it does with it. `with_context` is what the five-argument ' + + 'call `run.ts` makes answers and `without_context` is what the three-argument one this bench ' + + 'used to make answers; both are pinned, so a change is attributed rather than guessed. ' + + 'Deterministic: a re-run will not change it.', +}; +/** Every asserted arm for one language, derived so a new scale is covered by + * construction. The heap arm is present for EVERY language now — it used to be + * conditional on `HEAP_LANGS`, which is what let the other nine be measured by + * nothing and pinned by nothing; the two tiers below decide which gate the + * reading gets. The context arm is still conditional, on `CONTEXT_LANGS`, which + * the registry-arity arm at the foot of the file pins to the hooks that DECLARE + * a fifth parameter. */ +const armShapes = (lang) => [ + ...SCALES.map((scale) => [scale, SCALE_SHAPE]), + ['heap', HEAP_SHAPE], + ...(CONTEXT_LANGS.includes(lang) ? [['context', CONTEXT_SHAPE]] : []), +]; + for (const lang of LANGS) { const got = report[lang]; const want = baseline.languages[lang]; @@ -633,52 +2261,77 @@ for (const lang of LANGS) { failures.push( `${lang}: fingerprint drift ${got.fingerprint} != ${want.fingerprint} — the resolver ` + `returned a DIFFERENT target set. That is a behaviour change, not a perf one; see the ` + - `parity harness in test/unit/scope-resolution/import-target-index-parity.test.ts.`, + `parity harnesses in test/unit/scope-resolution/*-import-target-parity.test.ts and the ` + + `all-languages adapter guard in import-target-index-reuse.contract.test.ts.`, ); } // One shape, five facts, so the five budgets read side by side and the shared // trailing sentence exists once instead of drifting into five wordings. Each // `why` stays the arm's OWN: it is what tells a triager which corpus shape // regressed, and flattening it would cost the message its whole value. + // `key` is the baselines.json path the budget came from, so the presence + // check below can name it. const timingChecks = [ { label: 'scaling', + key: 'scaling_budget', got: got.scaling_ratio, budget: baseline.scaling_budget, why: 'per-import cost grows with corpus size again.', }, { label: 'depth', + key: `depth_budget.${lang}`, got: got.depth_ratio, - budget: baseline.depth_budget[lang], + budget: baseline.depth_budget?.[lang], why: 'cost grows with path DEPTH at a fixed file count, which scaling_ratio divides out and ' + 'cannot see.', }, { label: 'collide scaling', + key: `collide_scaling_budget.${lang}`, got: got.collide_scaling_ratio, - budget: baseline.collide_scaling_budget[lang], + budget: baseline.collide_scaling_budget?.[lang], why: 'on the SHARED-LEAF layout (svcN/internal, SrcN/Models, a repeated basename per package) ' + 'per-import cost grew beyond what this shape already costs by construction.', }, { label: 'small arm ms', + key: `small_ms_ceiling.${lang}`, got: got.small.ms, - budget: baseline.small_ms_ceiling[lang], + budget: baseline.small_ms_ceiling?.[lang], why: 'an ABSOLUTE bound, because a constant-factor regression that grows both arms equally ' + 'passes the ratio.', }, { label: 'collide arm ms', + key: `collide_ms_ceiling.${lang}`, got: got.collide.ms, - budget: baseline.collide_ms_ceiling[lang], + budget: baseline.collide_ms_ceiling?.[lang], why: 'the ABSOLUTE bound on the shared-leaf layout.', }, ]; for (const check of timingChecks) { + // PRESENCE FIRST — see `requireNumericBudget` for why. All five maps are + // complete today, which is exactly when the check is worth having: every one + // of the four per-language lookups above is one deleted key away from a + // silent no-op. The heap arm HAD THE SAME HOLE and the comment here used to + // deny it: iterating the BASELINE's keys protects that loop against a + // deleted MEASUREMENT, which is a different thing from a deleted BUDGET. + // See `heapBudgetChecks`. + const missing = requireNumericBudget({ + key: check.key, + value: check.budget, + reads: `${check.got} > undefined`, + scope: `That leaves ${lang}'s ${check.label} arm ungated.`, + }); + if (missing !== null) { + failures.push(`${lang}: ${missing}`); + continue; + } if (check.got > check.budget) { failures.push( `${lang}: ${check.label} ${check.got} > budget ${check.budget} — ${check.why} ` + @@ -707,36 +2360,209 @@ for (const lang of LANGS) { ); } } - for (const scale of SCALES) { - for (const field of ['files', 'imports', 'resolved', 'distinct_outcomes', 'fingerprint']) { - if (got[scale][field] !== want[scale][field]) { + // The `context` arm's own discriminator, and the same shape of gate as the + // deep/collide fingerprint comparison above: `armShapes` pins WHAT the two + // call shapes answer, and this pins that they still answer DIFFERENTLY. + // Without it the arm degrades exactly the way `DEEP_PAD = 0` degrades the + // depth arm — a probe on which both halves agree asserts two copies of one + // number. Deleting the fifth argument from `resolveOne`, deleting + // `importedSymbolKind` from PHP's import, or reverting Python to the + // `namespace` spelling all land here, and NOTHING else in this file would + // notice: on the main corpus the leg agrees with the cascade, so the + // fingerprints do not move, and a dropped context only makes the timing arms + // faster. + if (CONTEXT_LANGS.includes(lang) && got.context.with_context === got.context.without_context) { + failures.push( + `${lang}: context arm answers '${got.context.with_context}' with AND without the pass's ` + + `parsedFiles — the fifth argument is not reaching the resolver, or the leg behind it no ` + + `longer runs (PHP needs parsedImport.kind named|alias AND importedSymbolKind ` + + `function|const; Python needs named|alias, since a namespace import never reads ` + + `parsedFiles). run.ts calls resolveImportTarget with five arguments and this bench must ` + + `too. Deterministic: a re-run will not change it.`, + ); + } + // ONE loop for every arm's corpus shape, timing and heap alike. The heap arm + // was reported here and asserted nowhere, which made the four fields that + // decide WHAT it measures free to move: `HEAP_PROBE_TARGET.csharp_csproj` + // swapped for a target matching no `CSPROJ_CONFIGS` rootNamespace skips the + // whole config loop, so the `getFilesInDir` and `getInsensitive` legs never + // run, and the arm the MEMORY section calls "the witness that the read + // pattern IS the footprint" quietly becomes a two-map arm — measured + // 73 703 384 -> 59 921 216 B, ratio 1.017 -> 1.011, ceiling and floor both + // still passing. `HEAP_SMALL` set equal to `HEAP_LARGE` is the same shape of + // hole: it makes `ratio` identically ~1.0 and leaves `bytes_large` untouched. + for (const [arm, shape] of armShapes(lang)) { + for (const field of shape.fields) { + if (got[arm][field] !== want[arm]?.[field]) { failures.push( - `${lang}.${scale}.${field}: ${got[scale][field]} != ${want[scale][field]} — the corpus ` + - `changed shape or the resolver changed its answer for this arm. Every scale is ` + - `asserted separately: the arms differ only in padding and layout, so a defect that ` + - `touches one of them alone moves nothing in the others.`, + `${lang}.${arm}.${field}: ${got[arm][field]} != ${want[arm]?.[field]} — ${shape.why}`, ); } } } } -// Driven by the BASELINE's keys, not the report's, so deleting a heap -// measurement fails instead of silently dropping the gate. -for (const [lang, ceiling] of Object.entries(baseline.heap_ceiling_bytes)) { +// PRESENCE FIRST for the two SCALAR heap budgets, for exactly the reason the +// five timing budgets get it — and the reason the comment up there used to give +// for the heap arm not needing it was wrong. Iterating the baseline's keys +// protects the loop below against a deleted MEASUREMENT (`heap == null`, right +// there); it does nothing about a deleted BUDGET. These two keys are scalars +// rather than per-language maps, so deleting either is one keystroke that +// silently disables that arm for ALL EIGHT languages at once. That makes them +// the widest-blast-radius keys in this file, not the safest — which is what +// their `scope` sentence says and the per-language ones do not. +const heapBudgetChecks = [ + { key: 'heap_floor_fraction', value: baseline.heap_floor_fraction, reads: 'bytes_large < NaN' }, + { key: 'heap_ratio_budget', value: baseline.heap_ratio_budget, reads: 'ratio > undefined' }, +]; +const heapArmScope = `This one key gates all ${HEAP_BUDGETED.length} budgeted heap arms at once.`; +for (const check of heapBudgetChecks) { + const missing = requireNumericBudget({ ...check, scope: heapArmScope }); + if (missing !== null) failures.push(missing); +} + +// And EXACT KEY EQUALITY between each tier's CODE list and the baseline maps +// that gate it, because the loops below iterate the baseline: delete one +// language's ceiling and that language drops out of the loop entirely — still +// measured, still printed, never checked. Both directions, the same shape as the +// LANG_REGISTRY/SCOPE_RESOLVERS inventory arm at the bottom of the file. The +// forward direction (a language with no budget) is the per-language presence +// check inside each loop; this is the reverse (a budget with no arm). +// +// Three maps rather than two: `heap_bound_bytes` is reconciled against +// `HEAP_BOUNDED` exactly as the other two are against `HEAP_BUDGETED`, so a +// language promoted from bounded to budgeted has to move its key in the same +// edit — leave the bound behind and it is an orphan here, take the bound away +// without adding a ceiling and the presence check fires there. +const heapBudgetMaps = [ + ['heap_ceiling_bytes', baseline.heap_ceiling_bytes, HEAP_BUDGETED, 'HEAP_BUDGETED'], + ['heap_reading_bytes', baseline.heap_reading_bytes, HEAP_BUDGETED, 'HEAP_BUDGETED'], + ['heap_bound_bytes', baseline.heap_bound_bytes, HEAP_BOUNDED, 'HEAP_BOUNDED'], +]; +for (const [key, map, codeList, codeListName] of heapBudgetMaps) { + expectNoOrphanKeys( + `baselines.json ${key}`, + Object.keys(map ?? {}), + codeList, + codeListName, + 'the bench budgets a heap arm it does not measure.', + ); +} + +// The two heap tiers are a PARTITION of LANGS by construction (`HEAP_BOUNDED` +// is a filter over it), so the only way a name can be in neither is for +// `HEAP_BUDGETED` to hold one `LANGS` does not — a typo, or a language dropped +// from the registry with its budget left behind. That name would then be +// measured by nothing, and the loop below would report it as a missing arm +// without ever saying why; this says why. +expectNoOrphanKeys( + 'HEAP_BUDGETED', + HEAP_BUDGETED, + LANGS, + 'LANGS', + 'that name is in neither heap tier, because HEAP_BOUNDED is derived as the languages LANGS ' + + 'has and this list does not — so its budget gates nothing and its language, if it has one, ' + + 'is bounded by nothing.', +); +// The same, for the probe map. The forward direction — a language with no probe +// — is caught by the `heap.probe` shape assertion (`undefined` never equals a +// recorded string), so what is left is a probe kept for an arm that no longer +// runs, which reads as coverage and is not. +expectNoOrphanKeys( + 'HEAP_PROBE_TARGET', + Object.keys(HEAP_PROBE_TARGET), + LANGS, + 'LANGS', + 'the bench carries a heap probe for a language it does not benchmark.', +); + +// The same reverse direction for the context arm. The forward direction (a +// language in CONTEXT_LANGS with no baseline block) is `armShapes`, which +// compares against `want.context?.[field]` and fails on undefined; this is the +// other way round — a baseline block for a language the bench hands no context +// is a gate over an arm that is never measured, and `armShapes` would never +// look at it. +expectNoOrphanKeys( + 'baselines.json languages.*.context', + Object.keys(baseline.languages).filter((lang) => baseline.languages[lang].context !== undefined), + CONTEXT_LANGS, + 'CONTEXT_LANGS', + 'the bench pins an arm it does not run.', +); + +// TIER ONE, the budgeted arms: ceiling, floor and ratio, all three unchanged. +// +// Driven by HEAP_BUDGETED, the CODE's list, exactly as the timing arms iterate +// LANGS — so a deleted budget key is a presence failure rather than a language +// that quietly stops being iterated. A deleted MEASUREMENT still fails too: +// `measureHeap` now runs for every language, so a `heap == null` here is the arm +// having been removed or skipped. +for (const lang of HEAP_BUDGETED) { + const ceiling = baseline.heap_ceiling_bytes?.[lang]; + const reading = baseline.heap_reading_bytes?.[lang]; + // `reads` names the comparison each key gates further down: the ceiling is + // compared directly, the reading only after `reading * heap_floor_fraction` + // has turned a missing one into `NaN`. + for (const [key, value, reads] of [ + ['heap_ceiling_bytes', ceiling, 'bytes_large > undefined'], + ['heap_reading_bytes', reading, 'bytes_large < NaN'], + ]) { + const missing = requireNumericBudget({ + key: `${key}.${lang}`, + value, + reads, + scope: + `This loop iterates HEAP_BUDGETED precisely so that deleting the key fails here instead ` + + `of dropping ${lang} out of the gate.`, + }); + if (missing !== null) failures.push(`${lang}: ${missing}`); + } const heap = report[lang]?.heap; if (heap == null) { failures.push( - `${lang}: heap arm missing though heap_ceiling_bytes has a budget for it — the retained-` + - `index measurement was removed or skipped. It is the only arm that can see memory.`, + `${lang}: heap arm missing though HEAP_BUDGETED names it — the retained-index measurement ` + + `was removed or skipped. It is the only arm that can see memory.`, ); continue; } if (heap.bytes_large > ceiling) { failures.push( - `${lang}: retained WorkspaceFileIndex ${heap.mib_large} MiB at ${heap.files_large} files ` + - `(${heap.bytes_large} B) > ceiling ${ceiling} B — buildSuffixIndex is O(files × depth) ` + - `and this is the ABSOLUTE bound on it (#2649). Deterministic: a re-run will not change it.`, + `${lang}: retained per-pass import index ${heap.mib_large} MiB at ${heap.files_large} ` + + `files (${heap.bytes_large} B) > ceiling ${ceiling} B — these indexes are built at ` + + `O(files × depth) and this is the ABSOLUTE bound on that (#2649). Deterministic: a ` + + `re-run will not change it.`, + ); + } + // A FLOOR as well as a ceiling, and it is the arm that would have caught the + // one defect this whole block exists for. When `buildSuffixIndex` went lazy, + // these four arms stopped asking a suffix question, built no map and reported + // 0 B at 32 000 files — and 0 B is under every ceiling, so `--check` printed + // PASS over four gates that had become ceilings over nothing. A ceiling can + // only ever say "not too big"; nothing said "still measuring something". + // + // Taken as a fraction of the RECORDED READING, not of the ceiling. It used to + // be 0.33 x the ceiling, with the comment claiming that put it "at half the + // measured size" — true only for as long as every ceiling stayed at exactly + // 1.5x its reading, which is a convention this file states and nothing + // enforces. Re-tuning one ceiling upward would have loosened that language's + // floor by the same factor, in the one direction the floor exists to watch. + // 0.5 x the reading is the same effective floor today (within 0.8% for all + // eight) and says what it means. `heap_reading_bytes` is the measurement the + // ceiling is derived from too, so the pair still moves together on a + // re-baseline — far below any plausible drift (the readings reproduce to the + // byte across processes) and far above the collapse it watches for. A genuine + // 2x memory WIN trips it too, and that is intended: it must be explained and + // re-baselined, exactly like a fingerprint move. + const floor = reading * baseline.heap_floor_fraction; + if (heap.bytes_large < floor) { + failures.push( + `${lang}: retained per-pass import index ${heap.bytes_large} B at ${heap.files_large} ` + + `files < floor ${Math.round(floor)} B (${baseline.heap_floor_fraction} x recorded ` + + `reading ${reading}) — this arm has almost certainly stopped MEASURING rather than started ` + + `saving. Probe '${heap.probe}' resolves through the real resolver; if a leg it used to ` + + `reach now returns earlier, or an index it forced is now built lazily behind a question ` + + `nobody asks, the arm reads ~0 and every ceiling above passes. Deterministic: a re-run ` + + `will not change it.`, ); } if (heap.ratio > baseline.heap_ratio_budget) { @@ -748,6 +2574,143 @@ for (const [lang, ceiling] of Object.entries(baseline.heap_ceiling_bytes)) { } } +/** + * TIER TWO, the bounded arms: ONE comparison, and what it is a comparison FOR. + * + * `heap_bound_bytes` is the "exclusion still holds" bound. It does not claim + * these nine indexes are small enough, which is what a ceiling claims about a + * budgeted one; it claims each is still the SIZE the decision to leave it out + * was taken on. The re-entry condition the MEMORY section states — "if any of + * the four ever diverges in what it ASKS, it earns an arm the same way" — is a + * claim about growth, and this is the only thing in the file that can see it. + * + * NO FLOOR, and the reason is per language rather than uniform. rust reads 16 B + * because it builds nothing, so any floor at all would be a floor on noise and + * `1.5 x 0 B` is 0 — its bound is ABSOLUTE (1 MiB) for the same reason: a + * multiplier on 16 B fails on the first byte of anything. The other eight are + * stable enough today to floor (0.24% peak-to-peak at worst over five runs) and + * two of them — kotlin at 45.85 MiB and dart at 7.47 — are larger than budgeted + * arms, so a floor there would be worth having. That is a promotion to tier one, + * with a ceiling and a recorded reading, and it is not this change: a floor + * without them would assert "still measuring" against a number nothing else + * bounds. What this tier is NOT is a weaker version of tier one — it is the + * different question, asked of every language instead of eight. + */ +const heapBoundScope = + `That leaves the arm bounded by nothing, which is the state all nine of these were in before ` + + `they were measured.`; +for (const lang of HEAP_BOUNDED) { + const bound = baseline.heap_bound_bytes?.[lang]; + const missing = requireNumericBudget({ + key: `heap_bound_bytes.${lang}`, + value: bound, + reads: 'bytes_large > undefined', + scope: heapBoundScope, + }); + if (missing !== null) failures.push(`${lang}: ${missing}`); + const heap = report[lang]?.heap; + if (heap == null) { + failures.push( + `${lang}: heap arm missing though HEAP_BOUNDED names it — every registered language is ` + + `measured now, and the tier only decides which gate the reading gets.`, + ); + continue; + } + if (missing === null && heap.bytes_large > bound) { + failures.push( + `${lang}: retained per-pass import index ${heap.mib_large} MiB at ${heap.files_large} ` + + `files (${heap.bytes_large} B) > bound ${bound} B — this language is EXCLUDED from the ` + + `budgeted heap tier, and the bound is what says the exclusion still holds. It has grown ` + + `a structure, or started asking its index a question it did not ask when the exclusion ` + + `was recorded. Read _heap_bound_note in baselines.json for this language's reason and ` + + `its recorded reading, then either explain the growth or promote it to HEAP_BUDGETED ` + + `with a ceiling, a reading and a floor. Deterministic: a re-run will not change it.`, + ); + } +} + +// INVENTORY, the arm that makes "every registered language is gated" a checked +// claim instead of a comment. `LANG_REGISTRY` is a hand-written table — it has +// to be, since each row also implies five dispatcher branches — but which +// languages it must contain is not a judgement call, and this is where the two +// are reconciled. Both directions: a resolver registered with no arm here is +// the PR #2911 hole (a language shipping unmeasured), and an arm naming a +// language the registry does not have is a bench measuring something the +// pipeline no longer runs. +// +// Loaded HERE, after the last measurement, rather than imported at the top. +// Reaching `pipeline/registry.ts` drags in every registered scope resolver and +// its providers, and this arm is the only thing in the file that wants it. The +// side benefit is that both modes now measure in the same module state: report +// mode never loads the registry, and `--check` loads it only once every number +// has been taken. +// +// It is NOT cheap and the header says so plainly rather than rounding it down: +// 6.3-6.5 s on one box and 9.3-10.0 s on another, measured in isolation with +// this file's own static imports already resident, which is most of the +// `repsFor` win and the whole reason `--check` did not get faster. Kept anyway, +// because the `benchmarks` job runs ~4.5 minutes clear of CI's critical path, +// so the seconds buy nothing, and because the alternative reconciles arm NAMES +// where this reconciles the `SupportedLanguages` values the dispatchers key +// off. See COST in the header. +const { SCOPE_RESOLVERS } = + await import('../../src/core/ingestion/scope-resolution/pipeline/registry.ts'); +const registeredLanguages = [...SCOPE_RESOLVERS.keys()].sort(); +const benchedLanguages = [...new Set(Object.values(LANG_REGISTRY))].sort(); +for (const language of registeredLanguages) { + if (benchedLanguages.includes(language)) continue; + failures.push( + `${language} is registered in SCOPE_RESOLVERS but has no arm in LANG_REGISTRY — its ` + + `import-target resolver is ungated: nothing pins its output and nothing pins its scaling. ` + + `That is the state JavaScript was in at 25 972 µs per import (PR #2911). Add a row, then ` + + `the five dispatcher branches it needs (uniqueDir, collideDir, uniqueTarget, collideTarget, ` + + `resolveOne) and a baselines.json entry. Deterministic: a re-run will not change it.`, + ); +} +// The reverse half is the same loop as the two above it, so it goes through the +// same helper. Only the FORWARD half stays written out: its message is a +// five-step remediation for adding a language, which no shared framing carries. +expectNoOrphanKeys( + 'LANG_REGISTRY', + benchedLanguages, + registeredLanguages, + 'SCOPE_RESOLVERS', + 'this bench is gating a resolver the pipeline no longer registers.', +); + +// The SAME reconciliation for `CONTEXT_LANGS`, against the registry rather than +// against a claim in a comment. `run.ts` passes the fifth argument to every +// provider; which ones can OBSERVE it is decided by how many parameters each +// hook declares, and that is a number the registry can be asked for. Today +// exactly two answer 5 (php, python) and the other fourteen answer 3 or 4 — +// which is why fourteen arms could ignore this whole question and their numbers +// did not move when it was fixed. +// +// `Function.length` stops at the first defaulted or rest parameter, so a hook +// written as `(a, b, c, d, context = {})` would read 4 and slip past this arm. +// The shared contract declares the parameter as `context?:`, which compiles to +// a plain parameter, so every resolver written against it counts — and one that +// is not is one this arm asks you to look at. +const CONTEXT_PARAM_COUNT = 5; +const contextLanguages = new Set(CONTEXT_LANGS.map((lang) => LANG_REGISTRY[lang])); +for (const [language, resolver] of SCOPE_RESOLVERS) { + const declares = resolver.resolveImportTarget.length >= CONTEXT_PARAM_COUNT; + const benched = contextLanguages.has(language); + if (declares === benched) continue; + failures.push( + declares + ? `${language}'s resolveImportTarget declares ${resolver.resolveImportTarget.length} ` + + `parameters, so it can read the { parsedFiles, parsedImport } context run.ts passes, ` + + `but no arm here supplies one — that leg is measured by nothing. Add the language to ` + + `CONTEXT_LANGS, thread the context in resolveOne, and give it a CONTEXT_PROBE whose ` + + `two answers differ. Deterministic: a re-run will not change it.` + : `CONTEXT_LANGS names '${language}', whose resolveImportTarget declares only ` + + `${resolver.resolveImportTarget.length} parameters — it cannot observe a context, so ` + + `this bench is building a ParsedFile[] per pass that nothing reads and asserting a ` + + `context arm that cannot fail. Deterministic: a re-run will not change it.`, + ); +} + console.log(JSON.stringify(report, null, 2)); if (failures.length > 0) { console.error(`[import-target --check] FAIL\n - ${failures.join('\n - ')}`); diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/csharp.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/csharp.ts index d99dbd35b..24e9e6cfa 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/configs/csharp.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/csharp.ts @@ -31,8 +31,10 @@ export const csharpNamespaceStrategy: ImportResolverStrategy = (rawImportPath, _ const resolvedFiles = resolveCSharpImportInternal( rawImportPath, csharpConfigs, - ctx.normalizedFileList, - ctx.allFileList, + // The Set, not `ctx.normalizedFileList`/`ctx.allFileList`: the resolver + // derives both from it through the same per-pass memo the ctx's own arrays + // come from, so this is the identical pair by a shorter route. + ctx.allFilePaths, ctx.index, evidence, ); diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/swift.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/swift.ts index 8235edf73..05e4b13be 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/configs/swift.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/swift.ts @@ -39,6 +39,19 @@ interface SwiftTargetIndex { * stable reference and the index is built once — not once per import. A * fresh run produces a fresh array → a fresh index, so cross-run staleness * is impossible. + * + * DELIBERATELY NOT ON `import-resolvers/per-file-set.ts` (#2909 sweep): this is + * a TWO-input memo keyed on ONE of them. The index is a function of both + * `ctx` (`allFileList` + the index-aligned `normalizedFileList`) and `targets`, + * but the key is only `ctx.allFileList`, and `perFileSet`'s `build: (key) => T` + * hands the builder nothing but the key. It is sound here only because of an + * invariant OUTSIDE the memo — `targets` is `ctx.configs.swiftPackageConfig + * .targets`, so it shares `ctx`'s lifetime and cannot vary while + * `ctx.allFileList` is fixed — and `perFileSet` has no way to express "and this + * other input is pinned by the same lifetime". Re-keying on `ctx` to make + * `targets` derivable from the key would change what the cache is keyed on and + * force an unreachable null-config arm into the builder, so it is a behaviour + * change rather than a consolidation. Leave it hand-rolled. */ const SWIFT_TARGET_INDEX_CACHE = new WeakMap(); diff --git a/gitnexus/src/core/ingestion/import-resolvers/csharp.ts b/gitnexus/src/core/ingestion/import-resolvers/csharp.ts index 2ce21274f..9183fbb24 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/csharp.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/csharp.ts @@ -5,11 +5,225 @@ * This file contains shared helpers for namespace-based resolution. */ +import { perFileSet } from './per-file-set.js'; +import { getWorkspaceFileIndex } from './workspace-file-index.js'; import type { SuffixIndex } from './utils.js'; import { suffixResolve } from './utils.js'; import type { CSharpProjectConfig, CSharpNamespaceEvidence } from '../language-config.js'; import { csharpSuffixFallbackAllowed } from '../csharp-namespace-gate.js'; +/** + * Directory index backing the namespace-directory fallback below (step 3). + * + * That fallback used to be a full `normalizedFileList` pass per import, per + * matching csproj config — Θ(files), measured at ~1.08 ms per import over + * 50 000 `.cs` files (#2902). #2878 removed the per-import array REBUILD but + * not the scan itself. + * + * The scan's predicate depends only on the file's DIRECTORY, so it can be + * answered from an index built once per file list. Writing `D` for the + * normalized directory of a `.cs` file and `dirPrefix` for the query: + * + * let H = D + '/', P = dirPrefix + '/' + * match ⟺ H.length >= P.length && H.indexOf(P) === H.length - P.length + * + * Derivation, because both halves are load-bearing: + * - the scan keeps a file only when nothing after the matched occurrence holds + * a slash, so the occurrence's trailing '/' must be the file's LAST slash — + * i.e. `H` ends with `P`; + * - it uses `indexOf`, the FIRST occurrence, so `a/Models/b/Models/x.cs` does + * NOT answer `Models`: the first `Models/` is found and `b/Models/x.cs` + * still contains a slash. Dropping that half moves edges in every repo that + * nests a directory name inside itself. + * - the needle ends with '/', so every occurrence of it lies wholly inside + * `D + '/'` and never reaches into the file name — which is what lets the + * whole test be evaluated on `D` alone. + * + * NOT the same query as `package-dir-index.ts`, and the difference is exactly + * one character on each side: that module tests `'/'+D+'/'` against + * `'/'+pkgPath+'/'`, whose leading slash anchors the match to a segment + * boundary. This scan has no leading slash, so `dirPrefix = 'Models'` also + * matches `src/SubModels/` and `dirPrefix = 'src/Models'` also matches + * `vendor/mysrc/Models/`. Those hits are reachable (step 2 below answers only + * the segment-aligned ones, and step 3 runs precisely when step 2 found + * nothing), so the looser predicate is preserved verbatim rather than + * "cleaned up" into a reuse of `filesDirectlyInPkgDir` — see + * `test/unit/import-resolvers/csharp-csproj-parity.test.ts`. + * + * Candidates are narrowed by the directory's LAST segment, the same + * O(directories) bucket `package-dir-index.ts` uses instead of an + * O(files × depth) suffix map (#2649). + */ +interface CsharpNamespaceDirIndex { + /** Last path segment of a directory → every `.cs` directory ending in it. */ + readonly dirsByLastSegment: ReadonlyMap; + /** + * Directory → positions in `WorkspaceFileIndex.normalized` of the `.cs` files + * directly inside it, ascending. + * + * Positions rather than paths: the emitted value is the RAW path, and the two + * arrays are parallel by construction — `normalized` is `all.map(slash)` — so + * a position is the one key that reads correctly in either. Both arrays come + * from the same `getWorkspaceFileIndex(allFilePaths)` object as this index + * itself, so the pairing cannot drift; it used to be a precondition on the + * caller, who passed the two arrays independently. + */ + readonly positionsByDir: ReadonlyMap; + /** + * Directories with no slash of their own — the entire answer to an empty + * `dirPrefix`, which is the one query no last-segment bucket expresses. + */ + readonly singleSegmentDirs: readonly string[]; +} + +/** + * Memoized on the file SET's identity, the same key every other per-file-set + * index in this pipeline uses: the orchestrator builds one Set per pass and + * threads it through every import, so this build runs once. + * + * It used to key on the `normalizedFileList` ARRAY, which was a second key + * shape and — more to the point — one no guard could instrument. Copying an + * array mints a fresh `WeakMap` key while traversing the SET zero extra times, + * so a `[...normalized]` copy at the adapter boundary rebuilt this index once + * per `using` while every scan-counting guard stayed green and only the timing + * bench noticed (#2911 review). Taking the array from + * `getWorkspaceFileIndex(allFilePaths)` inside the builder retires that shape: + * the only way to defeat the memo now is to copy the Set, which is exactly what + * `CountingSet` counts. + * + * It also retires a precondition. The cached positions index `normalized` while + * the emitted value is read from `all`; both now come from the same + * `getWorkspaceFileIndex` object, so the caller can no longer pair a position + * list against a differently-ordered array. + */ +const getCsharpNamespaceDirIndex = perFileSet( + (allFilePaths: ReadonlySet): CsharpNamespaceDirIndex => { + const { normalized: normalizedFileList } = getWorkspaceFileIndex(allFilePaths); + const dirsByLastSegment = new Map(); + const positionsByDir = new Map(); + const singleSegmentDirs: string[] = []; + + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + if (!normalized.endsWith('.cs')) continue; + const lastSlash = normalized.lastIndexOf('/'); + // A file with no directory can never match: the needle always ends with + // '/', so `indexOf` on a slash-free path is always -1. + if (lastSlash < 0) continue; + + const dir = normalized.slice(0, lastSlash); + let positions = positionsByDir.get(dir); + if (positions === undefined) { + positions = []; + positionsByDir.set(dir, positions); + const lastSegment = dir.slice(dir.lastIndexOf('/') + 1); + if (lastSegment === dir) singleSegmentDirs.push(dir); + let dirs = dirsByLastSegment.get(lastSegment); + if (dirs === undefined) { + dirs = []; + dirsByLastSegment.set(lastSegment, dirs); + } + dirs.push(dir); + } + positions.push(i); + } + + return { dirsByLastSegment, positionsByDir, singleSegmentDirs }; + }, +); + +/** + * Every directory that could satisfy `dirPrefix`, as a superset — the exact + * test runs in `matchingDirPositions`. + * + * When `dirPrefix` contains a '/', its own slash forces a segment boundary in + * any matching directory: `H` ending with `…//` means `D` ends with + * `/`, so `D`'s last segment IS `lastSeg` and the exact bucket is + * complete. Without a '/', `D`'s last segment only has to END with `dirPrefix` + * (`SubModels` for `Models`), which no single bucket holds, so the last-segment + * KEYS are swept. That is the one term here that is not O(matches), and it is + * O(distinct last segments), not O(directories): C# repos reuse `Models`, + * `Services`, `Controllers` under every project, so the sweep collapses on the + * layouts that actually occur. Measured at 200 000 `.cs` files, 25 000 + * directories: 456 µs per import when every directory name is unique, 7.9 µs + * on a `SrcN/Models` layout. Closing the unique-name case needs a character- + * suffix map over the segments, which is the O(files × depth) memory shape + * `package-dir-index.ts` cites #2649 to avoid — a design change, not a tune. + * + * An empty `dirPrefix` would sweep every key and keep every directory, so it is + * answered from `singleSegmentDirs` instead: its needle is a bare '/', which + * only a slash-free directory can carry as its LAST slash. + */ +function* candidateDirs(index: CsharpNamespaceDirIndex, dirPrefix: string): Generator { + if (dirPrefix === '') { + yield* index.singleSegmentDirs; + return; + } + const lastSlash = dirPrefix.lastIndexOf('/'); + if (lastSlash >= 0) { + const bucket = index.dirsByLastSegment.get(dirPrefix.slice(lastSlash + 1)); + if (bucket !== undefined) yield* bucket; + return; + } + for (const [lastSegment, dirs] of index.dirsByLastSegment) { + if (!lastSegment.endsWith(dirPrefix)) continue; + yield* dirs; + } +} + +/** Positions of the `.cs` files in each directory matching `dirPrefix`. */ +function* matchingDirPositions( + index: CsharpNamespaceDirIndex, + dirPrefix: string, +): Generator { + const needle = dirPrefix + '/'; + for (const dir of candidateDirs(index, dirPrefix)) { + const haystack = dir + '/'; + // The length guard is not redundant: for a shorter `haystack`, `indexOf` + // returns -1 and `haystack.length - needle.length` can also be -1, which + // would report a bogus match. + if (haystack.length < needle.length) continue; + if (haystack.indexOf(needle) !== haystack.length - needle.length) continue; + const positions = index.positionsByDir.get(dir); + if (positions !== undefined) yield positions; + } +} + +/** + * Append every `.cs` file directly inside a directory matching `dirPrefix`, in + * `normalizedFileList` order — the order the single-pass scan emitted, which + * this function's callers return as the whole edge target list. + */ +function pushFilesDirectlyInNamespaceDir( + index: CsharpNamespaceDirIndex, + dirPrefix: string, + allFileList: readonly string[], + results: string[], +): void { + // One matching directory is the overwhelmingly common case, and its positions + // are already ascending, so the first bucket is held by reference. A second + // one promotes it to a real accumulator that is appended to from then on — + // never re-spread per directory, which would cost O(files × dirs²) copies in + // a monorepo carrying the same namespace directory under many projects. + let first: readonly number[] | null = null; + let merged: number[] | null = null; + for (const positions of matchingDirPositions(index, dirPrefix)) { + if (first === null) { + first = positions; + continue; + } + if (merged === null) merged = [...first]; + for (const position of positions) merged.push(position); + } + if (first === null) return; + if (merged === null) { + for (const position of first) results.push(allFileList[position]); + return; + } + merged.sort((a, b) => a - b); + for (const position of merged) results.push(allFileList[position]); +} + /** * Resolve a C# using-directive import path to matching .cs files (low-level helper). * Tries single-file match first, then directory match for namespace imports. @@ -17,15 +231,23 @@ import { csharpSuffixFallbackAllowed } from '../csharp-namespace-gate.js'; * The final unanchored suffix fallback is gated on `evidence` so BCL usings * (e.g. `System.Threading.Tasks`) can't match a coincidentally-named local * file (#1881). When `evidence` is omitted the fallback stays permissive. + * + * Takes the file SET, not the two materialized lists it used to take: both are + * derived here from the per-pass `getWorkspaceFileIndex` memo, which is where + * every caller already got them. That leaves one key shape for the indexes + * below and makes the `normalized`/`all` pairing structural rather than a + * contract the caller has to honour. `index` stays a parameter — the parity + * harness drives this resolver with and without one, and the no-index legs are + * a tested dimension, not a degenerate case. */ export function resolveCSharpImportInternal( importPath: string, csharpConfigs: CSharpProjectConfig[], - normalizedFileList: string[], - allFileList: string[], + allFilePaths: ReadonlySet, index?: SuffixIndex, evidence?: CSharpNamespaceEvidence, ): string[] { + const { normalized: normalizedFileList, all: allFileList } = getWorkspaceFileIndex(allFilePaths); const namespacePath = importPath.replace(/\./g, '/'); const results: string[] = []; @@ -75,21 +297,30 @@ export function resolveCSharpImportInternal( if (results.length > 0) return results; } - // 3. Linear scan fallback for directory matching - if (results.length === 0) { - const dirTrail = dirPrefix + '/'; - for (let i = 0; i < normalizedFileList.length; i++) { - const normalized = normalizedFileList[i]; - if (!normalized.endsWith('.cs')) continue; - const prefixIdx = normalized.indexOf(dirTrail); - if (prefixIdx < 0) continue; - const afterDir = normalized.substring(prefixIdx + dirTrail.length); - if (!afterDir.includes('/')) { - results.push(allFileList[i]); - } - } - if (results.length > 0) return results; - } + // 3. Directory matching, UNANCHORED. + // + // Not redundant with step 2, and not skippable when `index` is present: + // `getFilesInDir` is keyed on SEGMENT suffixes of a directory, while this + // leg's predicate is an unanchored substring one, so it additionally + // answers `Models` with `src/SubModels/` and `src/Models` with + // `vendor/mysrc/Models/`. It is also the only leg that answers an empty + // `dirPrefix` — the `relative = ''` branch above (the import IS the root + // namespace) with no `projectDir` to stand in for it — because + // `buildSuffixIndex` emits an empty directory suffix only for a path that + // BEGINS with '/', so over repo-relative paths `getFilesInDir('', '.cs')` + // is always empty. See `CsharpNamespaceDirIndex` above for the index that + // replaced the per-import Θ(files) scan this used to be (#2902). + // + // `results` is provably empty here: step 2 returns as soon as it pushes + // anything, and so does this leg, so every iteration of the config loop + // starts empty. + pushFilesDirectlyInNamespaceDir( + getCsharpNamespaceDirIndex(allFilePaths), + dirPrefix, + allFileList, + results, + ); + if (results.length > 0) return results; } // Fallback: suffix matching without namespace stripping (single file). diff --git a/gitnexus/src/core/ingestion/import-resolvers/go.ts b/gitnexus/src/core/ingestion/import-resolvers/go.ts index c33c46422..f4ac5048a 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/go.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/go.ts @@ -25,8 +25,8 @@ export function resolveGoPackageDir(importPath: string, goModule: GoModuleConfig export function resolveGoPackage( importPath: string, goModule: GoModuleConfig, - normalizedFileList: string[], - allFileList: string[], + normalizedFileList: readonly string[], + allFileList: readonly string[], ): string[] { if (!importPath.startsWith(goModule.modulePath)) return []; diff --git a/gitnexus/src/core/ingestion/import-resolvers/jvm.ts b/gitnexus/src/core/ingestion/import-resolvers/jvm.ts index 194cfdac8..b6723b8e2 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/jvm.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/jvm.ts @@ -31,8 +31,8 @@ export const appendKotlinWildcard = (importPath: string, importNode: SyntaxNode) */ export function resolveJvmWildcard( importPath: string, - normalizedFileList: string[], - allFileList: string[], + normalizedFileList: readonly string[], + allFileList: readonly string[], extensions: readonly string[], index?: SuffixIndex, ): string[] { @@ -90,8 +90,8 @@ export function resolveJvmWildcard( */ export function resolveJvmMemberImport( importPath: string, - normalizedFileList: string[], - allFileList: string[], + normalizedFileList: readonly string[], + allFileList: readonly string[], extensions: readonly string[], index?: SuffixIndex, ): string | null { diff --git a/gitnexus/src/core/ingestion/import-resolvers/pass-cache.ts b/gitnexus/src/core/ingestion/import-resolvers/pass-cache.ts new file mode 100644 index 000000000..4c307cfd0 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/pass-cache.ts @@ -0,0 +1,79 @@ +import { buildSuffixIndex, type SuffixIndex } from './utils.js'; + +/** + * Everything the standard `resolveTsTarget` path derives from one workspace + * file set: the file list, the lower-cased file list, the suffix index and the + * per-pass `resolveCache`. + * + * Without this memoization the resolver re-derived `allFileList` and + * `normalizedFileList` (both O(N_files)), rebuilt the index and threw away the + * `resolveCache` on every import — O(N_files × N_imports) total work for what + * should be O(N_files + N_imports). + */ +export interface ImportPassCache { + readonly allFilePaths: Set; + readonly allFileList: readonly string[]; + readonly normalizedFileList: readonly string[]; + readonly index: SuffixIndex; + readonly resolveCache: Map; +} + +/** + * Build that state. Shared by every adapter whose resolution runs through + * `resolveTsTarget`. + * + * Not a dedup of identical copies, and the difference is the point. At + * 49c5b7d81 each of those adapters carried this record inline and they did NOT + * agree: `languages/typescript/scope-resolver.ts` and + * `languages/vue/import-target.ts` held six byte-identical fields built around + * `index: buildSuffixIndex(normalizedFileList, allFileList)`, while + * `languages/javascript/import-target.ts` held five and never called + * `buildSuffixIndex` at all. That one missing field IS the O(imports × files) + * defect PR #2911 fixed — `resolveTsTarget` fell back to `suffixResolve`'s + * linear scan for every JavaScript import — and the header of + * `languages/javascript/import-target.ts` carries the measurements. Hoisting + * the builder is what makes a fourth adapter unable to omit it again: `index` + * is not optional on `ImportPassCache`. + * + * The BUILDER is shared; the MEMO deliberately is not. Each adapter wraps this + * in its own `perFileSet(...)`, so each gets its own `WeakMap`, its own index + * instance and — the one that would be a behaviour change — its own + * `resolveCache`. The languages disagree about what a specifier resolves to + * (`tsconfigPaths` is read from config for TypeScript and Vue, pinned to `null` + * for JavaScript, and the tried extension list differs), so one shared resolve + * cache across them would hand a language another language's answers. + * + * Sharing the builder is a code dedup and nothing more: it buys no runtime + * reuse, because there is none to buy. Each provider pass builds its own + * `allFilePaths` Set (`scope-resolution/pipeline/run.ts`, per provider), so + * TypeScript's set and JavaScript's set are different objects and therefore + * different `WeakMap` keys even where the two memos are the same code. + */ +export function buildImportPassCache(allFilePaths: ReadonlySet): ImportPassCache { + const allFileList = Array.from(allFilePaths); + // LOWERCASED, not slash-normalized — unlike every other caller of + // `buildSuffixIndex`. That is what `alreadyLowercased` below records. + const normalizedFileList = allFileList.map((f) => f.toLowerCase()); + return { + // Copied ONCE per file set, not once per import: `TsResolveContext` wants a + // mutable `Set` and the orchestrator hands us a `ReadonlySet`. The copy is + // not the #1918 hazard because the cache KEY is the caller's original Set. + allFilePaths: new Set(allFilePaths), + allFileList, + normalizedFileList, + // Every suffix of an all-lowercase path is itself lowercase, so the index's + // case-folded map came out a byte-for-byte copy of its exact map — same + // keys, same values, same insertion order — one per `ImportPassCache`, so + // once per adapter per pass. Measured 14.00 MiB at 32 000 paths, 29.8% of + // the retained `ImportPassCache`. The flag drops the copy; it does not change + // what `getInsensitive` answers, because the copy was the identity (see + // `SuffixIndexOptions`). Checked, not assumed: over 474 524 probes on four + // mixed-case corpora — Vue PascalCase plus alias specifiers, case-colliding + // twins, a 600-file deep monorepo, and Unicode paths carrying final sigma, + // dotted-I and sharp-S — the two maps came out byte-identical, the exact + // map was the sole answerer 0 times, and `get(s) || getInsensitive(s)` + // returned the same file 474 524 times out of 474 524. + index: buildSuffixIndex(normalizedFileList, allFileList, { alreadyLowercased: true }), + resolveCache: new Map(), + }; +} diff --git a/gitnexus/src/core/ingestion/import-resolvers/per-file-set.ts b/gitnexus/src/core/ingestion/import-resolvers/per-file-set.ts new file mode 100644 index 000000000..4de0ee361 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/per-file-set.ts @@ -0,0 +1,83 @@ +/** + * The one memo every per-file-set index in this pipeline is built on. + * + * The scope-resolution orchestrator builds ONE file-set object per provider + * pass and threads that same object through every `resolveImportTarget` call in + * the pass, so anything derived from it — a suffix index, a package-directory + * map, a basename bucket — can be built once and read by every import instead + * of rebuilt per import. Keying on the object's IDENTITY is what makes that + * work, and it is equally the contract callers must keep: the set is passed + * THROUGH, never copied. A defensive `new Set(allFilePaths)` at an adapter + * boundary hands a fresh key per import and silently restores + * O(imports × files) — the bug PR #1918 shipped and had to fix in review (P1). + * The guards are `test/integration/-import-index-reuse.test.ts` and, for + * every registered language at once, + * `test/unit/scope-resolution/import-target-index-reuse.contract.test.ts`, + * whose inventory arm fails when an entry of `SCOPE_RESOLVERS` has no fixture. + * That arm is why no language is named here: the registry is the census, and a + * hand-copied list of languages goes stale the release after it is written. + * + * A `WeakMap` rather than a `Map`: the entry is reclaimed with the file set it + * was derived from, so a pass can never read a previous pass's index and memory + * does not grow across runs. There is no invalidation rule to get wrong because + * there is nothing to invalidate — a new file set is a new key. + * + * The KEY TYPE is constrained rather than described, because which object is + * the key decides whether the guards above can see the memo fail, and a prose + * list of call sites is the thing this file elsewhere tells you not to write. + * `K` admits exactly the two shapes the orchestrator keeps stable for a pass: + * + * - `ReadonlySet`, the pass's file set — every index derived from it, + * including the derived header-closure sets that `languages/{c,cpp}/ + * scope-resolver.ts` memoize inside an outer per-file-set memo. Defeating + * one of these means copying the SET, which re-traverses it, which the + * `CountingSet` instrument (`test/helpers/counting-file-set.ts`) reads as a + * scan count rising with the import count. + * - `readonly ParsedFile[]`, the pass's parsed-file array. Not derived from + * the file set at all, so the file-set guards do not reach them; these key + * on the array the orchestrator already threads through the pass, and their + * contract is that same pass-through discipline. The instrument that CAN see + * them counts element reads on that array — `countedParsedFiles`, beside + * `CountingSet`, driven by the contract test's `minimumParsedFileReads`. + * + * A THIRD shape — an array materialized from the file set — is what the type + * exists to reject. `import-resolvers/csharp.ts` used one until #2911, and it + * is worth a compile error rather than a rule: copying an array mints a fresh + * `WeakMap` key while traversing the Set zero extra times, so every + * scan-counting guard stays green at its correct value while the index rebuilds + * once per import. That failure is invisible to the whole instrument family + * above and was caught only by a timing ratio in `bench/import-target/`. Derive + * the array inside the builder from `getWorkspaceFileIndex(allFilePaths)` + * instead. `string[]` is not assignable to `K`, so the shape cannot come back + * silently — `configs/swift.ts` keeps the one hand-rolled `WeakMap` on + * `ctx.allFileList` in the tree, deliberately and with its reasons written + * down, and it is deliberately NOT on this primitive. + * + * `T extends object` is deliberate, chosen over probing `has` before `get`. + * `WeakMap.get` returning `undefined` cannot distinguish "not built yet" from + * "built, and the value is `undefined`"; constraining the value to an object + * makes the second case unrepresentable rather than paying a second lookup on + * every import, and it needs no cast to type-check. Every index memoized here + * is a record, `Map` or `Set`, so the constraint costs nothing today — and a + * later caller wanting to memoize a `string | null` gets a compile error + * pointing at this line instead of a memo that silently rebuilds on every miss. + * + * A `build` that THROWS stores nothing, so the next call for that key runs it + * again: failures are not memoized, and a half-filled index is never published. + * Inert for the builders here — each is a pure, total pass over the file set — + * and the safer of the two behaviours if that ever stops being true. + */ +import type { ParsedFile } from 'gitnexus-shared'; + +export function perFileSet | readonly ParsedFile[], T extends object>( + build: (key: K) => T, +): (key: K) => T { + const cache = new WeakMap(); + return (key) => { + const cached = cache.get(key); + if (cached !== undefined) return cached; + const built = build(key); + cache.set(key, built); + return built; + }; +} diff --git a/gitnexus/src/core/ingestion/import-resolvers/php.ts b/gitnexus/src/core/ingestion/import-resolvers/php.ts index 303bf5546..6652ecbfa 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/php.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/php.ts @@ -37,8 +37,8 @@ export function resolvePhpImportInternal( importPath: string, composerConfig: ComposerConfig | null, allFiles: Set, - normalizedFileList: string[], - allFileList: string[], + normalizedFileList: readonly string[], + allFileList: readonly string[], index?: SuffixIndex, ): string | null { // Normalize: replace backslashes with forward slashes @@ -67,21 +67,38 @@ export function resolvePhpImportInternal( const lastSlash = remainder.lastIndexOf('/'); const nsDir = lastSlash >= 0 ? dirPrefix + '/' + remainder.slice(0, lastSlash) : dirPrefix; - // Prefer SuffixIndex directory lookup (O(log n + matches)) over linear scan + // Prefer SuffixIndex directory lookup (O(log n + matches)) over linear scan. + // + // An EMPTY bucket is a final answer, not a miss to retry with the scan + // below — which is what the `else` restores, and what this comment + // always claimed. Re-scanning on empty was the last per-import + // workspace traversal left in PHP resolution after #2901: any `use` + // matching a PSR-4 prefix whose directory holds no direct `.php` child + // (`App\Legacy\Ghost`) paid a full pass, measured at 201 traversals for + // 200 imports. + // + // The bucket is a superset of what the scan can find, for BOTH index + // shapes that reach here. A root-anchored direct child `nsDir/.php` + // has its directory exactly equal to `nsDir`, and `nsDir` is always one + // of that directory's own suffixes — so the shared `dirMap` (keyed on + // every directory suffix) necessarily contains it, as does the + // root-anchored parity index `languages/php/import-target.ts` builds. + // Empty superset therefore implies empty scan, and control falls + // through to the next PSR-4 prefix exactly as before. if (index) { const candidates = index.getFilesInDir(nsDir, '.php'); if (candidates.length > 0) return candidates[0]; - } - - // Fallback: linear scan (only when SuffixIndex unavailable) - const nsDirPrefix = nsDir.endsWith('/') ? nsDir : nsDir + '/'; - for (const f of allFiles) { - if ( - f.startsWith(nsDirPrefix) && - f.endsWith('.php') && - !f.slice(nsDirPrefix.length).includes('/') - ) { - return f; + } else { + // Linear scan, only when a SuffixIndex is genuinely unavailable. + const nsDirPrefix = nsDir.endsWith('/') ? nsDir : nsDir + '/'; + for (const f of allFiles) { + if ( + f.startsWith(nsDirPrefix) && + f.endsWith('.php') && + !f.slice(nsDirPrefix.length).includes('/') + ) { + return f; + } } } } diff --git a/gitnexus/src/core/ingestion/import-resolvers/python-file-index.ts b/gitnexus/src/core/ingestion/import-resolvers/python-file-index.ts new file mode 100644 index 000000000..d624bb45a --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/python-file-index.ts @@ -0,0 +1,371 @@ +/** + * The one per-file-set index behind Python import resolution, plus the two + * importer-chain memos that ride inside it. + * + * ## Why this is its own module + * + * Everything here is derived from `allFilePaths` and nothing here is specific + * to either CALLER, and there are two of them on opposite sides of a layer + * boundary: `import-resolvers/python.ts` resolves the single-segment bare tier + * and `languages/python/import-target.ts` resolves the dotted tiers. The second + * imports the first, so the index could not live in either without the other + * reaching back through a cycle — it used to live in `import-target.ts`, which + * is why the bare tier had no O(1) proof of absence and probed the whole + * ancestor chain for every `import os`. + * + * The shape is the one `workspace-file-index.ts` and `package-dir-index.ts` + * already use in this directory: an interface, one `perFileSet` builder, and + * query functions taking the index. + */ + +import { perFileSet } from './per-file-set.js'; + +/** + * The importer's ancestor directories, CLOSEST FIRST and excluding the + * workspace root — `["backend/routers", "backend"]` for `backend/routers/x.py` + * — memoized per importer DIRECTORY for the lifetime of the pass. + * + * This is the #2913 fix. Both consumers used to rebuild the chain inline, one + * `dirParts.slice(0, i).join('/')` per component, on EVERY import: a per-import + * cost proportional to the importer's path depth, and quadratic in characters, + * on a file index that is itself depth-free. Real Python layouts are deep + * (`src/pkg/sub/feature/impl/mod.py` is ordinary), so the resolver was 6.8x + * slower on a deep corpus than on a shallow one holding the file count fixed, + * where every other language sat between 1.0x and 3.4x. + * + * A directory's ancestors are a pure function of the directory, and a pass + * resolves many imports per file, so one entry serves every import issued from + * anywhere in that directory. + * + * ## Lifetime and memory + * + * The Map lives INSIDE the per-file-set index, so it is reclaimed with the file + * set it was reached through (`perFileSet` is a `WeakMap`): it cannot leak + * across passes or repos, and there is no invalidation rule to get wrong. It is + * filled lazily, so it holds one entry per directory that actually ISSUES a + * Python import, never one per file and never one per directory in the repo — + * the bound #2649 (kernel-scale OOM) asks for. Each entry's strings are + * `slice`s of the longest one, so a chain costs pointers rather than a copy of + * the path per component. + * + * The derived key is the importer's directory exactly as the old inline code + * computed it — `norm.split('/').slice(0, -1).join('/')`, which for a path + * without a separator is `''` (a root-level importer, whose chain is empty). + */ +/** + * The importer's own directory, normalized — the key BOTH per-directory memos + * below are stored under. + * + * One exported derivation rather than one per accessor: the two memos live in + * the same index and must agree on what "the importer's directory" is, and a + * caller that already holds the directory (the bare-import tier computes it for + * its own proximity check) should not pay for it twice. It was three copies of + * `replace / lastIndexOf / slice` across two modules before, byte-identical by + * inspection and by nothing else. + */ +export function importerDirOf(fromFile: string): string { + const norm = fromFile.replace(/\\/g, '/'); + const lastSlash = norm.lastIndexOf('/'); + return lastSlash === -1 ? '' : norm.slice(0, lastSlash); +} + +export function importerAncestors(index: PythonFileIndex, importerDir: string): readonly string[] { + const memoized = index.ancestorsByDir.get(importerDir); + if (memoized !== undefined) return memoized; + const built = buildImporterAncestors(importerDir); + index.ancestorsByDir.set(importerDir, built); + return built; +} + +/** + * `["a/b/c", "a/b", "a"]` for `a/b/c`. Empty components are dropped first, so + * an absolute `/a/b` yields `["a/b", "a"]` — matching the `filter(Boolean)` the + * two inline walks did, and with it the absolute-path gating pinned by + * `python-import-target-parity.test.ts` (PR #1918 review P3a). + */ +function buildImporterAncestors(importerDir: string): readonly string[] { + const chain: string[] = []; + const parts = importerDir.split('/').filter(Boolean); + if (parts.length === 0) return chain; + chain.push(parts.join('/')); + for (let i = 1; i < parts.length; i++) { + const child = chain[i - 1]; + chain.push(child.slice(0, child.lastIndexOf('/'))); + } + return chain; +} + +/** + * Per-file-set index for Python import resolution, memoized on the + * `allFilePaths` Set object (the same Set is passed for every import in a run, + * so the index is built once and reused). Replaces the per-import O(files) + * scans in `resolveAbsoluteFromFiles` (suffix match) and `hasRepoCandidate` + * (package-existence gate) with O(1)/O(bucket) lookups. + * + * - `normSet`: every file path, normalized to forward slashes (for the exact + * `f === rootFile|initFile` membership checks). It IS derivable from the two + * buckets below — both probes could be a `.some(c => c.norm === …)` over + * `byBasename.get(rootFile)` / `byInitParent.get(initFile)` — and it is kept + * anyway, deliberately. `byBasename` is keyed on the BASENAME, so its bucket + * for a common Python file name is not small and grows with the repo: on a + * 9 000-file service tree, `utils.py`, `models.py` and `views.py` hold 1 000 + * entries each. `import utils` would then scan every `utils.py` in the + * workspace on every import — a per-import cost proportional to corpus size, + * which is the exact defect class #2901/#2902/#2908 removed. The Set trades + * ~1.6 MB at 32 000 files, against a 6.4 MB reading, to keep both probes + * O(1). Do not "simplify" it away without re-measuring that bucket. + * - `byBasename`: last path component (e.g. `models.py`, `__init__.py`) -> + * all `{ raw, norm }` candidates, so suffix matches can be gathered from the + * relevant bucket and the exact tie-break applied across ALL of them. + * - `byInitParent`: `__init__.py` files keyed by their last TWO components + * (`/__init__.py`). The package suffix lookup (`pkg.sub` -> + * `…/sub/__init__.py`) targets only same-named package dirs via this map + * instead of scanning every `__init__.py` in the repo — the common + * multi-segment import path no longer scales with package count + * (PR #1918 review P2b). `__init__.py` files stay in `byBasename` too, for + * the rarer explicit `pkg.__init__` import that resolves via the module + * (`….py`) lookup. + * - `dirPrefixes`: every directory prefix of a `.py` file, trailing-slashed + * (`a/b/c.py` -> `a/`, `a/b/`), for "is there a .py file under `/`". + * - `nestedDirNames`: the NAME of every such directory that has a non-empty + * parent (`a/b/c.py` -> `b`, not `a`), which is exactly the set of segments + * `hasRepoCandidate`'s ancestor walk can ever match — so a segment absent + * from it settles the walk in one lookup (#2913). + * - `ancestorsByDir`: the per-importer-directory ancestor-chain memo behind + * `importerAncestors`. The one structure here that is NOT derived from the + * file set: it is filled lazily, from the importer paths the pass actually + * resolves against, and lives here so it dies with the pass. + * - `bareImportPrefixesByDir`: the same idea for the OTHER chain — the + * sys.path-style prefixes `resolvePythonImportInternal`'s single-segment + * walk probes. A different sequence, not a different spelling: see + * `importerBarePrefixes`. Two memos in one index rather than two indexes, + * because they are keyed on the same thing and must die together. + * + * Exported for `test/unit/scope-resolution/python/python-importer-ancestors.test.ts` + * and `test/unit/import-resolvers/python-importer-prefixes.test.ts`, which read + * the two memos after driving the production adapters. No counter ships for + * either — the Map IS the memo, and its SIZE is the assertion: one entry per + * importer directory, however many imports were resolved. Everything else about + * the index stays internal. + */ +export interface PythonFileIndex { + readonly normSet: Set; + readonly byBasename: Map; + readonly byInitParent: Map; + readonly dirPrefixes: Set; + readonly nestedDirNames: Set; + readonly ancestorsByDir: Map; + readonly bareImportPrefixesByDir: Map; +} + +export const getPythonFileIndex = perFileSet( + (allFilePaths: ReadonlySet): PythonFileIndex => { + // Runs on a cache miss only. That it happens once per run and not once per + // import is asserted by counting traversals of the Set itself, in + // `test/integration/python-import-index-reuse.test.ts` — the PR #1918 review + // P1 guard (#2909). + + const normSet = new Set(); + const byBasename = new Map(); + const byInitParent = new Map(); + const dirPrefixes = new Set(); + const nestedDirNames = new Set(); + + for (const raw of allFilePaths) { + const norm = raw.replace(/\\/g, '/'); + // Python import resolution only ever queries `.py` paths: module `.py` + // and package `/__init__.py` membership (normSet), `.py` / + // `__init__.py` basename buckets (byBasename), and `.py` directory prefixes + // (dirPrefixes). Non-`.py` files can never match any of those, so skip them + // — they were dead weight in every structure on polyglot monorepos + // (PR #1918 review P3b; dirPrefixes was already `.py`-gated). + if (!norm.endsWith('.py')) continue; + normSet.add(norm); + + // ONE entry object per file, shared by both buckets below: a package file + // lands in `byBasename` and `byInitParent`, and two literals for the same + // `(raw, norm)` pair cost ~40 B each on every `__init__.py`. + const entry = { raw, norm }; + + const lastSlash = norm.lastIndexOf('/'); + const base = lastSlash >= 0 ? norm.slice(lastSlash + 1) : norm; + // `set(base, [entry])` rather than `set(base, [])` then `push`: an empty + // array literal that is immediately pushed to makes V8 grow the backing + // store to its 16-slot minimum, so every bucket holding ONE file retains + // 15 empty pointer slots — 128 B — for the whole pass. `byBasename` has + // roughly one bucket per file, which made that the dominant term in this + // index: measured 5.50 MiB against 1.60 MiB for the one-element form at + // 32 000 `.py` paths, byte-identical contents. Same shape as + // `languages/php/import-target.ts`'s directory buckets. + const bucket = byBasename.get(base); + if (bucket === undefined) byBasename.set(base, [entry]); + else bucket.push(entry); + + // Package files also get a parent-keyed bucket so a `pkg.sub` lookup hits + // only `…/sub/__init__.py` candidates, not every `__init__.py` (P2b). + if (base === '__init__.py' && lastSlash >= 0) { + const dir = norm.slice(0, lastSlash); + const parentSlash = dir.lastIndexOf('/'); + const parentName = parentSlash >= 0 ? dir.slice(parentSlash + 1) : dir; + if (parentName) { + const initKey = `${parentName}/__init__.py`; + const ib = byInitParent.get(initKey); + if (ib === undefined) byInitParent.set(initKey, [entry]); + else ib.push(entry); + } + } + + // Directory prefixes: every slash-terminated prefix of the path (every + // index just past a '/', up to and including the file's own directory). + // Scanning the FULL normalized path — including any leading '/' for + // absolute paths — makes `dirPrefixes.has(X)` match exactly when the old + // gate's `f.startsWith(X)` (X always ends in '/') matched. The previous + // split+`filter(Boolean)` dropped the leading empty component, so an + // absolute file `/repo/svc/x.py` yielded `repo/svc/` (no leading slash) and + // gate-passed where `"/repo/svc/x.py".startsWith("repo/svc/")` is false + // (PR #1918 review P3a). For relative paths the set is identical. + // + // The walk runs from the DEEPEST prefix outward and stops at the first + // one already recorded. Every prefix is added together with all of its + // own ancestors, so a hit proves the rest of the chain is already there — + // which makes the second and later files of a directory cost ONE lookup + // instead of one insert per path component. This build was the last part + // of Python's resolution that still scaled with path depth (#2913): the + // same 400-file corpus moved sixteen directories down went from 800 + // inserts to 7200, for the same ~120 distinct prefixes. + // + // `nestedDirNames` rides the same walk. A directory prefix has the shape + // `//` — the only shape `hasRepoCandidate`'s check (3) + // probes — exactly when another slash precedes it at index > 0. Index 0 + // is excluded on purpose: `a/` and `/` name a directory whose parent is + // empty, which check (2) already answers and which the ancestor walk + // (non-empty ancestors only) never probes. + for (let i = lastSlash; i >= 0; i--) { + if (norm[i] !== '/') continue; + const dirPrefix = norm.slice(0, i + 1); + if (dirPrefixes.has(dirPrefix)) break; + dirPrefixes.add(dirPrefix); + const parentSlash = i > 0 ? norm.lastIndexOf('/', i - 1) : -1; + if (parentSlash > 0) nestedDirNames.add(norm.slice(parentSlash + 1, i)); + } + } + + return { + normSet, + byBasename, + byInitParent, + dirPrefixes, + nestedDirNames, + ancestorsByDir: new Map(), + bareImportPrefixesByDir: new Map(), + }; + }, +); + +/** + * The sys.path-style prefixes `resolvePythonImportInternal`'s single-segment + * bare-import walk probes, in order, for an importer sitting in `importerDir` — + * memoized per DIRECTORY for the lifetime of the pass, in the same index and + * for the same reasons as `importerAncestors`. + * + * ## Why this is not `ancestorsByDir` + * + * A DIFFERENT SEQUENCE, not a different spelling. For `backend/routers/cron.py`: + * + * importerAncestors ["backend/routers", "backend"] + * importerBarePrefixes ["backend/", ""] + * + * Three differences, each load-bearing: + * + * 1. `importerAncestors` opens with the importer's OWN directory; this walk + * does not, because its proximity check has already probed that directory. + * 2. This walk ENDS at the workspace root (`""`, which probes `.py` + * unprefixed); `importerAncestors` stops short of it, because + * `resolveAbsoluteFromFiles` probes the root before its walk instead. + * 3. `importerAncestors` drops empty components (`filter(Boolean)`); this walk + * keeps them, and the difference decides real resolutions — for + * `/abs/a/b/mod.py` this walk probes `/abs/a/`, `/abs/`, `""`, `""` where a + * filtered chain would probe `abs/a/b/`, `abs/a/`, `abs/`, none of which is + * a prefix of any file in an absolute-path workspace. + * + * So the two cannot share one chain without changing which files resolve. They + * do share the index, the key and the lifetime, which is what actually matters + * for #2649: both are filled lazily, hold one entry per directory that ISSUES + * an import, and die with the pass because the index does. + */ +export function importerBarePrefixes( + index: PythonFileIndex, + importerDir: string, +): readonly string[] { + const memoized = index.bareImportPrefixesByDir.get(importerDir); + if (memoized !== undefined) return memoized; + const built = buildImporterBarePrefixes(importerDir); + index.bareImportPrefixesByDir.set(importerDir, built); + return built; +} + +/** + * `["a/b/", "a/", ""]` for `a/b/c` — every proper ancestor of `importerDir`, + * closest first, slash-terminated, ending at the workspace root. + * + * Cutting the string at each `lastIndexOf('/')` walks the same ancestors the + * pre-#2913-followup `dirParts.slice(0, i).join('/')` produced, INCLUDING the + * empty components a `filter(Boolean)` would have dropped: `/abs/a/b` yields + * `["/abs/a/", "/abs/", "", ""]`, the second `""` being the `i === 0` step that + * followed the leading empty component. Byte-identical sequences, duplicates + * kept, so the probes this feeds are unchanged in content, order and count. + */ +function buildImporterBarePrefixes(importerDir: string): readonly string[] { + const prefixes: string[] = []; + let dir = importerDir; + let slash = dir.lastIndexOf('/'); + while (slash !== -1) { + dir = dir.slice(0, slash); + prefixes.push(dir === '' ? '' : `${dir}/`); + slash = dir.lastIndexOf('/'); + } + prefixes.push(''); + return prefixes; +} + +/** + * "No file anywhere in the workspace can be `/.py` or + * `//__init__.py`, for ANY prefix ``" — in two Map lookups. + * + * This is a PROOF OF ABSENCE, not a heuristic filter, and it is what lets the + * single-segment bare walk skip itself entirely. Both shapes it rules out are + * the only two shapes that walk probes: a probe `${prefix}${segment}.py` that + * is a member of the file set is a path with no backslash (the prefix comes + * from a normalized importer and the guard below rejects a segment carrying + * one), so it equals its own normalized form and its basename is exactly + * `${segment}.py` — which puts it in `byBasename`. A probe + * `${prefix}${segment}/__init__.py` that is a member likewise has parent + * directory name exactly `segment`, non-empty, which puts it in `byInitParent` + * whether or not `prefix` is empty. So a miss in both buckets means every probe + * the walk would issue is guaranteed to miss. + * + * Two inputs cannot be proven absent and get `false` — walk as before: + * + * - the EMPTY segment (a target spelled with a trailing dot). + * `byInitParent` skips `__init__.py` files whose parent directory name is + * empty, so its absence proves nothing. Same carve-out + * `resolveAbsoluteFromFiles` makes for `lastSeg === ''`. + * - a segment containing a BACKSLASH. The buckets are keyed on normalized + * paths, so a raw `a\b.py` is filed under basename `b.py`; a probe for the + * segment `a\b` would look up `a\b.py`, miss, and wrongly conclude absence + * while `allFilePaths.has('a\\b.py')` is true. Not reachable from a Python + * import statement, but this function is a proof and a proof has no + * unstated preconditions. + * + * The dotted tier in `languages/python/import-target.ts` asks the same question + * of the same two buckets and is deliberately NOT routed through here: it needs + * the candidate ARRAYS for its suffix fallback, so it does the two `get`s it + * already needs and derives the answer, rather than paying two extra `has` + * lookups per import to share four lines. + */ +export function pythonSegmentAbsent(index: PythonFileIndex, segment: string): boolean { + if (segment === '' || segment.includes('\\')) return false; + if (index.byBasename.has(`${segment}.py`)) return false; + if (index.byInitParent.has(`${segment}/__init__.py`)) return false; + return true; +} diff --git a/gitnexus/src/core/ingestion/import-resolvers/python.ts b/gitnexus/src/core/ingestion/import-resolvers/python.ts index 2de11cc55..9914a7613 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/python.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/python.ts @@ -6,6 +6,12 @@ * This file contains the shared internal helper used by the strategy and tests. */ +import { + getPythonFileIndex, + importerBarePrefixes, + importerDirOf, + pythonSegmentAbsent, +} from './python-file-index.js'; import { tryResolveWithExtensions } from './utils.js'; /** @@ -51,8 +57,24 @@ export function resolvePythonImportInternal( const pathLike = importPath.replace(/\./g, '/'); if (pathLike.includes('/')) return null; - // Normalize for Windows backslashes - const importerDir = currentFile.replace(/\\/g, '/').split('/').slice(0, -1).join('/'); + // O(1) proof of absence, before any probing. Every probe below — the two + // proximity probes and the two per ancestor step — has the shape + // `/.py` or `//__init__.py`, and + // `pythonSegmentAbsent` answers "no file in the workspace has EITHER shape, + // for any prefix" in two Map lookups on the index the dotted tiers already + // build. That is `true` for `os`, `sys`, `django` and every other + // distribution the repo does not vendor — i.e. for most imports in most + // Python repos — and it retires the whole walk for them instead of running + // it to the workspace root. It is exact, not a filter: a miss here means + // every probe the walk would have issued was guaranteed to miss. + const index = getPythonFileIndex(allFiles); + if (pythonSegmentAbsent(index, pathLike)) return null; + + // One derivation, shared with the index's other per-directory memo — see + // `importerDirOf`. It replaced `split('/').slice(0, -1).join('/')`: identical + // for every input (a path with no separator has no directory, which is `''` + // both ways) without the per-import array of one element per path component. + const importerDir = importerDirOf(currentFile); // Proximity check — only applies when the importer lives in a subdirectory. // Root-level importers (importerDir === '') skip straight to the ancestor @@ -68,10 +90,12 @@ export function resolvePythonImportInternal( // importer's directory to find the module in an ancestor, preferring the closest match. // This prevents cross-language misresolution (e.g., Python `from middleware import X` // resolving to a TypeScript middleware.ts via suffix matching). Issue #417. - const dirParts = importerDir.split('/'); - for (let i = dirParts.length - 1; i >= 0; i--) { - const ancestorDir = dirParts.slice(0, i).join('/'); - const prefix = ancestorDir ? `${ancestorDir}/` : ''; + // + // The prefixes come from `importerBarePrefixes`, built ONCE per importer + // directory per pass and stored in the same index consulted above. Rebuilding + // them here — `dirParts.slice(0, i).join('/')`, one array and one string per + // path component — was the last per-import ancestor walk left after #2913. + for (const prefix of importerBarePrefixes(index, importerDir)) { if (allFiles.has(`${prefix}${pathLike}/__init__.py`)) return `${prefix}${pathLike}/__init__.py`; if (allFiles.has(`${prefix}${pathLike}.py`)) return `${prefix}${pathLike}.py`; } diff --git a/gitnexus/src/core/ingestion/import-resolvers/ruby.ts b/gitnexus/src/core/ingestion/import-resolvers/ruby.ts index 4bf47d31f..b19a6b3c3 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/ruby.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/ruby.ts @@ -16,8 +16,8 @@ import { suffixResolve } from './utils.js'; */ export function resolveRubyImportInternal( importPath: string, - normalizedFileList: string[], - allFileList: string[], + normalizedFileList: readonly string[], + allFileList: readonly string[], index?: SuffixIndex, ): string | null { const pathParts = importPath.replace(/^\.\//, '').split('/').filter(Boolean); diff --git a/gitnexus/src/core/ingestion/import-resolvers/standard.ts b/gitnexus/src/core/ingestion/import-resolvers/standard.ts index 888e80208..4cc4c1c60 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/standard.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/standard.ts @@ -29,8 +29,8 @@ export const resolveImportPath = ( currentFile: string, importPath: string, allFiles: Set, - allFileList: string[], - normalizedFileList: string[], + allFileList: readonly string[], + normalizedFileList: readonly string[], resolveCache: Map, language: SupportedLanguages, tsconfigPaths: TsconfigPaths | null, diff --git a/gitnexus/src/core/ingestion/import-resolvers/utils.ts b/gitnexus/src/core/ingestion/import-resolvers/utils.ts index 6a033c1ee..baddb1b01 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/utils.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/utils.ts @@ -79,66 +79,295 @@ export function tryResolveWithExtensions( * etc. */ export interface SuffixIndex { - /** Exact suffix lookup (case-sensitive) */ + /** + * Exact suffix lookup (case-sensitive). + * + * The map behind this is built on the FIRST call and memoized — see + * `buildSuffixIndex`. All three maps are deferred; a consumer pays only for + * the questions it actually asks. + */ get(suffix: string): string | undefined; - /** Case-insensitive suffix lookup */ + /** + * Case-insensitive suffix lookup. + * + * Deferred like `get`, and — when `get` was asked first — DERIVED from that + * map rather than traversed for a second time. See `buildSuffixIndex`. + */ getInsensitive(suffix: string): string | undefined; - /** Get all files in a directory suffix */ - getFilesInDir(dirSuffix: string, extension: string): string[]; + /** + * Get all files in a directory suffix. + * + * `readonly` is the CONTRACT, and it is the contract for every implementation + * of this interface, not a description of any one of them: an implementation + * is free to return its own bucket by reference, so callers must treat the + * result as shared and never `sort`/`splice` it in place. The compiler now + * refuses that at the call site. Whether a given implementation shares or + * copies is its own business and documented where it is built — + * `buildSuffixIndex` shares, the root-anchored parity index in + * `languages/php/import-target.ts` returns a filtered copy. + * + * Implementations that memoize should note the directory map behind this may + * be built on the FIRST call rather than up front, so a caller that never + * asks a directory question never pays for it — see `buildSuffixIndex`. + */ + getFilesInDir(dirSuffix: string, extension: string): readonly string[]; } -export function buildSuffixIndex(normalizedFileList: string[], allFileList: string[]): SuffixIndex { - // Map: normalized suffix -> original file path - const exactMap = new Map(); - // Map: lowercase suffix -> original file path - const lowerMap = new Map(); - // Map: directory suffix -> list of file paths in that directory - const dirMap = new Map(); +export interface SuffixIndexOptions { + /** + * Promise from the caller that `normalizedFileList[i] === normalizedFileList[i].toLowerCase()` + * for every `i` — i.e. the "normalized" list is a LOWERCASED file list, not + * merely a slash-normalized one. + * + * `import-resolvers/pass-cache.ts` is the one caller that can make it: it + * builds `normalizedFileList` as `allFileList.map((f) => f.toLowerCase())`. + * Every suffix of an all-lowercase path is itself lowercase, so + * `suffix.toLowerCase() === suffix` and the case-folded map came out a + * byte-identical copy of the exact one — same keys, same values, same + * insertion order. Measured 14.00 MiB at 32 000 paths, 29.8% of the retained + * `ImportPassCache` — and one `ImportPassCache` is built per ts-family + * adapter per pass, so the waste was carried once for each of them. + * + * With this set, `getInsensitive` reads the exact map directly instead. It is + * the same map the derivation below would have produced, so this is a skipped + * copy and not a second lookup rule — see `getLowerMap`. + * + * Setting it over a list that is NOT all-lowercase is a behaviour change, not + * an optimization: `getInsensitive` would then answer case-sensitively. + */ + readonly alreadyLowercased?: boolean; +} - for (let i = 0; i < normalizedFileList.length; i++) { - const normalized = normalizedFileList[i]; - const original = allFileList[i]; - const parts = normalized.split('/'); +export function buildSuffixIndex( + normalizedFileList: readonly string[], + allFileList: readonly string[], + options?: SuffixIndexOptions, +): SuffixIndex { + const alreadyLowercased = options?.alreadyLowercased === true; - // Index all suffixes: "a/b/c.java" -> ["c.java", "b/c.java", "a/b/c.java"] - for (let j = parts.length - 1; j >= 0; j--) { - const suffix = parts.slice(j).join('/'); - // Only store first match (longest path wins for ambiguous suffixes) - if (!exactMap.has(suffix)) { - exactMap.set(suffix, original); + /** + * Map: normalized suffix -> original file path. + * + * DEFERRED, like `dirMap` below and for the same reason (#2903 extended to + * the two suffix maps). Several consumers on the ScopeResolver path ask only + * ONE of the two suffix questions and were paying for both: + * + * - `languages/java/import-target.ts` and the no-csproj leg of + * `languages/csharp/import-target.ts` call `get` and never + * `getInsensitive` — measured 49.98 MiB dead of a 100.82 MiB Java index + * at 32 000 paths (49.6%), against a gated ceiling of 146.9 MiB; + * - `languages/php/import-target.ts` calls `getInsensitive` and never `get` + * — 34.49 MiB of 69.85 MiB (49.4%). + * + * Ruby, the csproj leg of C#, `group/extractors/include-extractor.ts` and + * `suffixResolve` below read both, and all four read `get` FIRST (they are + * written `get(s) || getInsensitive(s)`), which is what makes the derivation + * in `getLowerMap` the cheap order rather than the expensive one. + */ + let exactMap: Map | null = null; + + const getExactMap = (): Map => { + if (exactMap !== null) return exactMap; + const built = new Map(); + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + const original = allFileList[i]; + + // Index all suffixes: "a/b/c.java" -> ["c.java", "b/c.java", "a/b/c.java"]. + // + // Walked as slash offsets into `normalized` rather than as + // `normalized.split('/')` + `parts.slice(j).join('/')`: the slice of the + // ORIGINAL string is byte-identical to the re-joined parts (no separator + // is invented or dropped — verified over 361 865 suffix strings including + // leading, doubled and trailing slashes), and it allocates one string + // instead of a parts array, a slice array and a joined string per suffix. + // Measured 357.4 ms -> 264.5 ms at 32 000 paths. + let slash = normalized.lastIndexOf('/'); + while (slash >= 0) { + const suffix = normalized.slice(slash + 1); + // Only store first match (longest path wins for ambiguous suffixes) + if (!built.has(suffix)) built.set(suffix, original); + // A path may begin with '/', whose suffix is the whole string below. + if (slash === 0) break; + slash = normalized.lastIndexOf('/', slash - 1); } - const lower = suffix.toLowerCase(); - if (!lowerMap.has(lower)) { - lowerMap.set(lower, original); + // j = 0 — the whole path, which the slash walk cannot emit. + if (!built.has(normalized)) built.set(normalized, original); + } + exactMap = built; + return built; + }; + + /** + * Map: lowercase suffix -> original file path. + * + * Deferred, and when the exact map already exists DERIVED from it instead of + * traversed for: one pass over that map's DISTINCT keys rather than a second + * pass over every (file × depth) suffix. Measured 330.3 ms total (200.6 build + * + 129.7 derive) against 388.8 ms for the single fused traversal that built + * both eagerly — so the two-map consumers get cheaper too, which per-map + * laziness on its own does not (407.1 ms, a second full traversal). + * + * The derivation is EQUAL, not approximate, and the argument is short. Let + * the fused loop's global order be the pairs (suffix, file) it visited. For a + * lowercase key L, let p be the first position whose suffix lowercases to L — + * the entry today's `lowerMap` keeps. Nothing before p carries that suffix + * spelled ANY way, so p is also the first occurrence of its exact spelling + * and is therefore in the exact map, holding that same file. Exact-map + * insertion order is by first-occurrence position, so among the exact keys + * folding to L, p's is reached first and first-wins keeps it. Insertion order + * of the derived map is the order of those p's, which is the order today's + * `lowerMap` inserts L. Verified rather than only argued: byte-equal keys, + * values and order over 968 418 entries across bench-shaped, PascalCase, + * case-colliding, deep-monorepo, Unicode-adversarial and 400 seeded-fuzz + * corpora. + * + * When `getInsensitive` is asked FIRST (PHP), there is nothing to derive + * from, so it is built straight — one traversal, one map, which is the point. + * Asking `get` afterwards would then cost the second traversal; no consumer + * does, and the fallback stays correct if one ever starts. + */ + let lowerMap: Map | null = null; + + const getLowerMap = (): Map => { + // Over an already-lowercased file list the derivation is the identity, so + // the exact map IS the case-folded map. Skip the copy. + if (alreadyLowercased) return getExactMap(); + if (lowerMap !== null) return lowerMap; + + const built = new Map(); + if (exactMap !== null) { + for (const [suffix, original] of exactMap) { + const lower = suffix.toLowerCase(); + if (!built.has(lower)) built.set(lower, original); } + lowerMap = built; + return built; } - // Index directory membership - const lastSlash = normalized.lastIndexOf('/'); - if (lastSlash >= 0) { - // Build all directory suffixes - const dirParts = parts.slice(0, -1); - const fileName = parts[parts.length - 1]; - const ext = fileName.substring(fileName.lastIndexOf('.')); + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + const original = allFileList[i]; + let slash = normalized.lastIndexOf('/'); + while (slash >= 0) { + const lower = normalized.slice(slash + 1).toLowerCase(); + if (!built.has(lower)) built.set(lower, original); + if (slash === 0) break; + slash = normalized.lastIndexOf('/', slash - 1); + } + const whole = normalized.toLowerCase(); + if (!built.has(whole)) built.set(whole, original); + } + lowerMap = built; + return built; + }; - for (let j = dirParts.length - 1; j >= 0; j--) { - const dirSuffix = dirParts.slice(j).join('/'); - const key = `${dirSuffix}:${ext}`; - let list = dirMap.get(key); + /** + * Map: `${directory suffix}:${extension}` -> file paths in that directory. + * + * DEFERRED, not dropped (#2903). This is the array-valued map of the three + * and by far the most expensive: one entry — and one array push — per file + * per directory component, so O(files × depth) in entries AND in array + * churn. Measured on the 32k-path arms of `bench/import-target/`, it is + * ~15% of the retained C# index and ~19% of the retained Ruby one. + * + * Only `getFilesInDir` reads it, and only four call sites reach that: + * `import-resolvers/{php,csharp,jvm}.ts` and `import-resolvers/configs/ + * python.ts`. Every other consumer of this index — `workspace-file-index.ts` + * serving Ruby, `languages/typescript/scope-resolver.ts`, + * `languages/vue/import-target.ts`, `group/extractors/include-extractor.ts` + * — asks only suffix questions and was paying the whole footprint for a map + * it never touched. Since these indexes are now retained for a whole + * resolution pass rather than rebuilt per import (#2877-#2880), that is + * retained memory against the #2649 kernel-scale OOM constraint. + * + * `null` until the first `getFilesInDir`; the MAP is memoized, not the + * decision to build it, so a repeated miss cannot rebuild it. Building it + * later is behaviour-identical because it is a pure function of + * `normalizedFileList` / `allFileList`, and it retains nothing new: every + * production caller already holds both arrays alive alongside the index + * (`WorkspaceFileIndex.normalized`/`.all`, the TS and Vue `PassCache`s, + * `IncludeExtractor.extract`'s locals). + */ + let dirMap: Map | null = null; + + const getDirMap = (): Map => { + if (dirMap !== null) return dirMap; + const built = new Map(); + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + const original = allFileList[i]; + const lastSlash = normalized.lastIndexOf('/'); + // A file at the repo root is in no directory suffix. + if (lastSlash < 0) continue; + + // The file name from its last '.', or the WHOLE file name when it carries + // none — `substring(-1)` clamps to 0, which is what the `parts` form + // (`fileName.substring(fileName.lastIndexOf('.'))`) spelled. A '.' in a + // DIRECTORY is not an extension, hence `dot > lastSlash` rather than + // `dot >= 0`. + const dot = normalized.lastIndexOf('.'); + const ext = dot > lastSlash ? normalized.slice(dot) : normalized.slice(lastSlash + 1); + + // Every directory suffix of `normalized.slice(0, lastSlash)`, shortest + // first — the order `for (j = dirParts.length - 1; j >= 0; j--)` emitted, + // and load-bearing: `php.ts` returns `candidates[0]` of a bucket, so a + // reordered bucket is a behaviour change, not a wash. + // + // Walked as slash offsets into `normalized`, the same rewrite `getExactMap` + // above documents and for the same reason — a slice of the ORIGINAL string + // is byte-identical to the re-joined parts, and it allocates one string per + // suffix instead of a parts array, a slice array and a joined string per + // suffix. This is the map where it pays most: one entry, one array push AND + // one key per file per directory component, the "by far the most expensive" + // of the three. Measured 226.9 ms -> 173.1 ms at 32 000 paths averaging + // ~10 directory components (min of 9, both loops alternating in one + // process). Verified rather than argued, over a 32 000-path corpus + // carrying absolute paths, doubled separators (`a//b`), backslash paths, + // root-level and extensionless files, dotted directories and trailing + // separators: 272 956 keys and 329 361 bucket entries came out with + // identical key sets in identical INSERTION order and identical buckets + // element-for-element, and 767 732 probes of the built index — every + // emitted (directory, extension) pair plus a wrong-extension and a + // one-level-deeper miss for each — answered exactly as the `parts` form's + // map did. 0 differences. + // + // `slash < 0` is the whole directory, which no slash search can emit and + // the only suffix a one-component directory has. + let start = lastSlash; + while (start >= 0) { + const slash = start > 0 ? normalized.lastIndexOf('/', start - 1) : -1; + const key = `${normalized.slice(slash + 1, lastSlash)}:${ext}`; + let list = built.get(key); if (!list) { list = []; - dirMap.set(key, list); + built.set(key, list); } list.push(original); + start = slash; } } - } + dirMap = built; + return built; + }; return { - get: (suffix: string) => exactMap.get(suffix), - getInsensitive: (suffix: string) => lowerMap.get(suffix.toLowerCase()), + get: (suffix: string) => getExactMap().get(suffix), + getInsensitive: (suffix: string) => getLowerMap().get(suffix.toLowerCase()), + // THIS implementation shares: it hands back `dirMap`'s own bucket rather + // than a copy. The map is built on first query and then held for the whole + // pass, so the window in which a mutating caller could corrupt later + // imports is the whole pass — which is why the interface makes the result + // `readonly` and the compiler refuses the mutation at the call site. + // + // Sharing beats copying because no caller keeps the array: two only measure + // it and two build a fresh array from it, so a defensive copy would + // allocate a whole bucket per import on the path this index exists to keep + // flat. `package-dir-index.ts` reached the same conclusion the same way — + // read-only containers, plus one copy where a bucket genuinely LEAVES + // (`sortedRootFiles`), which is the case `configs/swift.ts` is in. getFilesInDir: (dirSuffix: string, extension: string) => { - return dirMap.get(`${dirSuffix}:${extension}`) || []; + return getDirMap().get(`${dirSuffix}:${extension}`) || []; }, }; } @@ -148,8 +377,8 @@ export function buildSuffixIndex(normalizedFileList: string[], allFileList: stri */ export function suffixResolve( pathParts: string[], - normalizedFileList: string[], - allFileList: string[], + normalizedFileList: readonly string[], + allFileList: readonly string[], index?: SuffixIndex, ): string | null { if (index) { diff --git a/gitnexus/src/core/ingestion/import-resolvers/workspace-file-index.ts b/gitnexus/src/core/ingestion/import-resolvers/workspace-file-index.ts index 15910f21c..ad5c882ec 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/workspace-file-index.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/workspace-file-index.ts @@ -1,6 +1,6 @@ /** * Per-file-set workspace index for the import-target resolvers that need the - * shared `SuffixIndex` (C#, Ruby). + * shared `SuffixIndex` (C#, Java, PHP, Ruby). * * The scope-resolution orchestrator passes the SAME `allFilePaths` Set object to * every `resolveImportTarget` call in a pass (`pipeline/run.ts` builds it once), @@ -12,28 +12,55 @@ * per call and silently restores the O(imports × files) behaviour — the exact * bug PR #1918 shipped and had to fix in review (P1). * - * Two layers guard that, and they guard different things: + * Three layers guard that, and they guard different things: * - ADAPTER BOUNDARY, where the defensive-copy hazard actually lives: - * `test/integration/-import-index-reuse.test.ts` (csharp and ruby for - * this index; go, dart, kotlin and python for the sibling ones) resolves - * through `ScopeResolver.resolveImportTarget` — the orchestrator - * adapter — and asserts the file set is traversed once per run (twice for - * C#, which builds two indexes). Kotlin and Python instead count index - * BUILDS from production (`languages//index-stats.ts`); either way, a - * copy inserted in an adapter fails these. + * `test/integration/*-import-index-reuse.test.ts` resolves through + * `ScopeResolver.resolveImportTarget` — the orchestrator adapter — and + * pins the EXACT number of times a run traverses the file set, one file per + * covered language over that language's own corpus. (The expected count is + * per language and legitimately differs: it is however many times the + * adapter derives something from the Set — two indexes, or an index plus the + * mutable copy the ts-family context wants.) All of them count traversals + * of a `CountingSet` (`test/helpers/counting-file-set.ts`): one instrument, + * no production surface, and it catches both the per-import rebuild and a + * scan reintroduced beside a reused index (#2909). + * - EVERY REGISTERED LANGUAGE, at the same boundary but as one property rather + * than one corpus per language: `test/unit/scope-resolution/import-target-index-reuse.contract.test.ts` + * drives each entry of `SCOPE_RESOLVERS` and asserts the traversal count for + * many imports equals the count for two. A new language cannot skip it, and + * the enforcement is a test rather than a roster anyone maintains: that + * file's inventory arm compares `SCOPE_RESOLVERS`' keys against its own + * fixture table and fails on a registered resolver that has neither a + * fixture nor an exemption, and its next arm pins the exemption map empty. * - RESOLVER LEVEL: `test/unit/scope-resolution/import-target-index-parity.test.ts` * calls the resolvers directly, so it never crosses the adapter boundary and * a copy there leaves it green. What it catches is a rescan reintroduced * INSIDE a resolver, by counting how many times the Set is iterated. */ +import { perFileSet } from './per-file-set.js'; import { buildSuffixIndex, type SuffixIndex } from './utils.js'; +/** + * `normalized` and `all` are `readonly string[]`, and — like + * `SuffixIndex.getFilesInDir` — that is the CONTRACT rather than a description + * of the arrays: they are built once and then held for the whole pass, so an + * in-place `sort`/`splice`/`reverse` would corrupt every later import in that + * pass, and these two are the largest shared arrays here (one element per file, + * read by C#, Java, PHP and Ruby). `readonly` on the field is what makes the + * compiler refuse the mutation at the call site instead of leaving it to a + * comment. `ImportPassCache` (`pass-cache.ts`) states the same contract the + * same way for the ts-family lists. + * + * The positional pairing is load-bearing too and depends on it: `csharp.ts` + * caches POSITIONS into `normalized` and reads the answer out of `all`, so a + * reordering of either array alone silently re-points every cached position. + */ export interface WorkspaceFileIndex { /** Every path, backslashes normalized to `/`. Parallel to `all`. */ - readonly normalized: string[]; + readonly normalized: readonly string[]; /** Every path, exactly as it appears in the Set. Parallel to `normalized`. */ - readonly all: string[]; + readonly all: readonly string[]; /** Segment-suffix → first file (in Set iteration order) carrying that suffix. */ readonly index: SuffixIndex; /** @@ -46,27 +73,22 @@ export interface WorkspaceFileIndex { readonly normToRaw: Map; } -const WORKSPACE_FILE_INDEX_CACHE = new WeakMap, WorkspaceFileIndex>(); +export const getWorkspaceFileIndex = perFileSet( + (allFilePaths: ReadonlySet): WorkspaceFileIndex => { + const all = [...allFilePaths]; + const normalized = all.map((f) => f.replace(/\\/g, '/')); + const normToRaw = new Map(); + for (let i = 0; i < normalized.length; i++) { + // First wins, mirroring the `for (const raw of allFilePaths)` scans this + // replaces: they returned on the first match in iteration order. + if (!normToRaw.has(normalized[i])) normToRaw.set(normalized[i], all[i]); + } -export function getWorkspaceFileIndex(allFilePaths: ReadonlySet): WorkspaceFileIndex { - const cached = WORKSPACE_FILE_INDEX_CACHE.get(allFilePaths); - if (cached !== undefined) return cached; - - const all = [...allFilePaths]; - const normalized = all.map((f) => f.replace(/\\/g, '/')); - const normToRaw = new Map(); - for (let i = 0; i < normalized.length; i++) { - // First wins, mirroring the `for (const raw of allFilePaths)` scans this - // replaces: they returned on the first match in iteration order. - if (!normToRaw.has(normalized[i])) normToRaw.set(normalized[i], all[i]); - } - - const built: WorkspaceFileIndex = { - normalized, - all, - index: buildSuffixIndex(normalized, all), - normToRaw, - }; - WORKSPACE_FILE_INDEX_CACHE.set(allFilePaths, built); - return built; -} + return { + normalized, + all, + index: buildSuffixIndex(normalized, all), + normToRaw, + }; + }, +); diff --git a/gitnexus/src/core/ingestion/languages/c/import-target.ts b/gitnexus/src/core/ingestion/languages/c/import-target.ts index 495846030..5590bb2e6 100644 --- a/gitnexus/src/core/ingestion/languages/c/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/c/import-target.ts @@ -1,4 +1,5 @@ import { dirname, join } from 'path'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; /** * A workspace file path pre-decomposed for the suffix-match fallback: @@ -28,26 +29,20 @@ interface CSuffixCandidate { * `WeakMap`-keyed so it is reclaimed with the pass (no cross-pass staleness). * Shared by C and C++ (`resolveCppImportTarget` delegates here). */ -const suffixIndexByPaths = new WeakMap, Map>(); - -function suffixIndex(allFilePaths: ReadonlySet): Map { - let index = suffixIndexByPaths.get(allFilePaths); - if (index === undefined) { - index = new Map(); - for (const original of allFilePaths) { - const normalized = original.replace(/\\/g, '/'); - const basename = normalized.slice(normalized.lastIndexOf('/') + 1); - let bucket = index.get(basename); - if (bucket === undefined) { - bucket = []; - index.set(basename, bucket); - } - bucket.push({ original, normalized, depth: normalized.split('/').length }); +const suffixIndex = perFileSet((allFilePaths: ReadonlySet) => { + const index = new Map(); + for (const original of allFilePaths) { + const normalized = original.replace(/\\/g, '/'); + const basename = normalized.slice(normalized.lastIndexOf('/') + 1); + let bucket = index.get(basename); + if (bucket === undefined) { + bucket = []; + index.set(basename, bucket); } - suffixIndexByPaths.set(allFilePaths, index); + bucket.push({ original, normalized, depth: normalized.split('/').length }); } return index; -} +}); /** * Resolve a C #include path to a file in the workspace. diff --git a/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts index af31ec572..c3f9fb36f 100644 --- a/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts @@ -8,6 +8,7 @@ import { cArityCompatibility, cMergeBindings, resolveCImportTarget } from './ind import { scanHeaderFiles } from './header-scan.js'; import { expandCWildcardNames, isStaticName, clearStaticNames } from './static-linkage.js'; import { applyCStaticLinkageSideChannel } from './capture-side-channel.js'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; /** * Per-pass memo of the augmented `#include`-resolution file set @@ -19,31 +20,26 @@ import { applyCStaticLinkageSideChannel } from './capture-side-channel.js'; * handing it a new set identity each time. Both `allFilePaths` (built once in * scope-resolution `run.ts`) and the header set (`loadResolutionConfig` * result) are stable per pass, so the union is built once and reused. - * `WeakMap`-keyed → reclaimed with the pass (no cross-pass staleness). + * Reclaimed with the pass (no cross-pass staleness). + * + * Two inputs, so two levels of `perFileSet` composed rather than a second + * primitive: the outer memo's value is the inner memo, and a function is an + * object, which is all `T extends object` asks for. + * + * The MEMO stays private to this file even though the C++ resolver's twin is + * byte-identical. The augmented set's IDENTITY is load-bearing downstream — + * C++ delegates to `resolveCImportTarget`, whose `suffixIndex` memo is keyed on + * exactly this set — so one memo shared across the two languages would hand + * each the other's index. Same builder-shared/memo-separate rule as + * `import-resolvers/pass-cache.ts`. */ -const augmentedPathsByPass = new WeakMap< - ReadonlySet, - WeakMap, ReadonlySet> ->(); - -function augmentedFilePaths( - allFilePaths: ReadonlySet, - headerPaths: ReadonlySet, -): ReadonlySet { - let byHeaders = augmentedPathsByPass.get(allFilePaths); - if (byHeaders === undefined) { - byHeaders = new WeakMap(); - augmentedPathsByPass.set(allFilePaths, byHeaders); - } - let augmented = byHeaders.get(headerPaths); - if (augmented === undefined) { +const augmentedFilePathsFor = perFileSet((allFilePaths: ReadonlySet) => + perFileSet((headerPaths: ReadonlySet): ReadonlySet => { const set = new Set(allFilePaths); for (const h of headerPaths) set.add(h); - augmented = set; - byHeaders.set(headerPaths, augmented); - } - return augmented; -} + return set; + }), +); /** * C `ScopeResolver` registered in `SCOPE_RESOLVERS` and consumed by @@ -94,7 +90,7 @@ export const cScopeResolver: ScopeResolver = { return resolveCImportTarget( targetRaw, fromFile, - augmentedFilePaths(allFilePaths, headerPaths), + augmentedFilePathsFor(allFilePaths)(headerPaths), ); } return resolveCImportTarget(targetRaw, fromFile, allFilePaths); diff --git a/gitnexus/src/core/ingestion/languages/c/static-linkage.ts b/gitnexus/src/core/ingestion/languages/c/static-linkage.ts index 2cc195205..a81398354 100644 --- a/gitnexus/src/core/ingestion/languages/c/static-linkage.ts +++ b/gitnexus/src/core/ingestion/languages/c/static-linkage.ts @@ -1,4 +1,5 @@ import type { ParsedFile, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; /** * Per-file set of function names declared with `static` storage class. @@ -59,27 +60,23 @@ export function clearStaticNames(): void { * thousands of resolved includes) that is ~10^10+ comparisons on a single * thread — the dominant term in the scope-resolution finalize grind. * - * Building the lookup once collapses it to O(R_include + F). `WeakMap`-keyed - * on the array so the index is reclaimed with the pass — no cross-pass + * Building the lookup once collapses it to O(R_include + F). `perFileSet` keys + * on the array identity so the index is reclaimed with the pass — no cross-pass * staleness (mirrors the {@link clearStaticNames} discipline for server-mode * / multi-repo reuse), and a fresh array transparently rebuilds. */ -const moduleScopeIndexByPass = new WeakMap>(); - -function moduleScopeIndex(parsedFiles: readonly ParsedFile[]): Map { - let index = moduleScopeIndexByPass.get(parsedFiles); - if (index === undefined) { - index = new Map(); +const moduleScopeIndex = perFileSet( + (parsedFiles: readonly ParsedFile[]): Map => { + const index = new Map(); // First-wins to preserve `Array.find` semantics (returns the first match). // `moduleScope` is unique per file in practice, so collisions are absent; // the guard only formalises identical behaviour to the prior `.find`. for (const p of parsedFiles) { if (!index.has(p.moduleScope)) index.set(p.moduleScope, p); } - moduleScopeIndexByPass.set(parsedFiles, index); - } - return index; -} + return index; + }, +); /** * Return the names visible through a C wildcard import (`#include`). diff --git a/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts index 9e528ce16..cfea8f39e 100644 --- a/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/cobol/scope-resolver.ts @@ -12,12 +12,71 @@ import path from 'node:path'; import type { ParsedFile } from 'gitnexus-shared'; import { SupportedLanguages } from 'gitnexus-shared'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { populateClassOwnedMembers } from '../../scope-resolution/scope/walkers.js'; import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; import { cobolProvider } from '../cobol.js'; // Copybook file extensions for COPY name resolution const COPYBOOK_EXTENSIONS = new Set(['.cpy', '.copybook']); +// COBOL source files, searched only after every copybook has missed. +const COBOL_SOURCE_EXTENSIONS = new Set(['.cbl', '.cob', '.cobol']); + +/** + * Uppercased-basename → first file carrying it, one map PER TIER, memoized on + * the `allFilePaths` Set identity (#2908). + * + * `resolveImportTarget` used to run two full workspace scans per `COPY` — one + * for the copybook tier, one for the source tier — each calling `path.extname` + * + `path.basename` + `toUpperCase` on every entry. A `COPY` of a member that + * lives outside the repo (the common case: vendor and system copybooks) missed + * in both, so both scans always ran to completion, making resolution + * O(copies × files). The orchestrator passes the SAME Set to every import in a + * pass (`pipeline/run.ts` builds it once), so a `WeakMap` keyed on that Set + * turns the scans into one build per run. + * + * Two tiers rather than one map is the tie-break, not a stylistic choice: a + * `.cpy`/`.copybook` hit beats a `.cbl`/`.cob`/`.cobol` hit even when the source + * file comes FIRST in Set-iteration order, which is exactly what collapsing the + * tiers into a single first-wins map would silently discard. Within a tier the + * first file in Set-iteration order wins, mirroring the `return` on first match + * in the scans this replaces. + * + * The per-file key is derived with the same `path.extname(fp).toLowerCase()` → + * `path.basename(fp, ext)` → `toUpperCase()` sequence the scans used, including + * its quirk: `path.basename` strips the suffix only on an exact, case-sensitive + * match, so `Foo.CPY` indexes under `FOO.CPY` rather than `FOO`. Node's `path` + * stays in the loop for the same reason — on POSIX it does not treat `\` as a + * separator, and hand-rolled slicing on `/` would start resolving backslash + * paths the scans never resolved. + */ +interface CobolCopyIndex { + /** `.cpy` / `.copybook` files — tier 1. */ + readonly copybooks: ReadonlyMap; + /** `.cbl` / `.cob` / `.cobol` files — tier 2. */ + readonly sources: ReadonlyMap; +} + +const getCobolCopyIndex = perFileSet((allFilePaths: ReadonlySet): CobolCopyIndex => { + const copybooks = new Map(); + const sources = new Map(); + // One pass builds both tiers: the two scans walked the same files and + // classified each by the same extension test. + for (const fp of allFilePaths) { + const ext = path.extname(fp).toLowerCase(); + const tier = COPYBOOK_EXTENSIONS.has(ext) + ? copybooks + : COBOL_SOURCE_EXTENSIONS.has(ext) + ? sources + : undefined; + if (tier === undefined) continue; + const basename = path.basename(fp, ext).toUpperCase(); + // First in Set-iteration order wins, as the scans' first-match `return` did. + if (!tier.has(basename)) tier.set(basename, fp); + } + + return { copybooks, sources }; +}); const cobolScopeResolver: ScopeResolver = { language: SupportedLanguages.Cobol, @@ -27,22 +86,9 @@ const cobolScopeResolver: ScopeResolver = { // ── Resolve COPY bookname to file path ───────────────────────────── resolveImportTarget: (targetRaw, _fromFile, allFilePaths) => { const upper = targetRaw.toUpperCase(); - // Check copybook files first - for (const fp of allFilePaths) { - const ext = path.extname(fp).toLowerCase(); - if (!COPYBOOK_EXTENSIONS.has(ext)) continue; - const basename = path.basename(fp, ext).toUpperCase(); - if (basename === upper) return fp; - } - // Also search COBOL source files (.cbl, .cob, .cobol) - const COBOL_SOURCE_EXTS = new Set(['.cbl', '.cob', '.cobol']); - for (const fp of allFilePaths) { - const ext = path.extname(fp).toLowerCase(); - if (!COBOL_SOURCE_EXTS.has(ext)) continue; - const basename = path.basename(fp, ext).toUpperCase(); - if (basename === upper) return fp; - } - return null; + const index = getCobolCopyIndex(allFilePaths); + // Copybooks first, then COBOL sources — the tier order IS the tie-break. + return index.copybooks.get(upper) ?? index.sources.get(upper) ?? null; }, // COBOL has no binding-merge rules beyond the default (local-first-then-imports). diff --git a/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts b/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts index c6b383bf0..e578b20d4 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts @@ -1,4 +1,5 @@ import type { ParsedFile, Scope, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { isCppInlineNamespaceScope } from './inline-namespaces.js'; /** @@ -283,24 +284,20 @@ export function isCppDefGloballyVisible(filePath: string, nodeId: string): boole * `parsedFiles` reference; the old `parsedFiles.find(...)` was therefore O(F) * per edge → O(R·F) overall (at kernel scale the ~25–30k `.h` headers are * classified C++, so this fires hard — the C twin in `c/static-linkage.ts`). - * Building the lookup once collapses it to O(R+F). `WeakMap`-keyed so it is - * reclaimed with the pass (no cross-pass staleness; mirrors - * {@link clearFileLocalNames}). + * Building the lookup once collapses it to O(R+F). `perFileSet` keys on the + * array identity so it is reclaimed with the pass (no cross-pass staleness; + * mirrors {@link clearFileLocalNames}). */ -const moduleScopeIndexByPass = new WeakMap>(); - -function moduleScopeIndex(parsedFiles: readonly ParsedFile[]): Map { - let index = moduleScopeIndexByPass.get(parsedFiles); - if (index === undefined) { - index = new Map(); +const moduleScopeIndex = perFileSet( + (parsedFiles: readonly ParsedFile[]): Map => { + const index = new Map(); // First-wins to preserve `Array.find` semantics (returns the first match). for (const p of parsedFiles) { if (!index.has(p.moduleScope)) index.set(p.moduleScope, p); } - moduleScopeIndexByPass.set(parsedFiles, index); - } - return index; -} + return index; + }, +); export function expandCppWildcardNames( targetModuleScope: ScopeId, diff --git a/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts index 13f21183e..9421975ea 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts @@ -48,6 +48,7 @@ import { resolveCppReceiverMember, } from './member-lookup.js'; import { stripCppSpecifiers } from './interpret.js'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; /** A pointee worth binding: a bare identifier, not `T**`, `T[]`, `A::B` or a * template spelling. Hoisted — a literal here would mint a fresh RegExp on @@ -61,32 +62,25 @@ const CPP_SIMPLE_POINTEE_RE = /^[A-Za-z_]\w*$/; * a fresh ~F-entry `Set` on every call AND defeated the shared * `resolveCImportTarget` suffix-index memo (in `c/import-target.ts`) by handing * it a new set identity each time. Both inputs are stable per pass, so the - * union is built once and reused. `WeakMap`-keyed → reclaimed with the pass. - * (Twin of the C resolver's `augmentedFilePaths`.) + * union is built once and reused. Reclaimed with the pass. + * + * Two inputs, so two levels of `perFileSet` composed rather than a second + * primitive: the outer memo's value is the inner memo, and a function is an + * object, which is all `T extends object` asks for. + * + * (Twin of the C resolver's `augmentedFilePathsFor`.) The two memos stay + * SEPARATE deliberately. C++ delegates to `resolveCImportTarget`, whose + * `suffixIndex` memo is keyed on the augmented set, so a single memo shared + * with C would hand each language the other's index — same + * builder-shared/memo-separate rule as `import-resolvers/pass-cache.ts`. */ -const augmentedPathsByPass = new WeakMap< - ReadonlySet, - WeakMap, ReadonlySet> ->(); - -function augmentedFilePaths( - allFilePaths: ReadonlySet, - headerPaths: ReadonlySet, -): ReadonlySet { - let byHeaders = augmentedPathsByPass.get(allFilePaths); - if (byHeaders === undefined) { - byHeaders = new WeakMap(); - augmentedPathsByPass.set(allFilePaths, byHeaders); - } - let augmented = byHeaders.get(headerPaths); - if (augmented === undefined) { +const augmentedFilePathsFor = perFileSet((allFilePaths: ReadonlySet) => + perFileSet((headerPaths: ReadonlySet): ReadonlySet => { const set = new Set(allFilePaths); for (const h of headerPaths) set.add(h); - augmented = set; - byHeaders.set(headerPaths, augmented); - } - return augmented; -} + return set; + }), +); /** * C++ `ScopeResolver` registered in `SCOPE_RESOLVERS` and consumed by @@ -128,7 +122,7 @@ export const cppScopeResolver: ScopeResolver = { return resolveCppImportTarget( targetRaw, fromFile, - augmentedFilePaths(allFilePaths, headerPaths), + augmentedFilePathsFor(allFilePaths)(headerPaths), ); } return resolveCppImportTarget(targetRaw, fromFile, allFilePaths); diff --git a/gitnexus/src/core/ingestion/languages/csharp/import-target.ts b/gitnexus/src/core/ingestion/languages/csharp/import-target.ts index 18d406ba6..68f1e484d 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/import-target.ts @@ -31,6 +31,7 @@ import { firstFileDirectlyInPkgDir, type PackageDirIndex, } from '../../import-resolvers/package-dir-index.js'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { csharpSuffixFallbackAllowed } from '../../csharp-namespace-gate.js'; export interface CsharpResolveContext { @@ -46,15 +47,10 @@ export interface CsharpResolveContext { * `import-resolvers/package-dir-index.ts`), which the no-csproj path calls once * for the direct match and then up to once per stripped namespace prefix. */ -const csharpDirIndexCache = new WeakMap, PackageDirIndex>(); - -function getCsharpDirIndex(allFilePaths: ReadonlySet): PackageDirIndex { - const cached = csharpDirIndexCache.get(allFilePaths); - if (cached) return cached; - const built = buildPackageDirIndex(allFilePaths, (normalized) => normalized.endsWith('.cs')); - csharpDirIndexCache.set(allFilePaths, built); - return built; -} +const getCsharpDirIndex = perFileSet( + (allFilePaths: ReadonlySet): PackageDirIndex => + buildPackageDirIndex(allFilePaths, (normalized) => normalized.endsWith('.cs')), +); export function resolveCsharpImportTarget( parsedImport: ParsedImport, @@ -69,12 +65,11 @@ export function resolveCsharpImportTarget( const csharpConfigs = ctx.csharpConfigs ?? []; if (csharpConfigs.length > 0) { - const { normalized, all, index } = getWorkspaceFileIndex(ctx.allFilePaths); + const { index } = getWorkspaceFileIndex(ctx.allFilePaths); const fromCsproj = resolveCSharpImportInternal( targetRaw, [...csharpConfigs], - normalized, - all, + ctx.allFilePaths, index, evidence, ); diff --git a/gitnexus/src/core/ingestion/languages/dart/import-target.ts b/gitnexus/src/core/ingestion/languages/dart/import-target.ts index 371ff1f11..fd6c5224a 100644 --- a/gitnexus/src/core/ingestion/languages/dart/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/dart/import-target.ts @@ -13,6 +13,7 @@ * `targetRaw` arrives already quote-stripped from `interpretDartImport`. */ +import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { DART_HERITAGE_PREFIX } from './interpret.js'; /** @@ -35,11 +36,7 @@ interface DartFileIndex { readonly byBasename: Map; } -const DART_FILE_INDEX_CACHE = new WeakMap, DartFileIndex>(); - -function getDartFileIndex(allFilePaths: ReadonlySet): DartFileIndex { - const cached = DART_FILE_INDEX_CACHE.get(allFilePaths); - if (cached !== undefined) return cached; +const getDartFileIndex = perFileSet((allFilePaths: ReadonlySet): DartFileIndex => { const byBasename = new Map(); for (const fp of allFilePaths) { const base = fp.slice(fp.lastIndexOf('/') + 1); @@ -50,10 +47,8 @@ function getDartFileIndex(allFilePaths: ReadonlySet): DartFileIndex { } bucket.push(fp); } - const built: DartFileIndex = { byBasename }; - DART_FILE_INDEX_CACHE.set(allFilePaths, built); - return built; -} + return { byBasename }; +}); /** First file (in Set-iteration order) that IS `candidate` or ends with * `/` — the exact predicate of the scans this replaces. */ diff --git a/gitnexus/src/core/ingestion/languages/go/import-target.ts b/gitnexus/src/core/ingestion/languages/go/import-target.ts index ceb3fa62b..847af0c09 100644 --- a/gitnexus/src/core/ingestion/languages/go/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/go/import-target.ts @@ -5,6 +5,7 @@ import { sortedRootFiles, type PackageDirIndex, } from '../../import-resolvers/package-dir-index.js'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; /** * Resolve a Go import path to ALL .go files in the matching package directory. @@ -56,6 +57,11 @@ export function resolveGoImportTarget( return null; } +/** Go packages exclude `_test.go` files: they are a separate package. */ +function isGoPackageFile(normalized: string): boolean { + return normalized.endsWith('.go') && !normalized.endsWith('_test.go'); +} + /** * Package index over the file set, memoized on the Set's identity (#2877). * @@ -69,20 +75,10 @@ export function resolveGoImportTarget( * is built once per run. `resolveGoImportTarget` must therefore never copy the * Set before this point — see `import-resolvers/workspace-file-index.ts`. */ -const GO_PACKAGE_INDEX_CACHE = new WeakMap, PackageDirIndex>(); - -/** Go packages exclude `_test.go` files: they are a separate package. */ -function isGoPackageFile(normalized: string): boolean { - return normalized.endsWith('.go') && !normalized.endsWith('_test.go'); -} - -function getGoPackageIndex(allFilePaths: ReadonlySet): PackageDirIndex { - const cached = GO_PACKAGE_INDEX_CACHE.get(allFilePaths); - if (cached !== undefined) return cached; - const built = buildPackageDirIndex(allFilePaths, isGoPackageFile); - GO_PACKAGE_INDEX_CACHE.set(allFilePaths, built); - return built; -} +const getGoPackageIndex = perFileSet( + (allFilePaths: ReadonlySet): PackageDirIndex => + buildPackageDirIndex(allFilePaths, isGoPackageFile), +); function findRootPackageFiles(allFilePaths: ReadonlySet): string[] { return sortedRootFiles(getGoPackageIndex(allFilePaths)); diff --git a/gitnexus/src/core/ingestion/languages/java/import-target.ts b/gitnexus/src/core/ingestion/languages/java/import-target.ts index b78b6369b..74ba8182b 100644 --- a/gitnexus/src/core/ingestion/languages/java/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/java/import-target.ts @@ -8,27 +8,79 @@ * 4. Progressive prefix stripping for non-standard layouts * * Returns `null` for unresolvable / JDK imports. + * + * ## Why the scans are gone (#2908) + * + * Every leg above used to be answered by `for (const raw of ctx.allFilePaths)`, + * and the stripping loop ran that scan again per stripped segment — so one + * unresolvable `import a.b.c.D;` (the COMMON case: JDK and third-party imports + * run the whole cascade to completion) cost four full workspace passes. This is + * byte-for-byte the shape C# carried until #2878; both now read the same two + * per-file-set indexes, memoized on the Set's identity: + * + * - `getWorkspaceFileIndex` — `normToRaw` (whole-path lookup) and `index` + * (segment-suffix lookup); + * - `getJavaDirIndex` — `firstFileDirectlyInPkgDir`'s package-directory index. + * + * ## The tie-breaks the scans encoded, and where they now live + * + * 1. The first pass `break`s on an exact whole-path hit but keeps scanning + * otherwise, then returns `exactFile ?? suffixFile ?? directoryChild`. So an + * exact match wins over a suffix or directory-child match found EARLIER in + * iteration order — hence `normToRaw` before `index`, which conflates the + * two (see `resolveDirectMatch`). + * 2. The stripping loop instead `return`s mid-scan on `f === tailFile || + * f.endsWith(tailSuffix)`, i.e. at the first hit of EITHER, and only returns + * its directory child after the scan completes. So file/suffix beats + * directory child within one `skip` level regardless of order, and the + * conflated `index.get` is the CORRECT lookup there (see + * `resolveByProgressiveStripping`). + * 3. Wildcard imports drop their trailing `.*` before resolution, so + * `com.example.*` resolves as the package directory. + * 4. `.java` filter and backslash normalization, with the RAW path returned: + * the indexes normalize for their keys and hand back the raw Set member, and + * only a `.java` file can carry a `…/.java` suffix key, so the + * extension filter is implied on the file/suffix legs and explicit in the + * directory index's `accept`. + * 5. The directory-child leg matched on the FIRST `'/' + pathLike + '/'` + * occurrence, so `com/example/com/example/Deep.java` does NOT answer + * `com.example`. `firstFileDirectlyInPkgDir` encodes exactly that rule (see + * the header of `import-resolvers/package-dir-index.ts`). */ import type { ParsedImport, WorkspaceIndex } from 'gitnexus-shared'; +import { + getWorkspaceFileIndex, + type WorkspaceFileIndex, +} from '../../import-resolvers/workspace-file-index.js'; +import { + buildPackageDirIndex, + firstFileDirectlyInPkgDir, + type PackageDirIndex, +} from '../../import-resolvers/package-dir-index.js'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; export interface JavaResolveContext { readonly fromFile: string; readonly allFilePaths: ReadonlySet; } +/** + * Package-directory index over the `.java` files, memoized on the Set's + * identity. Feeds `firstFileDirectlyInPkgDir`, which is called once for the + * direct match and then up to once per stripped package prefix. + */ +const getJavaDirIndex = perFileSet( + (allFilePaths: ReadonlySet): PackageDirIndex => + buildPackageDirIndex(allFilePaths, (normalized) => normalized.endsWith('.java')), +); + export function resolveJavaImportTarget( parsedImport: ParsedImport, workspaceIndex: WorkspaceIndex, ): string | null { - const ctx = workspaceIndex as JavaResolveContext | undefined; - if ( - ctx === undefined || - typeof (ctx as { fromFile?: unknown }).fromFile !== 'string' || - !((ctx as { allFilePaths?: unknown }).allFilePaths instanceof Set) - ) { - return null; - } + const ctx = narrowContext(workspaceIndex); + if (ctx === null) return null; if (parsedImport.kind === 'dynamic-unresolved') return null; if (parsedImport.targetRaw === null || parsedImport.targetRaw === '') return null; @@ -40,69 +92,87 @@ export function resolveJavaImportTarget( // Package path: `com.example.User` → `com/example/User` const pathLike = target.replace(/\./g, '/'); - const suffix = `/${pathLike}`; - let exactFile: string | null = null; - let suffixFile: string | null = null; - let directoryChild: string | null = null; - const dirPrefix = `${pathLike}/`; - const suffixDirPrefix = `/${dirPrefix}`; + const ws = getWorkspaceFileIndex(ctx.allFilePaths); + const dirs = getJavaDirIndex(ctx.allFilePaths); - for (const raw of ctx.allFilePaths) { - const f = raw.replace(/\\/g, '/'); - if (!f.endsWith('.java')) continue; - if (f === `${pathLike}.java`) { - exactFile = raw; - break; - } - if (suffixFile === null && f.endsWith(`${suffix}.java`)) { - suffixFile = raw; - } - if (directoryChild === null) { - const atRoot = f.startsWith(dirPrefix); - const atNested = f.includes(suffixDirPrefix); - if (atRoot || atNested) { - const idx = atRoot ? 0 : f.indexOf(suffixDirPrefix) + 1; - const after = f.slice(idx + dirPrefix.length); - if (after.length > 0 && !after.includes('/')) { - directoryChild = raw; - } - } - } - } - - if (exactFile !== null) return exactFile; - if (suffixFile !== null) return suffixFile; - if (directoryChild !== null) return directoryChild; + const direct = resolveDirectMatch(ws, dirs, pathLike); + if (direct !== null) return direct; // Progressive prefix stripping — handles `import com.example.User;` // in a repo laid out `User.java` (no `com/example/` prefix). + return resolveByProgressiveStripping(ws, dirs, pathLike); +} + +/** + * `WorkspaceIndex` is an opaque `unknown` placeholder in the shared contract; + * the orchestrator hands us a `JavaResolveContext`-shaped object. Narrow + * structurally rather than via a cast chain so unexpected shapes fail cleanly. + */ +function narrowContext(workspaceIndex: WorkspaceIndex): JavaResolveContext | null { + const ctx = workspaceIndex as JavaResolveContext | undefined; + if ( + ctx === undefined || + typeof (ctx as { fromFile?: unknown }).fromFile !== 'string' || + !((ctx as { allFilePaths?: unknown }).allFilePaths instanceof Set) + ) { + return null; + } + return ctx; +} + +/** + * First-pass resolution against the full package path: + * exact whole-path file > nested suffix file > first `.java` directly inside + * the package directory. + */ +function resolveDirectMatch( + ws: WorkspaceFileIndex, + dirs: PackageDirIndex, + pathLike: string, +): string | null { + const exactName = `${pathLike}.java`; + // The scan `break`s here, so an exact whole-path match wins even when a + // `…/` suffix match appeared EARLIER in iteration order. The two + // lookups therefore stay separate: `index.get` conflates them and would + // return the earlier suffix hit. + const exact = ws.normToRaw.get(exactName); + if (exact !== undefined) return exact; + // No whole-path file exists, so every segment-suffix hit is a `/` + // match and `index.get` yields the first one in iteration order — exactly the + // `suffixFile` the scan kept. Only a `.java` file can carry a `.java` suffix + // key, so the old `endsWith('.java')` filter is implied. + const suffixFile = ws.index.get(exactName); + if (suffixFile !== undefined) return suffixFile; + // First `.java` file living directly inside the package directory `pathLike` + // (at repo root or nested under a source-root prefix), not deeper — the leg + // wildcard imports land on. + return firstFileDirectlyInPkgDir(dirs, pathLike); +} + +/** + * Try each suffix of the package path against `.java` files and directories, + * stripping leading segments one at a time. Models `import com.example.User;` + * resolving to `User.java` in a repo laid out without the `com/example/` prefix. + */ +function resolveByProgressiveStripping( + ws: WorkspaceFileIndex, + dirs: PackageDirIndex, + pathLike: string, +): string | null { const segments = pathLike.split('/').filter(Boolean); for (let skip = 1; skip < segments.length; skip++) { const tail = segments.slice(skip).join('/'); if (tail === '') continue; - const tailFile = `${tail}.java`; - const tailSuffix = `/${tailFile}`; - const tailDir = `${tail}/`; - const tailSuffixDir = `/${tailDir}`; - let tailDirectChild: string | null = null; - for (const raw of ctx.allFilePaths) { - const f = raw.replace(/\\/g, '/'); - if (!f.endsWith('.java')) continue; - if (f === tailFile) return raw; - if (f.endsWith(tailSuffix)) return raw; - if (tailDirectChild === null) { - const atRoot = f.startsWith(tailDir); - const atNested = f.includes(tailSuffixDir); - if (atRoot || atNested) { - const idx = atRoot ? 0 : f.indexOf(tailSuffixDir) + 1; - const after = f.slice(idx + tailDir.length); - if (after.length > 0 && !after.includes('/')) tailDirectChild = raw; - } - } - } - if (tailDirectChild !== null) return tailDirectChild; + // `f === tailFile || f.endsWith('/' + tailFile)`, first in iteration order — + // the scan returned at the first hit of EITHER, with no exact-wins rule, + // so here the conflated suffix lookup is the right one. + const tailFileMatch = ws.index.get(`${tail}.java`); + if (tailFileMatch !== undefined) return tailFileMatch; + // Collected mid-scan but returned only after it, so the file/suffix hit + // above beats it even when this one came first in iteration order. + const child = firstFileDirectlyInPkgDir(dirs, tail); + if (child !== null) return child; } - return null; } diff --git a/gitnexus/src/core/ingestion/languages/javascript/import-target.ts b/gitnexus/src/core/ingestion/languages/javascript/import-target.ts index bfdfe9951..aa1914522 100644 --- a/gitnexus/src/core/ingestion/languages/javascript/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/javascript/import-target.ts @@ -18,25 +18,81 @@ * tsconfig-based aliases alongside JavaScript can still resolve via the * standard extension-suffix fallback; the alias branch is a no-op when * `tsconfigPaths` is null. + * + * ## The suffix index changes bare-specifier answers (PR #2911) + * + * Supplying `index` is not only a speed-up: `suffixResolve` answers a different + * question with one than without. Without an index it tests + * `filePath.endsWith('/' + suffix)`, so only a PROPER suffix can match; with + * one it reads `buildSuffixIndex`, which indexes `j = 0` and therefore matches + * WHOLE paths too. Two classes of answer move, both only on the bare/absolute + * specifier leg (relative imports resolve by exact `Set.has` and never reach + * it), and both toward what TypeScript and Vue have always answered: + * + * 1. a repo-root file becomes reachable at all — `require('config')` now + * finds `config.js`, where before no proper suffix existed and the answer + * was null; + * 2. a whole-path candidate outranks a proper-suffix candidate found at a + * SHORTER path suffix or a later extension — `import 'app/main'` resolved + * to `node_modules/dep/lib/main.js` (the first `/main.js` in file order) + * and now resolves to `app/main.js`. + * + * Measured over 211 200 old-vs-new pairs there is no third class: the index + * never loses a match the scan found, and its answer is never matched at a less + * specific (path-part, extension) position. `test/unit/scope-resolution/ + * javascript-import-target-parity.test.ts` is that differential, and pins both + * classes by witness. */ import { SupportedLanguages } from 'gitnexus-shared'; import { resolveTsTarget, type TsResolveContext } from '../typescript/import-target.js'; +import { buildImportPassCache } from '../../import-resolvers/pass-cache.js'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; export type JsResolveContext = TsResolveContext; -type PassCache = { - readonly key: ReadonlySet; - readonly allFilePaths: Set; - readonly allFileList: readonly string[]; - readonly normalizedFileList: readonly string[]; - readonly resolveCache: Map; -}; +/** + * Everything `resolveTsTarget` derives from one workspace file set, built once + * per set rather than once per import. + * + * `index` is not optional, and its absence was the defect (PR #2911). The + * TypeScript adapter has carried a `SuffixIndex` since #1918; this one did not, + * so every JavaScript import reached `suffixResolve` with `index === undefined` + * and took its linear-`findIndex` fallback — one pass over `normalizedFileList` + * per path part per extension, and `EXTENSIONS` has ~39 entries. Measured on + * mostly-missing bare specifiers (imports scaling with files, as in + * `bench/import-target/`): 6448.9 µs per import at 2000 files and 25972.6 µs at + * 8000 — 4.12x the per-import cost for 4x the files, which is O(imports × + * files) — against 25.0 / 27.0 µs for TypeScript over the identical corpus. + * With the index it is 28.5 / 27.4 µs and the scaling factor is 1.09x. + * + * No instrument on the #2901-#2909 branch could see it: `CountingSet` counts + * traversals of the SET, and this scan walks the materialized array behind it. + * See `test/integration/javascript-import-index-reuse.test.ts` for the guard + * that can. + * + * Memoized on the `allFilePaths` Set identity, like every other language's + * import index (`import-resolvers/workspace-file-index.ts` and friends). + * + * A single-slot `let cached` keyed on `cached.key !== allFilePaths` — what this + * adapter used before — is correct for one file set and degenerate for two: + * alternating calls across two sets rebuild everything every time. Measured on + * the TypeScript adapter at 4000 files × 400 imports: 12.0 ms for one set, + * 1438.2 ms alternating between two (120x). A `WeakMap` has no such state to + * thrash, which is also what lets this adapter carry the standard + * `expectDistinctFileSetsGetOwnIndex` guard every other indexed adapter + * carries. + * + * The Set must be passed THROUGH by the caller, never copied: a defensive + * `new Set(allFilePaths)` at the adapter boundary hands a fresh key per import + * and restores the per-import rebuild (PR #1918 review P1). + */ +const passCacheFor = perFileSet(buildImportPassCache); /** * Build a memoized `resolveImportTarget` adapter for JavaScript. - * Caches the derived arrays and per-pass resolve cache across - * `resolveImportTarget` calls within a single workspace pass. + * Caches the derived arrays, the suffix index and the per-pass resolve cache + * across `resolveImportTarget` calls over one workspace file set. */ export function makeJsResolveImportTarget(): ( targetRaw: string, @@ -44,19 +100,8 @@ export function makeJsResolveImportTarget(): ( allFilePaths: ReadonlySet, resolutionConfig?: unknown, ) => string | readonly string[] | null { - let cached: PassCache | null = null; - return (targetRaw, fromFile, allFilePaths) => { - if (cached === null || cached.key !== allFilePaths) { - const allFileList = Array.from(allFilePaths); - cached = { - key: allFilePaths, - allFilePaths: new Set(allFilePaths), - allFileList, - normalizedFileList: allFileList.map((f) => f.toLowerCase()), - resolveCache: new Map(), - }; - } + const cached = passCacheFor(allFilePaths); const ws: JsResolveContext = { fromFile, @@ -64,6 +109,7 @@ export function makeJsResolveImportTarget(): ( allFilePaths: cached.allFilePaths, allFileList: cached.allFileList, normalizedFileList: cached.normalizedFileList, + index: cached.index, resolveCache: cached.resolveCache, tsconfigPaths: null, }; diff --git a/gitnexus/src/core/ingestion/languages/kotlin/import-target.ts b/gitnexus/src/core/ingestion/languages/kotlin/import-target.ts index 8ad3c82ee..6f20e5d6f 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/import-target.ts @@ -1,6 +1,6 @@ import type { ParsedImport, WorkspaceIndex } from 'gitnexus-shared'; import { KOTLIN_EXTENSIONS } from '../../import-resolvers/jvm.js'; -import { recordKotlinFileIndexBuild } from './index-stats.js'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; export interface KotlinResolveContext { readonly fromFile: string; @@ -180,14 +180,10 @@ interface KotlinFileIndex { readonly dirChildren: Map; } -const KOTLIN_FILE_INDEX_CACHE = new WeakMap, KotlinFileIndex>(); - -function getKotlinFileIndex(allFilePaths: ReadonlySet): KotlinFileIndex { - const cached = KOTLIN_FILE_INDEX_CACHE.get(allFilePaths); - if (cached !== undefined) return cached; - // Cache miss: materialize a fresh index. Counted so a test can assert this - // happens once per run, not once per import. - recordKotlinFileIndexBuild(); +const getKotlinFileIndex = perFileSet((allFilePaths: ReadonlySet): KotlinFileIndex => { + // Runs on a cache miss only. That it happens once per run and not once per + // import is asserted by counting traversals of the Set itself, in + // `test/integration/kotlin-import-index-reuse.test.ts` (#2909). const exactByStem = new Map(); const suffixByStem = new Map(); @@ -255,10 +251,8 @@ function getKotlinFileIndex(allFilePaths: ReadonlySet): KotlinFileIndex // future mutation is a loud TypeError instead of a silent edge move. for (const bucket of dirChildren.values()) Object.freeze(bucket); - const index: KotlinFileIndex = { exactByStem, suffixByStem, dirChildren }; - KOTLIN_FILE_INDEX_CACHE.set(allFilePaths, index); - return index; -} + return { exactByStem, suffixByStem, dirChildren }; +}); function addChild(dirChildren: Map, dir: string, raw: string): void { const bucket = dirChildren.get(dir); diff --git a/gitnexus/src/core/ingestion/languages/kotlin/index-stats.ts b/gitnexus/src/core/ingestion/languages/kotlin/index-stats.ts deleted file mode 100644 index a909101d6..000000000 --- a/gitnexus/src/core/ingestion/languages/kotlin/index-stats.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Build counter for the per-file-set Kotlin import-resolution index - * (`getKotlinFileIndex` in `import-target.ts`). - * - * A "build" is a `WeakMap` cache MISS that materializes a fresh - * `KotlinFileIndex` (O(files)). Mirrors `../python/index-stats.ts`: the counter - * is always live rather than gated behind a profiling env var, because an index - * build happens at most once per resolution run, so the single increment is - * negligible and an unconditional counter avoids env-var load-order fragility - * in tests. - * - * Used by `test/integration/kotlin-import-index-reuse.test.ts` to assert the - * index is reused across imports (built once per run) rather than rebuilt per - * import — the regression guard for the quadratic resolution this replaced. - */ - -let INDEX_BUILDS = 0; - -export function recordKotlinFileIndexBuild(): void { - INDEX_BUILDS++; -} - -export function getKotlinFileIndexBuildCount(): number { - return INDEX_BUILDS; -} - -export function resetKotlinFileIndexBuildCount(): void { - INDEX_BUILDS = 0; -} diff --git a/gitnexus/src/core/ingestion/languages/php/import-target.ts b/gitnexus/src/core/ingestion/languages/php/import-target.ts index 523c2b1c7..96711ea02 100644 --- a/gitnexus/src/core/ingestion/languages/php/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/php/import-target.ts @@ -18,6 +18,9 @@ import type { ParsedFile, ParsedImport, WorkspaceIndex } from 'gitnexus-shared'; import type { ImportResolutionContext } from '../../scope-resolution/contract/scope-resolver.js'; import { resolvePhpImportInternal } from '../../import-resolvers/php.js'; +import type { SuffixIndex } from '../../import-resolvers/utils.js'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; +import { getWorkspaceFileIndex } from '../../import-resolvers/workspace-file-index.js'; import type { ComposerConfig } from '../../language-config.js'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; @@ -72,12 +75,6 @@ function namespaceDirectories( return [...directories]; } -// A scope-resolution pass shares one stable parsedFiles array across imports. -const phpDirectoryIndexCache = new WeakMap< - readonly ParsedFile[], - ReadonlyMap ->(); - function parentDirectory(filePath: string): string { const normalizedPath = normalizePhpPath(filePath); const separator = normalizedPath.lastIndexOf('/'); @@ -98,24 +95,210 @@ function directoryAliases(filePath: string): string[] { return [...aliases]; } -function filesByDirectory( - parsedFiles: readonly ParsedFile[], -): ReadonlyMap { - const cached = phpDirectoryIndexCache.get(parsedFiles); - if (cached) return cached; - - const mutable = new Map(); - for (const parsed of parsedFiles) { - for (const directory of directoryAliases(parsed.filePath)) { - const files = mutable.get(directory) ?? []; - files.push(parsed); - mutable.set(directory, files); +/** + * Directory alias → the files under it, built once per pass. + * + * A scope-resolution pass shares one stable `parsedFiles` array across imports, + * so the array identity is the memo key — see `perFileSet`. + */ +const filesByDirectory = perFileSet( + (parsedFiles: readonly ParsedFile[]): ReadonlyMap => { + const mutable = new Map(); + for (const parsed of parsedFiles) { + for (const directory of directoryAliases(parsed.filePath)) { + const files = mutable.get(directory) ?? []; + files.push(parsed); + mutable.set(directory, files); + } } - } - phpDirectoryIndexCache.set(parsedFiles, mutable); - return mutable; + return mutable; + }, +); + +// ─── workspace index (#2901) ─────────────────────────────────────────────── + +/** + * PHP's view of the shared per-file-set workspace index. + * + * Both adapters below used to materialize `[...allFilePaths]` twice per import + * and then hand `resolvePhpImportInternal` an `index` of `undefined`, which + * dropped it onto `suffixResolve`'s linear `findIndex` — one full pass over + * every file per path-part × per extension (≈50 extensions). That is the 98 ms + * per import measured at 20k files, and the arrays were the small half of it. + * + * PASSING THE SHARED `SuffixIndex` STRAIGHT THROUGH IS NOT A HOIST — IT MOVES + * IMPORTS EDGES. `resolvePhpImportInternal` reads the index at three sites, and + * all three answer a DIFFERENT question than the scan they short-circuit + * (measured, one example each): + * + * 1. `index.getInsensitive(filePath)` on the PSR-4 class-style leg has no + * no-index counterpart at all — that leg is `allFiles.has(filePath)`, an + * exact whole-path test. The index turns it into a case-insensitive SUFFIX + * probe, so `App\Models\User` under `psr-4: {"App\\": "src"}` would start + * matching `vendor/x/src/models/user.php`. + * 2. `index.getFilesInDir(nsDir, '.php')` is keyed on every directory SUFFIX, + * while the scan it replaces is anchored at the repo root + * (`f.startsWith(nsDir + '/')`). With `app/Models/Aaa.php` and + * `vendor/pkg/app/Models/Zed.php` present, `use function App\Models\getUser` + * resolves to the former today and to the latter with the raw index. + * 3. `suffixResolve` with an index probes `index.get(S) || index.getInsensitive(S)`, + * which matches WHOLE paths too (`buildSuffixIndex` indexes the `j = 0` + * suffix); the scan compares `endsWith('/' + S)` and so can only match a + * PROPER suffix. Root-level `Foo.php` is unresolvable for `use Foo;` today + * and resolvable with the raw index; and where both match, + * `App/Models/User.php` (whole path, later in iteration order) would beat + * `vendor/x/Models/User.php` (proper suffix, earlier), which is the file the + * scan returns. + * + * So this builds a PARITY view instead: the same memoized arrays, and a + * `SuffixIndex` whose three methods reproduce the no-index answers exactly. + * - `getInsensitive` returns `undefined` unconditionally, which makes site 1 a + * no-op and falls through exactly as `index === undefined` did. It is safe to + * hollow out because `suffixResolve` reads it only as + * `get(S) || getInsensitive(S)`, so `get` can carry both halves — see below. + * - `getFilesInDir` answers from a root-anchored raw-path directory bucket, so + * site 2 returns what the scan returned, in the same order. + * - `get` answers site 3, defined as "first file in Set order whose normalized + * path has `S` as a proper segment suffix, compared case-insensitively". + * That single rule IS the scan: its predicate is + * `endsWith(p) || toLowerCase().endsWith(p.toLowerCase())`, whose first + * disjunct is subsumed by the second, so a case-sensitive hit never outranks + * an earlier case-insensitive one the way `get() || getInsensitive()` does. + * + * `get` is built on the shared `index.getInsensitive`, which is that same rule + * plus the whole-path (`j = 0`) entries. The correction needs one extra map, and + * only O(files) of it: the shared lookup can only over-match when `S` IS some + * file's whole normalized path, so `firstProperSuffixMatch` is keyed on exactly + * those strings. (Whole-string vs per-segment lowercasing agree here: no case + * mapping in Unicode produces or consumes `/`, so `lower(p).split('/')` and + * `p.split('/').map(lower)` are the same list.) + * + * `index.getInsensitive` is the ONLY shared-index method this file calls — it + * never asks the case-sensitive question — which is why `buildSuffixIndex` + * defers its two suffix maps rather than fusing them: PHP builds and retains + * one of the pair instead of both (34.49 MiB of 69.85 MiB at 32 000 paths). + * + * The two maps built HERE are deferred for the same reason and are each cheap + * only in ENTRIES, not in the walk that fills them — see the notes on + * `getFirstProperSuffixMatch` (O(paths × depth) to fill, typically zero entries) + * and `getFilesByRawDirectory` (unreachable without a `composer.json`). + */ +interface PhpWorkspaceIndex { + /** Every path, backslashes normalized to `/`. Parallel to `all`. */ + readonly normalized: readonly string[]; + /** Every path, exactly as it appears in the Set. Parallel to `normalized`. */ + readonly all: readonly string[]; + /** Scan-equivalent `SuffixIndex` for `resolvePhpImportInternal`. */ + readonly suffixIndex: SuffixIndex; } +/** Memoized on the `allFilePaths` Set identity, like `getWorkspaceFileIndex`. */ +const getPhpWorkspaceIndex = perFileSet((allFilePaths: ReadonlySet): PhpWorkspaceIndex => { + // The Set is passed THROUGH to the shared cache, never copied — a defensive + // `new Set(...)` here or in `scope-resolver.ts` would hand both WeakMaps a + // fresh key per import and silently restore O(imports × files) (#1918 P1). + const { normalized, all, index } = getWorkspaceFileIndex(allFilePaths); + + /** + * Whole-path-lowercase → the first PROPER-suffix match, the correction `get` + * applies to a whole-path hit from the shared index. + * + * DEFERRED, and deferred all the way to the branch that reads it rather than + * to the first `get`. The builder walks every slash of every path and + * lowercases a slice at each, so it is O(paths × depth) in both time and + * allocation — measured 46.0 ms at 32 000 paths on the PHP arm of + * `bench/import-target/`, filling a map that held ZERO entries, because it + * can only hold one when some file's whole path is also a proper suffix of + * another's. Most repos never produce that, and the ones that do reach this + * branch only for the imports that actually hit a whole path. Pure function + * of `normalized`/`all`, both of which the returned object already retains, + * so building it late is behaviour-identical and retains nothing new. + * + * `wholePathLower` is a scratch set of the builder, not state: nothing reads + * it afterwards, so deferring the map defers it too. + */ + let firstProperSuffixMatch: Map | null = null; + const getFirstProperSuffixMatch = (): Map => { + if (firstProperSuffixMatch !== null) return firstProperSuffixMatch; + const wholePathLower = new Set(); + for (const path of normalized) wholePathLower.add(path.toLowerCase()); + + // Only the suffixes that a whole path can shadow are worth storing; see the + // header. Built from `normalized`, so it costs no traversal of the Set. + const built = new Map(); + for (let i = 0; i < normalized.length; i++) { + const lower = normalized[i].toLowerCase(); + for (let slash = lower.indexOf('/'); slash >= 0; slash = lower.indexOf('/', slash + 1)) { + const suffix = lower.slice(slash + 1); + if (!wholePathLower.has(suffix)) continue; + if (!built.has(suffix)) built.set(suffix, all[i]); + } + } + firstProperSuffixMatch = built; + return built; + }; + + /** + * Raw directory → the files directly in it, for `getFilesInDir`. + * + * DEFERRED for the same reason as the shared `dirMap` (#2903), and here the + * case is stronger: `getFilesInDir` has exactly one caller, + * `import-resolvers/php.ts`'s PSR-4 function/constant fallback, and that + * caller sits inside `if (composerConfig) { … }`. `resolvePhpImportTarget` + * hard-codes `composerConfig: null`, so on the LanguageProvider path the map + * is statically unreachable; on the ScopeResolver path it is reachable only + * in a repo that has a parseable `composer.json` with `autoload.psr-4`. + * Measured 6.8 ms / 3.56 MiB at 32 000 paths, paid by every PHP repo without + * one. Pure function of `all`, which the returned object retains. + */ + let filesByRawDirectory: Map | null = null; + const getFilesByRawDirectory = (): Map => { + if (filesByRawDirectory !== null) return filesByRawDirectory; + // Raw paths, not normalized: the scan this replaces tests `f.startsWith(...)` + // against the Set's own strings, so a backslash path is a miss there and must + // stay a miss here. Insertion order is Set order, so `[0]` is the file the + // scan would have returned first. + const built = new Map(); + for (const raw of all) { + const separator = raw.lastIndexOf('/'); + if (separator < 0) continue; + const directory = raw.slice(0, separator); + const bucket = built.get(directory); + if (bucket === undefined) built.set(directory, [raw]); + else bucket.push(raw); + } + filesByRawDirectory = built; + return built; + }; + + const suffixIndex: SuffixIndex = { + get: (suffix: string): string | undefined => { + const hit = index.getInsensitive(suffix); + if (hit === undefined) return undefined; + const lower = suffix.toLowerCase(); + // A proper-suffix hit is already the scan's answer: the shared map holds + // the first file matching EITHER way, so nothing earlier matched at all. + if (hit.replace(/\\/g, '/').toLowerCase() !== lower) return hit; + // Whole-path hit — invisible to `endsWith('/' + S)`. The scan keeps going. + // The only branch that needs the correction map, hence the only one that + // builds it. + return getFirstProperSuffixMatch().get(lower); + }, + // Site 1 must stay a no-op, and `suffixResolve` folds this into `get`. + getInsensitive: (): undefined => undefined, + getFilesInDir: (dirSuffix: string, extension: string): string[] => { + // `nsDirPrefix` is `nsDir` when it already ends in `/`, else `nsDir + '/'` + // — either way the directory is `nsDir` minus one trailing slash. + const directory = dirSuffix.endsWith('/') ? dirSuffix.slice(0, -1) : dirSuffix; + const bucket = getFilesByRawDirectory().get(directory); + if (bucket === undefined) return []; + return bucket.filter((file) => file.endsWith(extension)); + }, + }; + + return { normalized, all, suffixIndex }; +}); + // ─── loadResolutionConfig ────────────────────────────────────────────────── /** @@ -181,17 +364,17 @@ export function resolvePhpImportTarget( if (parsedImport.kind === 'dynamic-unresolved') return null; if (parsedImport.targetRaw === null || parsedImport.targetRaw === '') return null; + // Cast, not copy: `getPhpWorkspaceIndex` memoizes on this exact Set object. const allFiles = ctx.allFilePaths as Set; - const normalizedFileList = [...allFiles].map((f) => f.replace(/\\/g, '/')); - const allFileList = [...allFiles]; + const { normalized, all, suffixIndex } = getPhpWorkspaceIndex(allFiles); return resolvePhpImportInternal( parsedImport.targetRaw, null, // composerConfig not available through LanguageProvider path allFiles, - normalizedFileList, - allFileList, - undefined, + normalized, + all, + suffixIndex, ); } @@ -216,17 +399,17 @@ export function resolvePhpImportTargetInternal( ? (resolutionConfig as ComposerConfig) : null; + // Cast, not copy: `getPhpWorkspaceIndex` memoizes on this exact Set object. const allFiles = allFilePaths as Set; - const normalizedFileList = [...allFiles].map((f) => f.replace(/\\/g, '/')); - const allFileList = [...allFiles]; + const { normalized, all, suffixIndex } = getPhpWorkspaceIndex(allFiles); const resolved = resolvePhpImportInternal( targetRaw, composerConfig, allFiles, - normalizedFileList, - allFileList, - undefined, + normalized, + all, + suffixIndex, ); const parsedImport = context?.parsedImport; diff --git a/gitnexus/src/core/ingestion/languages/python/import-target.ts b/gitnexus/src/core/ingestion/languages/python/import-target.ts index 2ab1ccf40..e31f038b0 100644 --- a/gitnexus/src/core/ingestion/languages/python/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/python/import-target.ts @@ -11,8 +11,13 @@ */ import type { ParsedFile, ParsedImport, WorkspaceIndex } from 'gitnexus-shared'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; +import { + getPythonFileIndex, + importerAncestors, + importerDirOf, +} from '../../import-resolvers/python-file-index.js'; import { resolvePythonImportInternal } from '../../import-resolvers/python.js'; -import { recordPythonFileIndexBuild } from './index-stats.js'; export interface PythonResolveContext { readonly fromFile: string; @@ -82,7 +87,35 @@ export function resolvePythonImportTarget( workspaceIndex, ); if (submodule !== null) return submodule; - if (packageTarget !== null) return packageTarget; + + // `return packageTarget`, not `if (packageTarget !== null) return …` — + // falling through when it is null RE-RAN THE ENTIRE TAIL BELOW, a second + // time, with byte-identical arguments. + // + // `packageTarget` IS this function's tail for this import. The recursion + // above differs from the outer frame in exactly one field, + // `targetIncludesImportedName`, whose only effect is to make + // `pythonImportedSubmoduleTarget` return null and so skip this branch: the + // spread preserves `kind` (still `named`/`alias`, so the + // `dynamic-unresolved` guard cannot fire) and `targetRaw` (which already + // passed the null/empty guard), and `workspaceIndex` is the same object, so + // `ctx.fromFile`, `ctx.allFilePaths` and `ctx.parsedFiles` are the same + // references. The recursion therefore ran `resolvePythonImportInternal` → + // relative gate → `hasRepoCandidate` → `resolveAbsoluteFromFiles` on + // exactly the inputs the fallthrough would use. + // + // That tail is a pure function of (`fromFile`, `targetRaw`, + // `allFilePaths`): it only reads the Set and indexes memoized on the Set, + // and the `submodule` probe in between is equally read-only, so nothing can + // have changed the answer. Reaching this line means the tail already + // returned null; running it again returns null again, after another + // proximity probe and another full ancestor walk to the workspace root. + // + // Measured before this change, `from x import y` at four directory + // components: 24 `allFilePaths.has` probes per import, of which probes + // 12-23 were byte-identical repeats of 0-11. `python-import-probe-count + // .test.ts` is the gate. + return packageTarget; } // PEP-328 relative + single-segment proximity bare imports. @@ -144,6 +177,13 @@ export function resolvePythonImportTarget( * that classification is what open issue #2882 is about, so it belongs with * that fix rather than bolted on here. Not a regression: both halves behave * exactly as they did before #2864. + * + * The `parsedFiles.find` this used to open with was the same O(imports x files) + * shape #2913 removes on the path Set, keyed on the other collection the + * orchestrator threads: every import whose package probe resolves scanned the + * whole parsed workspace, and on a repo where `from pkg import X` usually + * resolves that is most imports. `parsedFileByPath` replaces it with one pass + * per pass. */ function pythonFileExportsName( targetFile: string, @@ -151,7 +191,7 @@ function pythonFileExportsName( parsedFiles: readonly ParsedFile[] | undefined, ): boolean { if (parsedFiles === undefined) return false; - const parsed = parsedFiles.find((file) => file.filePath === targetFile); + const parsed = parsedFileByPath(parsedFiles).get(targetFile); if (parsed === undefined) return false; return parsed.localDefs.some((def) => { const qualifiedName = def.qualifiedName; @@ -161,6 +201,27 @@ function pythonFileExportsName( }); } +/** + * `filePath -> ParsedFile`, memoized on the identity of the pass's + * `parsedFiles` array — the second stable object the orchestrator threads + * through `resolveImportTarget`, beside the path Set. + * + * FIRST WINS on a duplicated path, which is what `Array.prototype.find` + * returned, so the answer is unchanged for a workspace that somehow parsed one + * path twice. Values are references to the array's own elements: the Map costs + * one pointer per parsed file and, living in a `WeakMap` keyed on the array, + * is reclaimed with the pass rather than accumulating across runs (#2649). + */ +const parsedFileByPath = perFileSet( + (parsedFiles: readonly ParsedFile[]): Map => { + const byPath = new Map(); + for (const file of parsedFiles) { + if (!byPath.has(file.filePath)) byPath.set(file.filePath, file); + } + return byPath; + }, +); + /** * Resolve `package/sub/module` style paths (already dot-flattened) to a * concrete file in `allFilePaths`. Tries the exact path first, then walks @@ -196,19 +257,44 @@ function resolveAbsoluteFromFiles( if (allFilePaths.has(directFile)) return directFile; if (allFilePaths.has(directPkg)) return directPkg; + // Both remaining tiers — the ancestor walk and the suffix fallback — can only + // ever land on a file whose basename is `.py`, or on an `__init__.py` + // whose parent directory is named ``. The two buckets the suffix + // fallback already needs therefore also decide, in O(1) and before the walk, + // whether the walk can hit at all: neither bucket present means no tier below + // can match, and one bucket absent removes that tier's probe from EVERY step + // of the walk. On the deep corpus that is half the walk's probes (#2913). + // + // `pythonSegmentAbsent` states this same rule for the single-segment bare + // tier. It is deliberately not called here: that tier needs only the answer, + // this one needs the candidate ARRAYS for the suffix fallback below, so + // sharing would mean two extra `has` lookups per import to save four lines. + const index = getPythonFileIndex(allFilePaths); + const lastSeg = pathLike.slice(pathLike.lastIndexOf('/') + 1); + const moduleCandidates = index.byBasename.get(`${lastSeg}.py`); + const packageCandidates = index.byInitParent.get(`${lastSeg}/__init__.py`); + const mayBeModule = moduleCandidates !== undefined; + // `byInitParent` skips `__init__.py` files whose parent directory name is + // empty (a doubled separator), so an empty `` — a target spelled + // with a trailing dot — cannot use the bucket as proof of absence and keeps + // probing exactly as before. + const mayBePackage = packageCandidates !== undefined || lastSeg === ''; + if (!mayBeModule && !mayBePackage) return null; + // Ancestor walk — match the single-segment resolver's behavior at - // multi-segment granularity. Closest match wins. Stop at `i > 0` because - // `i === 0` would re-check the workspace-root candidates already covered - // by the direct check above. - const importerDir = fromFile.replace(/\\/g, '/').split('/').slice(0, -1).join('/'); - if (importerDir) { - const dirParts = importerDir.split('/').filter(Boolean); - for (let i = dirParts.length; i > 0; i--) { - const ancestor = dirParts.slice(0, i).join('/'); - const prefix = `${ancestor}/`; - const candidateFile = `${prefix}${directFile}`; - const candidatePkg = `${prefix}${directPkg}`; + // multi-segment granularity. Closest match wins. The chain stops short of the + // workspace root because the root candidates are the direct check above. + // + // The chain comes from `importerAncestors`, which builds it ONCE per importer + // directory per pass. Rebuilding it here — one `slice(0, i).join('/')` per + // path component, on every import — was half of the depth quadratic in #2913. + for (const ancestor of importerAncestors(index, importerDirOf(fromFile))) { + if (mayBeModule) { + const candidateFile = `${ancestor}/${directFile}`; if (allFilePaths.has(candidateFile)) return candidateFile; + } + if (mayBePackage) { + const candidatePkg = `${ancestor}/${directPkg}`; if (allFilePaths.has(candidatePkg)) return candidatePkg; } } @@ -237,17 +323,15 @@ function resolveAbsoluteFromFiles( // shared buildSuffixIndex is deliberately NOT used: it keeps only one // path per suffix (longest wins) and so cannot reproduce this exact // fewest-segments-then-lexicographic tie-break across all candidates. - const index = getPythonFileIndex(allFilePaths); - const lastSeg = pathLike.slice(pathLike.lastIndexOf('/') + 1); const matches: { raw: string; norm: string }[] = []; - for (const cand of index.byBasename.get(`${lastSeg}.py`) ?? []) { + for (const cand of moduleCandidates ?? []) { if (cand.norm.endsWith(suffixFile)) matches.push(cand); } // Package form: only `__init__.py` files whose parent dir is named `` // can match `…//__init__.py` — look them up by parent key (P2b) and // confirm the full suffix. Same final candidate set as the old `__init__.py` // scan, just without iterating unrelated packages. - for (const cand of index.byInitParent.get(`${lastSeg}/__init__.py`) ?? []) { + for (const cand of packageCandidates ?? []) { if (cand.norm.endsWith(suffixPkg)) matches.push(cand); } if (matches.length === 0) return null; @@ -293,131 +377,33 @@ function hasRepoCandidate( const rootFile = `${leadingSegment}.py`; const initFile = `${leadingSegment}/__init__.py`; - // Build importer-ancestor prefixes: for `backend/routers/cron.py`, - // produces `["backend/routers/services/", "backend/services/"]` for - // segment `services` (closest first, root excluded — covered above). - const importerDir = fromFile.replace(/\\/g, '/').split('/').slice(0, -1).join('/'); - const dirParts = importerDir ? importerDir.split('/').filter(Boolean) : []; - const ancestorPrefixes: string[] = []; - for (let i = dirParts.length; i > 0; i--) { - ancestorPrefixes.push(`${dirParts.slice(0, i).join('/')}/${leadingSegment}/`); - } - // Indexed equivalents of the old O(files) scan: // (1) `f === rootFile || f === initFile` -> normalized-path membership. // (2) `f.startsWith(`${seg}/`) && f.endsWith('.py')` -> some .py file lives // under directory `${seg}/`, i.e. `${seg}/` is a known .py dir prefix. // (3) ancestor namespace case -> `${ancestor}/${seg}/` is a known .py dir - // prefix. + // prefix, for some ancestor of the importer's directory. const index = getPythonFileIndex(allFilePaths); if (index.normSet.has(rootFile) || index.normSet.has(initFile)) return true; if (index.dirPrefixes.has(prefix)) return true; - for (const ap of ancestorPrefixes) { - if (index.dirPrefixes.has(ap)) return true; + // (3) used to MATERIALIZE one `${ancestor}/${seg}/` string per component of + // the importer's directory, eagerly, before checks (1) and (2) had even run — + // O(depth^2) characters on every import, and the other half of #2913. Two + // things replace that: `nestedDirNames` answers "is `seg` the name of any + // directory sitting under a non-empty parent?" in O(1), which is `false` for + // every external import (`os`, `django`, an unknown distribution) and skips + // the walk outright; and what remains walks the per-directory ancestor chain, + // built once per pass, closest first, so the common in-repo hit exits after a + // step or two. `nestedDirNames` is exact, not a filter: `${A}/${seg}/` can + // only be a directory prefix if `seg` names a directory under the non-empty + // parent `A`, so a miss here means the old loop would have missed too. + if (!index.nestedDirNames.has(leadingSegment)) return false; + for (const ancestor of importerAncestors(index, importerDirOf(fromFile))) { + if (index.dirPrefixes.has(`${ancestor}/${prefix}`)) return true; } return false; } -/** - * Per-file-set index for Python import resolution, memoized on the - * `allFilePaths` Set object (the same Set is passed for every import in a run, - * so the index is built once and reused). Replaces the per-import O(files) - * scans in `resolveAbsoluteFromFiles` (suffix match) and `hasRepoCandidate` - * (package-existence gate) with O(1)/O(bucket) lookups. - * - * - `normSet`: every file path, normalized to forward slashes (for the exact - * `f === rootFile|initFile` membership checks). - * - `byBasename`: last path component (e.g. `models.py`, `__init__.py`) -> - * all `{ raw, norm }` candidates, so suffix matches can be gathered from the - * relevant bucket and the exact tie-break applied across ALL of them. - * - `byInitParent`: `__init__.py` files keyed by their last TWO components - * (`/__init__.py`). The package suffix lookup (`pkg.sub` -> - * `…/sub/__init__.py`) targets only same-named package dirs via this map - * instead of scanning every `__init__.py` in the repo — the common - * multi-segment import path no longer scales with package count - * (PR #1918 review P2b). `__init__.py` files stay in `byBasename` too, for - * the rarer explicit `pkg.__init__` import that resolves via the module - * (`….py`) lookup. - * - `dirPrefixes`: every directory prefix of a `.py` file, trailing-slashed - * (`a/b/c.py` -> `a/`, `a/b/`), for "is there a .py file under `/`". - */ -interface PythonFileIndex { - readonly normSet: Set; - readonly byBasename: Map; - readonly byInitParent: Map; - readonly dirPrefixes: Set; -} - -const PYTHON_FILE_INDEX_CACHE = new WeakMap, PythonFileIndex>(); - -function getPythonFileIndex(allFilePaths: ReadonlySet): PythonFileIndex { - const cached = PYTHON_FILE_INDEX_CACHE.get(allFilePaths); - if (cached !== undefined) return cached; - // Cache miss: materialize a fresh index. Counted so a test can assert this - // happens once per run, not once per import (PR #1918 review P1 guard). - recordPythonFileIndexBuild(); - - const normSet = new Set(); - const byBasename = new Map(); - const byInitParent = new Map(); - const dirPrefixes = new Set(); - - for (const raw of allFilePaths) { - const norm = raw.replace(/\\/g, '/'); - // Python import resolution only ever queries `.py` paths: module `.py` - // and package `/__init__.py` membership (normSet), `.py` / - // `__init__.py` basename buckets (byBasename), and `.py` directory prefixes - // (dirPrefixes). Non-`.py` files can never match any of those, so skip them - // — they were dead weight in every structure on polyglot monorepos - // (PR #1918 review P3b; dirPrefixes was already `.py`-gated). - if (!norm.endsWith('.py')) continue; - normSet.add(norm); - - const lastSlash = norm.lastIndexOf('/'); - const base = lastSlash >= 0 ? norm.slice(lastSlash + 1) : norm; - let bucket = byBasename.get(base); - if (bucket === undefined) { - bucket = []; - byBasename.set(base, bucket); - } - bucket.push({ raw, norm }); - - // Package files also get a parent-keyed bucket so a `pkg.sub` lookup hits - // only `…/sub/__init__.py` candidates, not every `__init__.py` (P2b). - if (base === '__init__.py' && lastSlash >= 0) { - const dir = norm.slice(0, lastSlash); - const parentSlash = dir.lastIndexOf('/'); - const parentName = parentSlash >= 0 ? dir.slice(parentSlash + 1) : dir; - if (parentName) { - const initKey = `${parentName}/__init__.py`; - let ib = byInitParent.get(initKey); - if (ib === undefined) { - ib = []; - byInitParent.set(initKey, ib); - } - ib.push({ raw, norm }); - } - } - - // Directory prefixes: every slash-terminated prefix of the path (every - // index just past a '/', up to and including the file's own directory). - // Scanning the FULL normalized path — including any leading '/' for - // absolute paths — makes `dirPrefixes.has(X)` match exactly when the old - // gate's `f.startsWith(X)` (X always ends in '/') matched. The previous - // split+`filter(Boolean)` dropped the leading empty component, so an - // absolute file `/repo/svc/x.py` yielded `repo/svc/` (no leading slash) and - // gate-passed where `"/repo/svc/x.py".startsWith("repo/svc/")` is false - // (PR #1918 review P3a). For relative paths the set is identical. - for (let i = 0; i <= lastSlash; i++) { - if (norm[i] === '/') dirPrefixes.add(norm.slice(0, i + 1)); - } - } - - const index: PythonFileIndex = { normSet, byBasename, byInitParent, dirPrefixes }; - PYTHON_FILE_INDEX_CACHE.set(allFilePaths, index); - return index; -} - function pythonImportedSubmoduleTarget(parsedImport: ParsedImport): string | null { if (parsedImport.kind !== 'named' && parsedImport.kind !== 'alias') return null; if (parsedImport.targetIncludesImportedName === true) return null; diff --git a/gitnexus/src/core/ingestion/languages/python/index-stats.ts b/gitnexus/src/core/ingestion/languages/python/index-stats.ts deleted file mode 100644 index 2e3d2ae82..000000000 --- a/gitnexus/src/core/ingestion/languages/python/index-stats.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Build counter for the per-file-set Python import-resolution index - * (`getPythonFileIndex` in `import-target.ts`). - * - * A "build" is a `WeakMap` cache MISS that materializes a fresh - * `PythonFileIndex` (O(files)). Unlike `cache-stats.ts` (which gates its - * counters behind `PROF_SCOPE_RESOLUTION` because they sit on the per-capture - * hot path), this counter is always live: an index build happens at most once - * per resolution run, so the single increment is negligible and an unconditional - * counter avoids env-var load-order fragility in tests. - * - * Used by `test/integration/python-import-index-reuse.test.ts` to assert the - * index is reused across imports (built once per run) rather than rebuilt per - * import — the regression guard for PR #1918 review finding P1. - */ - -let INDEX_BUILDS = 0; - -export function recordPythonFileIndexBuild(): void { - INDEX_BUILDS++; -} - -export function getPythonFileIndexBuildCount(): number { - return INDEX_BUILDS; -} - -export function resetPythonFileIndexBuildCount(): void { - INDEX_BUILDS = 0; -} diff --git a/gitnexus/src/core/ingestion/languages/rust/qualified-call.ts b/gitnexus/src/core/ingestion/languages/rust/qualified-call.ts index 65d1dd5f4..54047c6ff 100644 --- a/gitnexus/src/core/ingestion/languages/rust/qualified-call.ts +++ b/gitnexus/src/core/ingestion/languages/rust/qualified-call.ts @@ -33,6 +33,7 @@ */ import type { ParsedFile, Scope, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { isOverloadableCallable } from '../../utils/callable-labels.js'; import { lookupBindingsAt } from '../../scope-resolution/scope/walkers.js'; import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; @@ -53,16 +54,9 @@ import { * The hook is invoked per call site; rebuilding the index each time would make * qualified-call resolution O(sites x files). */ -const MODULE_INDEX_CACHE = new WeakMap, RustModuleIndex>(); - -function moduleIndexFor(allFilePaths: ReadonlySet): RustModuleIndex { - let index = MODULE_INDEX_CACHE.get(allFilePaths); - if (index === undefined) { - index = buildRustModuleIndex(allFilePaths); - MODULE_INDEX_CACHE.set(allFilePaths, index); - } - return index; -} +const moduleIndexFor = perFileSet( + (allFilePaths: ReadonlySet): RustModuleIndex => buildRustModuleIndex(allFilePaths), +); export function resolveRustQualifiedFreeCall( site: { readonly name: string; readonly rawQualifiedName?: string; readonly inScope: ScopeId }, @@ -488,6 +482,15 @@ interface PassModuleIndex { readonly inlineModuleKeys: ReadonlySet; } +/** + * DELIBERATELY NOT ON `import-resolvers/per-file-set.ts` (#2909 sweep), unlike + * {@link moduleIndexFor} above. {@link passIndexFor} takes THREE inputs — + * `workspaceIndex`, `index` and `scopes` — and keys on the first alone; the + * builder reads `scopes.defs.byId` and `index`, neither of which is derivable + * from the key, and `perFileSet`'s `build: (key) => T` hands the builder + * nothing but the key. Sound here only because all three share the resolution + * pass's lifetime, which is an invariant the primitive cannot express. + */ const MODULE_SCOPE_CACHE = new WeakMap(); function moduleKey(module: RustModule): string { diff --git a/gitnexus/src/core/ingestion/languages/swift/import-target.ts b/gitnexus/src/core/ingestion/languages/swift/import-target.ts index 5c0c2f662..e0d5e8219 100644 --- a/gitnexus/src/core/ingestion/languages/swift/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/swift/import-target.ts @@ -25,6 +25,7 @@ */ import type { ParsedImport, WorkspaceIndex } from 'gitnexus-shared'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; export interface SwiftResolveContext { readonly fromFile: string; @@ -39,12 +40,7 @@ interface SwiftModuleIndex { readonly byModule: Map; } -const SWIFT_MODULE_INDEX_CACHE = new WeakMap, SwiftModuleIndex>(); - -function getSwiftModuleIndex(allFilePaths: ReadonlySet): SwiftModuleIndex { - const cached = SWIFT_MODULE_INDEX_CACHE.get(allFilePaths); - if (cached !== undefined) return cached; - +const getSwiftModuleIndex = perFileSet((allFilePaths: ReadonlySet): SwiftModuleIndex => { const byModule = new Map(); for (const raw of allFilePaths) { const norm = raw.replace(/\\/g, '/'); @@ -66,10 +62,8 @@ function getSwiftModuleIndex(allFilePaths: ReadonlySet): SwiftModuleInde } } - const index: SwiftModuleIndex = { byModule }; - SWIFT_MODULE_INDEX_CACHE.set(allFilePaths, index); - return index; -} + return { byModule }; +}); export function resolveSwiftImportTarget( parsedImport: ParsedImport, diff --git a/gitnexus/src/core/ingestion/languages/typescript/import-target.ts b/gitnexus/src/core/ingestion/languages/typescript/import-target.ts index 7d39e1f64..782dc9cbf 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/import-target.ts @@ -82,8 +82,8 @@ export function resolveTsTarget(targetRaw: string, ctx: TsResolveContext): strin ctx.fromFile, targetRaw, ctx.allFilePaths, - allFileList as string[], - normalizedFileList as string[], + allFileList, + normalizedFileList, resolveCache, language, ctx.tsconfigPaths ?? null, diff --git a/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts index fe7a30da4..bcaeffa51 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/scope-resolver.ts @@ -22,7 +22,8 @@ import { simpleKey } from '../../scope-resolution/graph-bridge/node-lookup.js'; import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; import { typescriptProvider } from '../typescript.js'; import { loadTsconfigPaths, type TsconfigPaths } from '../../language-config.js'; -import { buildSuffixIndex, type SuffixIndex } from '../../import-resolvers/utils.js'; +import { buildImportPassCache } from '../../import-resolvers/pass-cache.js'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { indexOnlyElementType } from '../../type-extractors/shared.js'; import { typescriptArityCompatibility, @@ -55,42 +56,31 @@ const TYPESCRIPT_TYPE_ONLY_BINDING_TYPES = new Set([ ]); /** - * Build a `resolveImportTarget` adapter that memoizes the workspace - * file list, the lower-cased file list, and the per-pass `resolveCache` - * across every import lookup in a single workspace pass. The - * orchestrator passes the same `ReadonlySet` reference for every call - * within a pass — we use that identity to detect when the workspace - * changes and recompute the derived state lazily. + * Memoized on the `allFilePaths` Set identity, like every other language's + * import index (`import-resolvers/workspace-file-index.ts` and friends). * - * Without this memoization, `resolveTsTarget` re-derived - * `allFileList` and `normalizedFileList` (both O(N_files)) and threw - * away the `resolveCache` on every import — O(N_files × N_imports) - * total work for what should be O(N_files + N_imports). + * This used to be a single-slot `let cached` invalidated by + * `cached.key !== allFilePaths` — correct for one file set and degenerate for + * two: alternating calls across two sets rebuilt everything every time. + * Measured here at 4000 files × 400 imports: 12.0 ms for one set, 1438.2 ms + * alternating between two (120x). A `WeakMap` has no such state to thrash, and + * it is what lets this adapter carry the standard + * `expectDistinctFileSetsGetOwnIndex` guard the other languages carry + * (`test/integration/typescript-import-index-reuse.test.ts`). + * + * The Set must be passed THROUGH by the caller, never copied: a defensive + * `new Set(allFilePaths)` at the adapter boundary hands a fresh key per import + * and restores the per-import rebuild (PR #1918 review P1). + */ +const tsPassCacheFor = perFileSet(buildImportPassCache); + +/** + * Build a `resolveImportTarget` adapter that reads the memoized per-file-set + * state above rather than re-deriving it on every import lookup. */ function makeTsResolveImportTarget(): ScopeResolver['resolveImportTarget'] { - interface PassCache { - readonly key: ReadonlySet; - readonly allFilePaths: Set; - readonly allFileList: readonly string[]; - readonly normalizedFileList: readonly string[]; - readonly index: SuffixIndex; - readonly resolveCache: Map; - } - let cached: PassCache | null = null; - return (targetRaw, fromFile, allFilePaths, resolutionConfig) => { - if (cached === null || cached.key !== allFilePaths) { - const allFileList = Array.from(allFilePaths); - const normalizedFileList = allFileList.map((f) => f.toLowerCase()); - cached = { - key: allFilePaths, - allFilePaths: new Set(allFilePaths), - allFileList, - normalizedFileList, - index: buildSuffixIndex(normalizedFileList, allFileList), - resolveCache: new Map(), - }; - } + const cached = tsPassCacheFor(allFilePaths); const cfg = resolutionConfig as TypescriptResolutionConfig | undefined; const ws: TsResolveContext = { diff --git a/gitnexus/src/core/ingestion/languages/vue/import-target.ts b/gitnexus/src/core/ingestion/languages/vue/import-target.ts index a16877459..a50c4aa54 100644 --- a/gitnexus/src/core/ingestion/languages/vue/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/vue/import-target.ts @@ -14,27 +14,39 @@ * logic fires. * * Memoization mirrors the TypeScript adapter: workspace file-list - * arrays, the suffix index, and the per-pass resolve cache are rebuilt - * lazily when `allFilePaths` reference changes (once per workspace pass). + * arrays, the suffix index and the per-pass resolve cache are built + * once per `allFilePaths` Set and memoized on that Set's identity. */ import { SupportedLanguages } from 'gitnexus-shared'; import { resolveTsTarget, type TsResolveContext } from '../typescript/import-target.js'; -import { buildSuffixIndex, type SuffixIndex } from '../../import-resolvers/utils.js'; +import { buildImportPassCache } from '../../import-resolvers/pass-cache.js'; +import { perFileSet } from '../../import-resolvers/per-file-set.js'; import type { TsconfigPaths } from '../../language-config.js'; interface VueResolutionConfig { readonly tsconfigPaths: TsconfigPaths | null; } -interface PassCache { - readonly key: ReadonlySet; - readonly allFilePaths: Set; - readonly allFileList: readonly string[]; - readonly normalizedFileList: readonly string[]; - readonly index: SuffixIndex; - readonly resolveCache: Map; -} +/** + * Memoized on the `allFilePaths` Set identity, like every other language's + * import index (`import-resolvers/workspace-file-index.ts` and friends). + * + * This used to be a single-slot `let cached` invalidated by + * `cached.key !== allFilePaths` — correct for one file set and degenerate for + * two: alternating calls across two sets rebuilt everything every time. + * Measured on the identical TypeScript adapter at 4000 files × 400 imports: + * 12.0 ms for one set, 1438.2 ms alternating between two (120x). A `WeakMap` + * has no such state to thrash, and it is what lets this adapter carry the + * standard + * `expectDistinctFileSetsGetOwnIndex` guard the other languages carry + * (`test/integration/vue-import-index-reuse.test.ts`). + * + * The Set must be passed THROUGH by the caller, never copied: a defensive + * `new Set(allFilePaths)` at the adapter boundary hands a fresh key per import + * and restores the per-import rebuild (PR #1918 review P1). + */ +const passCacheFor = perFileSet(buildImportPassCache); /** * Build a memoized `resolveImportTarget` adapter for Vue SFCs. @@ -49,21 +61,8 @@ export function makeVueResolveImportTarget(): ( allFilePaths: ReadonlySet, resolutionConfig?: unknown, ) => string | readonly string[] | null { - let cached: PassCache | null = null; - return (targetRaw, fromFile, allFilePaths, resolutionConfig) => { - if (cached === null || cached.key !== allFilePaths) { - const allFileList = Array.from(allFilePaths); - const normalizedFileList = allFileList.map((f) => f.toLowerCase()); - cached = { - key: allFilePaths, - allFilePaths: new Set(allFilePaths), - allFileList, - normalizedFileList, - index: buildSuffixIndex(normalizedFileList, allFileList), - resolveCache: new Map(), - }; - } + const cached = passCacheFor(allFilePaths); const cfg = resolutionConfig as VueResolutionConfig | undefined; const ws: TsResolveContext = { diff --git a/gitnexus/test/helpers/counting-file-set.ts b/gitnexus/test/helpers/counting-file-set.ts index c9cc6dd39..45bc6cb38 100644 --- a/gitnexus/test/helpers/counting-file-set.ts +++ b/gitnexus/test/helpers/counting-file-set.ts @@ -1,16 +1,18 @@ /** - * A `Set` that counts how many times it is TRAVERSED in full — the - * measuring instrument behind the import-target index-reuse guards - * (`test/unit/scope-resolution/import-target-index-parity.test.ts` and the - * per-language `test/integration/-import-index-reuse.test.ts` files). + * A `Set` that counts how many times it is TRAVERSED in full — the one + * measuring instrument behind every import-target index-reuse guard + * (`test/unit/scope-resolution/import-target-index-parity.test.ts`, + * `test/unit/scope-resolution/import-target-index-reuse.contract.test.ts`, and + * the per-language `test/integration/-import-index-reuse.test.ts` files). * * ## Why a counting Set rather than a production build counter * - * Kotlin and Python count index BUILDS from production (`languages// - * index-stats.ts`). That catches the per-import rebuild, but it is blind to a - * scan added BESIDE a reused index: the cache still hits, the build count still - * reads 1. Counting traversals of the file set instead needs no production - * surface at all and catches both failures with one number: + * Kotlin and Python used to count index BUILDS, through a counter module that + * shipped in production for no reason but this observation (deleted in #2909). + * A build count catches the per-import rebuild, but it is blind to a scan added + * BESIDE a reused index: the cache still hits, the count still reads 1. Counting + * traversals of the file set instead needs no production surface at all and + * catches both failures with one number: * * - an adapter that copies the set (`new Set(allFilePaths)`) hands a fresh * `WeakMap` key per import, so the count rises to the import count; @@ -40,6 +42,14 @@ * Guarding that would mean either instrumenting production or proxying an index * internal; see the header of the parity test for why neither is in place. * + * It is equally blind to the OTHER per-file-set key the orchestrator threads — + * `ImportResolutionContext.parsedFiles`, the fifth argument of + * `resolveImportTarget`. PHP's `filesByDirectory` memo (`languages/php/ + * import-target.ts`) is keyed on that array, not on this Set, so defeating it + * rebuilds a `Map` per import at O(files × depth) + * without moving this counter by one. `countedParsedFiles` below is the + * instrument for that channel. + * * `instanceof Set` still holds, which matters: C#'s `narrowContext` rejects a * workspace context whose `allFilePaths` is not a `Set`, so a plain object with * a counter would silently resolve nothing and every assertion would pass on @@ -47,9 +57,14 @@ * * `expectDistinctFileSetsGetOwnIndex` below is the one arm of those guards that * is identical in every language once the four values that differ are named, so - * it lives here beside the instrument it reads rather than in each guard. + * it lives here beside the instrument it reads rather than in each guard. The + * `ChainMemoArm` section at the bottom applies the same rule to the guards that + * watch a MEMO instead of a scan count — the two Python importer-chain guards, + * which this instrument provably cannot see (their headers say why) and which + * were arm-for-arm the same suite written twice. */ import { expect } from 'vitest'; +import type { ParsedFile, ParsedImport } from 'gitnexus-shared'; import type { ScopeResolver } from '../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; export class CountingSet extends Set { @@ -85,6 +100,81 @@ export class CountingSet extends Set { } } +/** + * The `CountingSet` of the OTHER per-file-set key: the `parsedFiles` array the + * orchestrator passes as `resolveImportTarget`'s fifth argument + * (`scope-resolution/pipeline/run.ts`). PHP memoizes `filesByDirectory` on that + * array's identity and Python reads it in `pythonFileExportsName`, and neither + * touches the path Set while doing so — so without this the whole `context` + * channel is unmeasured. + * + * ## Element reads, not traversal entry points + * + * `CountingSet` can override the five ways a `Set` is walked and be done. An + * array has no such closed list: `for…of`, `forEach`, `map`, `filter`, + * `flatMap`, `reduce`, `find`, `some`, `every`, `indexOf` and a bare + * `for (let i = 0; i < a.length; i++)` all walk the same elements, and the last + * one goes through no method at all. Overriding a chosen subset would build in + * exactly the blind spot this instrument exists to remove — PHP's builder is a + * `for…of` today and one refactor away from an index loop. + * + * So the trap is on the read of an own indexed element. Every route above goes + * through it, including the index loop, and nothing else does: `length`, + * method lookups and `Symbol.iterator` are not counted. A full pass over N + * files therefore reads exactly N, and the number is a function of the file + * count and the number of passes — never of wall time. + * + * ## It counts THIS array only + * + * Reads of arrays DERIVED from it — the `ParsedFile[]` buckets inside PHP's + * directory index, the `candidateFiles` list filtered per import — are + * invisible, and deliberately so. That per-import work is bounded by the + * candidate set rather than by the workspace, so counting it would make the + * count grow with the import count for correct code and there would be no + * property left to assert. + * + * The `ParsedFile`s are minimal on purpose: `filePath` is the only field either + * consumer reads to build its index, and empty `localDefs` keeps both languages + * on their fallback answer, so the fixture measures the index and changes no + * resolution result. A test that needs the declaration legs to FIRE wants + * `php-import-target-parity.test.ts`, which carries defs. + */ +export interface CountedFileList { + /** Pass as `ImportResolutionContext.parsedFiles`. Stable identity, so it is + * a usable `perFileSet` key for the whole run. */ + readonly parsedFiles: readonly ParsedFile[]; + /** Reads of an own indexed element of `parsedFiles`, by any route. */ + readonly reads: () => number; +} + +/** Own array indices — `'0'`, `'12'`; not `'length'`, `'-1'` or `'01'`. */ +const ARRAY_INDEX = /^(?:0|[1-9][0-9]*)$/; + +/** + * A counted `parsedFiles` workspace for `filePaths`, one minimal `ParsedFile` + * each, in order. Build a FRESH one per run: the indexes are memoized on the + * array's identity, so two runs sharing one would have the second read the + * first's index and report zero. + */ +export function countedParsedFiles(filePaths: readonly string[]): CountedFileList { + const backing: ParsedFile[] = filePaths.map((filePath) => ({ + filePath, + moduleScope: `module:${filePath}`, + scopes: [], + parsedImports: [], + localDefs: [], + referenceSites: [], + })); + let reads = 0; + const counting = new Proxy(backing, { + get(target, key, receiver): unknown { + reads += typeof key === 'string' && ARRAY_INDEX.test(key) ? 1 : 0; + return Reflect.get(target, key, receiver); + }, + }); + return { parsedFiles: counting, reads: () => reads }; +} + /** * Everything that differs between the per-language spellings of the * distinct-file-set arm. Nothing else about that arm varies, which is why it is @@ -159,3 +249,240 @@ export function expectDistinctFileSetsGetOwnIndex(arm: DistinctFileSetArm): void expect(a.scans).toBe(arm.expectedScans); expect(b.scans).toBe(arm.expectedScans); } + +// ─── Python import shapes ──────────────────────────────────────────────────── + +/** + * `from import Widget`. The shape that makes + * `resolvePythonImportTarget` run the package-attribute probe + * (`pythonFileExportsName`, the `context.parsedFiles` reader) ahead of the + * submodule fallback — so it is the shape that re-enters the resolver and pays + * the importer's chain TWICE, and the shape `pythonImportedSubmoduleTarget` + * fires for. + * + * The default the adapter synthesizes when `context` is absent is a `namespace` + * import, and that shape never reaches the probe. A guard that means to measure + * either leg therefore has to pass this one explicitly. + */ +export const pythonNamedImport = (targetRaw: string): ParsedImport => ({ + kind: 'named', + localName: 'Widget', + importedName: 'Widget', + targetRaw, +}); + +/** `import ` — the single-walk shape, and the adapter's default. */ +export const pythonNamespaceImport = (targetRaw: string): ParsedImport => ({ + kind: 'namespace', + localName: '_', + importedName: '_', + targetRaw, +}); + +/** + * ONE array per file that uses it, never a fresh `[]` per call: + * `parsedFileByPath` memoizes on its identity, and a new array per import would + * mint a `WeakMap` key per import for a channel these guards are not measuring + * (`countedParsedFiles` above is the instrument for that one). Empty, so + * `pythonFileExportsName` answers false and the package-vs-submodule precedence + * never fires — the walk, not the precedence, is what the numbers measure. + */ +export const NO_PARSED_FILES: readonly ParsedFile[] = []; + +// ─── the Python importer-chain memo guards ─────────────────────────────────── + +/** + * What one resolution answered, as the chain-memo arms read it: a path, a path + * list (the `ScopeResolver` signature allows one), or `null`. The two values + * the non-vacuity pairing rule counts are `arm.hitResult` and `null`. + */ +export type ChainMemoResult = string | readonly string[] | null; + +/** + * Everything that differs between the two Python importer-chain memo guards: + * `test/unit/import-resolvers/python-importer-prefixes.test.ts` + * (`bareImportPrefixesByDir`) and + * `test/unit/scope-resolution/python/python-importer-ancestors.test.ts` + * (`ancestorsByDir`). + * + * The two memos hold DIFFERENT SEQUENCES under the same key — self included or + * not, workspace root included or not, empty components kept or dropped; see + * `importerBarePrefixes`'s header for why neither guard can be deleted in + * favour of the other. But each guard is the same four arms over the same + * importer corpus once these four values are named, so the arms live here and + * each guard supplies its own four. + */ +export interface ChainMemoArm { + /** The memo under test, read off the pass's per-file-set index. */ + readonly memoOf: (files: ReadonlySet) => ReadonlyMap; + /** + * Drives a production surface `perImporter` times from `fromFile`, with + * spellings that reach the memo, and answers what each call resolved to. + * Exactly one call per invocation must answer `arm.hitResult`, and at least + * one must answer `null`. Must pass `files` THROUGH: both memos are keyed on + * its identity, so a copy here would measure nothing. + */ + readonly drive: ( + files: Set, + fromFile: string, + perImporter: number, + ) => readonly ChainMemoResult[]; + /** + * The verbatim pre-change chain builder, which is the specification: the memo + * agreeing with it is what makes the change a hoist rather than a behaviour + * change. + */ + readonly legacyChain: (fromFile: string) => readonly string[]; + /** What the one must-resolve spelling in `drive` answers, once per importer. */ + readonly hitResult: string; +} + +/** Every file that issues an import in the chain-memo arms. */ +export const CHAIN_MEMO_IMPORTERS: readonly string[] = [ + 'svc/a/one.py', + 'svc/a/two.py', + 'svc/b/one.py', + 'deep/x/y/z/one.py', + 'root.py', +]; + +/** Four directories for those five importers — `svc/a` holds two of them. */ +export const CHAIN_MEMO_IMPORTER_DIRS: readonly string[] = ['svc/a', 'svc/b', 'deep/x/y/z', '']; + +/** The directory two importers share, which is where identity is measured. */ +const SHARED_DIR = 'svc/a'; +const SHARED_DIR_IMPORTERS: readonly string[] = ['svc/a/one.py', 'svc/a/two.py']; + +/** Imports one directory issues before its chain's identity is re-read. */ +const CHAIN_IDENTITY_REPEATS = 40; + +/** A sorted copy, so a key set is compared without depending on fill order. */ +export const sortedStrings = (values: Iterable): string[] => [...values].sort(); + +/** + * The importer directory both memos are keyed on, derived exactly as the + * pre-change inline code derived it — `''` for a path with no separator. + */ +const importerDirOf = (fromFile: string): string => + fromFile.replace(/\\/g, '/').split('/').slice(0, -1).join('/'); + +/** + * The path-shape space an importer chain has to be correct over, as ONE table + * both guards run: they enumerate the same space and nothing kept the two + * copies in lockstep. + * + * Each `why` names the SHAPE, not what either chain does with it, because the + * two chains do different things with several of these rows — the bare-prefix + * chain KEEPS the empty component an absolute path or a doubled separator + * produces and the ancestor chain drops it, and that difference decides real + * resolutions. Which is why every arm below compares against the guard's own + * `legacyChain` rather than against a shared expectation. + */ +export const IMPORTER_PATH_SHAPES: readonly { readonly fromFile: string; readonly why: string }[] = + [ + { fromFile: 'svc/a/one.py', why: 'a two-component directory' }, + { fromFile: 'deep/x/y/z/one.py', why: 'a four-component directory' }, + { fromFile: 'root.py', why: 'a workspace-root importer' }, + { fromFile: '/abs/svc/a/one.py', why: 'an absolute path (leading empty component)' }, + { fromFile: 'svc//a/one.py', why: 'a doubled separator (empty component)' }, + { fromFile: 'svc\\a\\one.py', why: 'Windows separators' }, + { fromFile: 'trailing/', why: 'a path ending in a separator' }, + ]; + +/** + * The gate: N imports from five importers over four directories leave FOUR + * entries, for every N. That is "the chain work is O(1) amortized after the + * first import from a given directory", stated as a number. A chain rebuilt per + * import cannot be memoized at all (size 0); a chain keyed on the importing + * FILE reads five. + * + * Paired with the non-vacuity assertions every guard in this family states: a + * perfect memo count is equally true of an adapter that resolves nothing. + */ +export function expectOneChainPerImporterDir( + arm: ChainMemoArm, + files: Set, + perImporter: number, +): void { + const resolved: ChainMemoResult[] = []; + for (const fromFile of CHAIN_MEMO_IMPORTERS) { + resolved.push(...arm.drive(files, fromFile, perImporter)); + } + + expect(arm.memoOf(files).size).toBe(CHAIN_MEMO_IMPORTER_DIRS.length); + expect(sortedStrings(arm.memoOf(files).keys())).toEqual(sortedStrings(CHAIN_MEMO_IMPORTER_DIRS)); + + expect(resolved.filter((value) => value === arm.hitResult)).toHaveLength( + CHAIN_MEMO_IMPORTERS.length, + ); + expect(resolved.filter((value) => value === null).length).toBeGreaterThan(0); +} + +/** + * The stored-object arm: a memo that stores a FRESH chain on every import posts + * a perfect size while doing all of the work again, so the size gate above is + * paired with reference identity across many later imports from the same + * directory — issued from BOTH files in it, so a chain keyed on the importing + * file would be replaced rather than reused. + * + * Contents are asserted FIRST: `toBe` against an absent entry would pass on + * `undefined === undefined` if the memo were deleted outright. + */ +export function expectSameChainObjectReused(arm: ChainMemoArm, files: Set): void { + const [firstImporter] = SHARED_DIR_IMPORTERS; + arm.drive(files, firstImporter, 1); + const first = arm.memoOf(files).get(SHARED_DIR); + expect(first).toEqual(arm.legacyChain(firstImporter)); + + for (const fromFile of SHARED_DIR_IMPORTERS) { + arm.drive(files, fromFile, CHAIN_IDENTITY_REPEATS); + } + + expect(arm.memoOf(files).get(SHARED_DIR)).toBe(first); +} + +/** + * The legacy-equality arm for one path shape: what the memo stored under + * `fromFile`'s directory is what the pre-change inline code built for it. + * + * Returns the memoized chain, so a guard whose memo feeds a SECOND consumer can + * go on to assert that consumer's derived form of it. + */ +export function expectMemoizedChainMatchesLegacy( + arm: ChainMemoArm, + files: Set, + fromFile: string, +): readonly string[] { + arm.drive(files, fromFile, 1); + + const chain = arm.memoOf(files).get(importerDirOf(fromFile)); + expect(chain).toEqual(arm.legacyChain(fromFile)); + return chain ?? []; +} + +/** + * The distinct-file-set arm: two independently built file sets each get their + * own memo — equal in content, never the same object, neither leaking into the + * other. The two are driven interleaved, so a memo keyed on anything but the + * Set's identity shows up here as a SHARED entry rather than as a stale one. + */ +export function expectDistinctFileSetsGetOwnChainMemo( + arm: ChainMemoArm, + a: Set, + b: Set, + perImporter: number, +): void { + for (const fromFile of CHAIN_MEMO_IMPORTERS) { + arm.drive(a, fromFile, perImporter); + arm.drive(b, fromFile, perImporter); + } + + const memoA = arm.memoOf(a); + const memoB = arm.memoOf(b); + + expect(memoA).not.toBe(memoB); + expect(memoA.get(SHARED_DIR)).not.toBe(memoB.get(SHARED_DIR)); + expect(memoA.get(SHARED_DIR)).toEqual(memoB.get(SHARED_DIR)); + expect(memoA.size).toBe(CHAIN_MEMO_IMPORTER_DIRS.length); + expect(memoB.size).toBe(CHAIN_MEMO_IMPORTER_DIRS.length); +} diff --git a/gitnexus/test/integration/cobol-import-index-reuse.test.ts b/gitnexus/test/integration/cobol-import-index-reuse.test.ts new file mode 100644 index 000000000..bcde62b05 --- /dev/null +++ b/gitnexus/test/integration/cobol-import-index-reuse.test.ts @@ -0,0 +1,133 @@ +/** + * Production-path regression guard for the COBOL `COPY`-target index (#2908). + * + * The two-tier basename index (`getCobolCopyIndex` in + * `languages/cobol/scope-resolver.ts`) is memoized on the `allFilePaths` Set + * identity via a WeakMap, so the file set must be passed THROUGH from the + * orchestrator, never copied. A defensive `new Set(allFilePaths)` in the + * adapter hands a fresh WeakMap key per call and rebuilds the index on every + * `COPY`, restoring the O(copies × files) scans this replaced — the exact bug + * PR #1918 shipped for Python and had to fix in review (P1). + * + * COBOL is the language where that copy costs the most: every `COPY` used to + * run TWO full scans, and mainframe repos are copybook-dense — one program can + * carry dozens of `COPY` statements. + * + * Unlike the other languages in this family, COBOL has no separate + * `resolveImportTarget` function; the adapter IS the resolver. The unit + * parity test (`test/unit/scope-resolution/cobol-import-target-parity.test.ts`) + * therefore reaches the same entry point — but it says nothing about Set + * identity, so a copy inserted there leaves every one of its arms green. This + * file is what notices, by counting traversals of the set. + * + * The traversal-count assertions are the perf guard. They are paired with + * result assertions on purpose: a count of 1 is equally true of an adapter that + * has stopped resolving anything at all, so counting alone would stay green + * while every COBOL COPY edge disappeared. + * + * Expected count is 1: both tiers are filled in a single pass over the set. + */ +import { describe, it, expect } from 'vitest'; +import { cobolScopeResolver } from '../../src/core/ingestion/languages/cobol/scope-resolver.js'; +import { CountingSet, expectDistinctFileSetsGetOwnIndex } from '../helpers/counting-file-set.js'; + +const { resolveImportTarget } = cobolScopeResolver; + +const FROM_FILE = 'src/PROG.cbl'; + +/** + * A synthetic mainframe checkout: many copybooks under `copybooks/`, plus the + * three files the arms below address — a copybook, a program reachable only + * through the SOURCE tier, and a program that a copybook of the same name must + * beat despite coming first in Set-iteration order. + */ +function buildWorkspace(fileCount: number): CountingSet { + const files: string[] = []; + // Inserted before the copybook below so the tier-order arm is a real + // tie-break rather than an artefact of ordering. + files.push('src/CUSTREC.cbl'); + for (let i = 0; i < fileCount; i++) { + files.push(`copybooks/BOOK${String(i).padStart(5, '0')}.cpy`); + } + files.push('copybooks/CUSTREC.cpy'); + files.push('src/PAYROLL.cbl'); + files.push('src/PROG.cbl'); + return new CountingSet(files); +} + +describe('COBOL COPY resolution — index reuse across imports (#2908)', () => { + it('builds the file index once for many COPY statements over a stable file set', () => { + const files = buildWorkspace(300); + const resolved: (string | readonly string[] | null)[] = []; + + for (let i = 0; i < 200; i++) { + // Three shapes: a copybook hit, a hit that only the SOURCE tier answers, + // and a member that is not in the repo at all — the last is the common + // case in real COBOL (vendor and system copybooks) and the one that used + // to cost TWO full workspace scans per statement. + resolved.push(resolveImportTarget('CUSTREC', FROM_FILE, files, undefined)); + resolved.push(resolveImportTarget('PAYROLL', FROM_FILE, files, undefined)); + resolved.push(resolveImportTarget(`VENDOR${i}`, FROM_FILE, files, undefined)); + } + + expect(files.scans).toBe(1); + + // Paired result assertions — a count of 1 must not be the count of an + // adapter that resolves nothing. The first also pins the tier order: the + // `.cbl` twin was inserted FIRST. + expect(resolved[0]).toBe('copybooks/CUSTREC.cpy'); + expect(resolved[1]).toBe('src/PAYROLL.cbl'); + expect(resolved[2]).toBeNull(); + }); + + it('a distinct file set gets its own index (no stale cross-run reuse)', () => { + expectDistinctFileSetsGetOwnIndex({ + resolveImportTarget, + buildWorkspace: () => buildWorkspace(20), + targetRaw: 'CUSTREC', + fromFile: FROM_FILE, + resolutionConfig: undefined, + expected: 'copybooks/CUSTREC.cpy', + expectedScans: 1, + }); + }); + + it('still resolves real COPY statements correctly (the perf test is not vacuous)', () => { + const files = new CountingSet([ + 'src/CUSTREC.cbl', + 'copybooks/CUSTREC.cpy', + 'copybooks/custrec-lower.copybook', + 'copybooks/Mixed.CPY', + 'src/PAYROLL.cob', + 'src/TAXCALC.cobol', + 'docs/CUSTREC.txt', + 'copybooks/NOEXT', + ]); + + // Tier order: the copybook wins over the `.cbl` inserted before it. + expect(resolveImportTarget('CUSTREC', FROM_FILE, files, undefined)).toBe( + 'copybooks/CUSTREC.cpy', + ); + // Case: the COPY operand and the file's stem are both upper-cased. + expect(resolveImportTarget('custrec', FROM_FILE, files, undefined)).toBe( + 'copybooks/CUSTREC.cpy', + ); + expect(resolveImportTarget('CUSTREC-LOWER', FROM_FILE, files, undefined)).toBe( + 'copybooks/custrec-lower.copybook', + ); + // Source tier, reached only after every copybook missed. + expect(resolveImportTarget('PAYROLL', FROM_FILE, files, undefined)).toBe('src/PAYROLL.cob'); + expect(resolveImportTarget('TAXCALC', FROM_FILE, files, undefined)).toBe('src/TAXCALC.cobol'); + // `path.basename(fp, '.cpy')` will not strip `.CPY`, so the stem keeps it. + expect(resolveImportTarget('MIXED.CPY', FROM_FILE, files, undefined)).toBe( + 'copybooks/Mixed.CPY', + ); + expect(resolveImportTarget('MIXED', FROM_FILE, files, undefined)).toBeNull(); + // Neither tier: a `.txt`, a file with no extension, and an absent member. + expect(resolveImportTarget('NOEXT', FROM_FILE, files, undefined)).toBeNull(); + expect(resolveImportTarget('ABSENT', FROM_FILE, files, undefined)).toBeNull(); + + // One traversal covered all of it. + expect(files.scans).toBe(1); + }); +}); diff --git a/gitnexus/test/integration/csharp-import-index-reuse.test.ts b/gitnexus/test/integration/csharp-import-index-reuse.test.ts index cd6887400..8f4978260 100644 --- a/gitnexus/test/integration/csharp-import-index-reuse.test.ts +++ b/gitnexus/test/integration/csharp-import-index-reuse.test.ts @@ -27,35 +27,95 @@ * result assertions on purpose: a count of 2 is equally true of an adapter that * has stopped resolving anything at all, so counting alone would stay green * while every C# IMPORTS edge disappeared. + * + * ## The csproj leg is guarded by the SAME instrument (#2911 review) + * + * With `.csproj` configs present the adapter takes a different branch entirely + * — `resolveCSharpImportInternal` — and that branch used to be unguarded here: + * no arm supplied `csharpConfigs`, so no counting Set ever entered it. Worse, + * its namespace-directory index was keyed on the `normalizedFileList` ARRAY, a + * shape no scan count can instrument — a `[...normalized]` copy at the adapter + * boundary rebuilt the index once per `using` while traversing the Set exactly + * zero extra times. Reproduced against this PR's tree: the copy left all 67 + * tests of the four import-index guards green and only the timing bench + * noticed (`csharp_csproj scaling 3.556 > 1.8`). + * + * #2911 rekeyed that index onto the Set, so the array shape is gone and the + * only remaining way to defeat the memo — copying the Set — is what + * `CountingSet` already counts. The csproj arms below therefore read the same + * one number as the arms above, with no second instrument. */ import { describe, it, expect } from 'vitest'; import { csharpScopeResolver } from '../../src/core/ingestion/languages/csharp/scope-resolver.js'; +import type { CsharpResolutionConfig } from '../../src/core/ingestion/languages/csharp/resolution-config.js'; import { CountingSet, expectDistinctFileSetsGetOwnIndex } from '../helpers/counting-file-set.js'; const { resolveImportTarget } = csharpScopeResolver; const FROM_FILE = 'App/Program.cs'; +/** Where a workspace's padded filler files go, and what is appended after them. */ +interface WorkspaceLayout { + /** Directory the filler files live in. */ + readonly dir: string; + /** Basename stem the filler files are numbered from. */ + readonly stem: string; + /** The files the resolutions actually target, appended in order. */ + readonly extras: readonly string[]; +} + +/** + * `fileCount` filler files under `layout.dir`, then `layout.extras`. The filler + * is what makes a traversal expensive enough for a per-`using` rebuild to be a + * different number rather than a different constant; the counting instrument + * reads the traversals either way. + */ +function buildWorkspace(fileCount: number, layout: WorkspaceLayout): CountingSet { + const files: string[] = []; + for (let i = 0; i < fileCount; i++) { + files.push(`${layout.dir}/${layout.stem}${String(i).padStart(5, '0')}.cs`); + } + return new CountingSet([...files, ...layout.extras]); +} + /** * A synthetic C# solution with no `.csproj` discovered, which is the leg #2878 * moved onto the indexes. `App/Models/User.cs` answers the whole-path lookup, * `App/Services/` answers the namespace-directory lookup, and `Domain/Order.cs` * is reachable only after progressive prefix stripping. */ -function buildWorkspace(fileCount: number): CountingSet { - const files: string[] = []; - for (let i = 0; i < fileCount; i++) { - files.push(`App/Services/Service${String(i).padStart(5, '0')}.cs`); - } - files.push('App/Models/User.cs'); - files.push('Domain/Order.cs'); - files.push('App/Program.cs'); - return new CountingSet(files); -} +const NO_CSPROJ_LAYOUT: WorkspaceLayout = { + dir: 'App/Services', + stem: 'Service', + extras: ['App/Models/User.cs', 'Domain/Order.cs', 'App/Program.cs'], +}; + +/** The one `.csproj` config that puts the adapter on the csproj leg. */ +const CSPROJ_CONFIG: CsharpResolutionConfig = { + csharpConfigs: [{ rootNamespace: 'App', projectDir: 'App' }], +}; + +/** + * A workspace whose `App.Models` resolution reaches the namespace-DIRECTORY + * index, which is the only thing on the csproj leg keyed on the array. + * + * That takes a layout the first two legs both miss. `src/MyApp/Models/` answers + * `dirPrefix = 'App/Models'` under the unanchored substring rule ('MyApp/' + * supplies the 'App/'), and under nothing weaker: no file is named + * `App/Models.cs` or `Models.cs`, so the single-file leg misses, and no + * directory has the SEGMENT suffix `App/Models`, so `getFilesInDir` misses too. + * A layout where the first two legs answer would leave the index unbuilt and + * the build count blind to the very copy it is here to catch. + */ +const CSPROJ_LAYOUT: WorkspaceLayout = { + dir: 'src/MyApp/Models', + stem: 'Entity', + extras: ['App/Program.cs'], +}; describe('C# import resolution — index reuse across usings (#2878)', () => { it('builds each index once for many usings over a stable file set', () => { - const files = buildWorkspace(300); + const files = buildWorkspace(300, NO_CSPROJ_LAYOUT); const resolved: (string | readonly string[] | null)[] = []; for (let i = 0; i < 200; i++) { @@ -82,7 +142,7 @@ describe('C# import resolution — index reuse across usings (#2878)', () => { it('a distinct file set gets its own indexes (no stale cross-run reuse)', () => { expectDistinctFileSetsGetOwnIndex({ resolveImportTarget, - buildWorkspace: () => buildWorkspace(20), + buildWorkspace: () => buildWorkspace(20, NO_CSPROJ_LAYOUT), targetRaw: 'App.Models.User', fromFile: FROM_FILE, resolutionConfig: undefined, @@ -94,7 +154,7 @@ describe('C# import resolution — index reuse across usings (#2878)', () => { }); it('still resolves real usings correctly (the perf test is not vacuous)', () => { - const files = buildWorkspace(5); + const files = buildWorkspace(5, NO_CSPROJ_LAYOUT); // Whole-path match on the namespace path. expect(resolveImportTarget('App.Models.User', FROM_FILE, files, undefined)).toBe( @@ -114,3 +174,43 @@ describe('C# import resolution — index reuse across usings (#2878)', () => { expect(resolveImportTarget('Vendor.Ghost.Missing', FROM_FILE, files, undefined)).toBeNull(); }); }); + +describe('C# import resolution — index reuse on the csproj leg (#2911)', () => { + it('builds each index once for many usings over a stable file set', () => { + const files = buildWorkspace(300, CSPROJ_LAYOUT); + const resolved: (string | readonly string[] | null)[] = []; + + for (let i = 0; i < 200; i++) { + // A namespace-directory hit and a miss, both reaching the array-keyed + // index — the miss under a fresh namespace each time so no upstream + // string-level memo can stand in for the index being reused. + resolved.push(resolveImportTarget('App.Models', FROM_FILE, files, CSPROJ_CONFIG)); + resolved.push(resolveImportTarget(`App.Ghost${i}`, FROM_FILE, files, CSPROJ_CONFIG)); + } + + // One traversal for 400 usings. One, not two: `getCsharpDirIndex` belongs to + // the no-csproj leg, and the namespace-directory index this branch DOES + // build reads its file list from the same `getWorkspaceFileIndex` memo + // rather than re-walking the Set. A defensive copy of the Set at the + // adapter boundary reads 400 here. + expect(files.scans).toBe(1); + + // Paired result assertions — the count must not be the count of an adapter + // that resolves nothing. + expect(resolved[0]).toBe('src/MyApp/Models/Entity00000.cs'); + expect(resolved[1]).toBeNull(); + }); + + it('a distinct file set gets its own indexes (no stale cross-run reuse)', () => { + expectDistinctFileSetsGetOwnIndex({ + resolveImportTarget, + buildWorkspace: () => buildWorkspace(20, CSPROJ_LAYOUT), + targetRaw: 'App.Models', + fromFile: FROM_FILE, + resolutionConfig: CSPROJ_CONFIG, + expected: 'src/MyApp/Models/Entity00000.cs', + // One, not two: see the scan-count comment above. + expectedScans: 1, + }); + }); +}); diff --git a/gitnexus/test/integration/go-import-index-reuse.test.ts b/gitnexus/test/integration/go-import-index-reuse.test.ts index b0672760b..ac798e471 100644 --- a/gitnexus/test/integration/go-import-index-reuse.test.ts +++ b/gitnexus/test/integration/go-import-index-reuse.test.ts @@ -12,11 +12,11 @@ * replaced. Python hit exactly that (PR #1918 review P1), and the parity test * cannot see it: it never crosses the adapter. * - * Kotlin and Python count index BUILDS from production (`index-stats.ts`). - * These four use `CountingSet` (`test/helpers/counting-file-set.ts`) instead, + * Every one of these guards uses `CountingSet` (`test/helpers/counting-file-set.ts`), * which counts full traversals of the file set and so catches BOTH the * per-import rebuild and a scan reintroduced beside a reused index — with no - * production surface added for a test-only observation. + * production surface added for a test-only observation. Kotlin and Python + * counted index BUILDS from production until #2909 moved them onto this one. * * The traversal-count assertions are the perf guard. They are paired with * result assertions on purpose: a count of 1 is equally true of an adapter that diff --git a/gitnexus/test/integration/java-import-index-reuse.test.ts b/gitnexus/test/integration/java-import-index-reuse.test.ts new file mode 100644 index 000000000..cb2030005 --- /dev/null +++ b/gitnexus/test/integration/java-import-index-reuse.test.ts @@ -0,0 +1,130 @@ +/** + * Production-path regression guard for the Java import-resolution indexes + * (#2908). + * + * `resolveJavaImportTarget` reads TWO per-file-set indexes, each memoized on + * the `allFilePaths` Set identity via its own WeakMap: the shared + * `getWorkspaceFileIndex` (`import-resolvers/workspace-file-index.ts`, which + * answers the whole-path and segment-suffix legs) and `getJavaDirIndex` + * (`languages/java/import-target.ts`, the package-directory index behind + * `firstFileDirectlyInPkgDir`). Before the hoist every leg was a full + * `allFilePaths` scan, and the progressive-stripping loop re-ran that scan once + * per stripped segment — so a four-segment `import` that resolves to nothing, + * which is what every JDK and third-party import does, cost four full passes. + * + * Resolution reaches both indexes through `javaScopeResolver.resolveImportTarget` + * — the orchestrator adapter — not by calling `resolveJavaImportTarget` directly + * the way the unit parity test does. The adapter must therefore pass the Set + * THROUGH; a defensive copy (`new Set(allFilePaths)`) would hand a fresh WeakMap + * key per call and rebuild BOTH indexes on every import, restoring the + * O(imports × files) behaviour this replaced. Python hit exactly that (PR #1918 + * review P1), and `test/unit/scope-resolution/java-import-target-parity.test.ts` + * cannot see it: it never crosses the adapter. + * + * The counting instrument has to be a real `Set` subclass: `narrowContext` + * rejects a workspace context whose `allFilePaths` fails `instanceof Set`, and a + * rejected context resolves nothing — every assertion would then pass on + * `null === null`. + * + * The traversal-count assertions are the perf guard. They are paired with result + * assertions on purpose: a count of 2 is equally true of an adapter that has + * stopped resolving anything at all, so counting alone would stay green while + * every Java IMPORTS edge disappeared. + */ +import { describe, it, expect } from 'vitest'; +import { javaScopeResolver } from '../../src/core/ingestion/languages/java/scope-resolver.js'; +import { CountingSet, expectDistinctFileSetsGetOwnIndex } from '../helpers/counting-file-set.js'; + +const { resolveImportTarget } = javaScopeResolver; + +const FROM_FILE = 'src/main/java/com/example/App.java'; + +/** + * A synthetic Java source tree covering all four legs of the cascade: + * `com/example/model/User.java` answers the whole-path lookup, + * `src/main/java/com/example/service/` answers the package-directory lookup a + * wildcard import lands on, `src/main/java/com/example/util/Strings.java` + * answers the nested-suffix lookup, and `domain/Order.java` is reachable only + * after progressive prefix stripping. + */ +function buildWorkspace(fileCount: number): CountingSet { + const files: string[] = []; + for (let i = 0; i < fileCount; i++) { + files.push(`src/main/java/com/example/service/Service${String(i).padStart(5, '0')}.java`); + } + files.push('com/example/model/User.java'); + files.push('src/main/java/com/example/util/Strings.java'); + files.push('domain/Order.java'); + files.push(FROM_FILE); + return new CountingSet(files); +} + +describe('Java import resolution — index reuse across imports (#2908)', () => { + it('builds each index once for many imports over a stable file set', () => { + const files = buildWorkspace(300); + const resolved: (string | readonly string[] | null)[] = []; + + for (let i = 0; i < 200; i++) { + // A whole-path hit, a nested-suffix hit, a package-directory hit via a + // wildcard, and a miss that runs the full progressive-stripping cascade — + // the case that used to re-scan the workspace once per stripped prefix. + resolved.push(resolveImportTarget('com.example.model.User', FROM_FILE, files, undefined)); + resolved.push(resolveImportTarget('com.example.util.Strings', FROM_FILE, files, undefined)); + resolved.push(resolveImportTarget('com.example.service.*', FROM_FILE, files, undefined)); + resolved.push( + resolveImportTarget(`vendor${i}.ghost.deep.Missing`, FROM_FILE, files, undefined), + ); + } + + // Two passes: the shared workspace/suffix index and the package-dir index. + // Not one: they are separate WeakMaps and `buildPackageDirIndex` takes the + // Set, so each iterates it once — the same accounting as C# (#2878). + expect(files.scans).toBe(2); + + // Paired result assertions — a count of 2 must not be the count of an + // adapter that resolves nothing. + expect(resolved[0]).toBe('com/example/model/User.java'); + expect(resolved[1]).toBe('src/main/java/com/example/util/Strings.java'); + expect(resolved[2]).toBe('src/main/java/com/example/service/Service00000.java'); + expect(resolved[3]).toBeNull(); + }); + + it('a distinct file set gets its own indexes (no stale cross-run reuse)', () => { + expectDistinctFileSetsGetOwnIndex({ + resolveImportTarget, + buildWorkspace: () => buildWorkspace(20), + targetRaw: 'com.example.model.User', + fromFile: FROM_FILE, + resolutionConfig: undefined, + expected: 'com/example/model/User.java', + // Two, not one: the shared workspace/suffix index and the package-dir + // index are separate WeakMaps over the same Set. + expectedScans: 2, + }); + }); + + it('still resolves real imports correctly (the perf test is not vacuous)', () => { + const files = buildWorkspace(5); + + // Whole-path match on the package path. + expect(resolveImportTarget('com.example.model.User', FROM_FILE, files, undefined)).toBe( + 'com/example/model/User.java', + ); + // Nested suffix match under the source root. + expect(resolveImportTarget('com.example.util.Strings', FROM_FILE, files, undefined)).toBe( + 'src/main/java/com/example/util/Strings.java', + ); + // Wildcard: `.*` is stripped and the package directory answers with its + // first `.java` child in file-set order. + expect(resolveImportTarget('com.example.service.*', FROM_FILE, files, undefined)).toBe( + 'src/main/java/com/example/service/Service00000.java', + ); + // Progressive prefix stripping: the repo has no `com/shop/` prefix. + expect(resolveImportTarget('com.shop.domain.Order', FROM_FILE, files, undefined)).toBe( + 'domain/Order.java', + ); + + // Unknown packages resolve to nothing. + expect(resolveImportTarget('vendor.ghost.Missing', FROM_FILE, files, undefined)).toBeNull(); + }); +}); diff --git a/gitnexus/test/integration/javascript-import-index-reuse.test.ts b/gitnexus/test/integration/javascript-import-index-reuse.test.ts new file mode 100644 index 000000000..d1768c742 --- /dev/null +++ b/gitnexus/test/integration/javascript-import-index-reuse.test.ts @@ -0,0 +1,175 @@ +/** + * Production-path regression guard for the JavaScript import-resolution index + * (#2910). + * + * `makeJsResolveImportTarget`'s `PassCache` was the TypeScript one minus its + * `index` field, so every JavaScript import reached `suffixResolve` with + * `index === undefined` and took the linear-`findIndex` fallback: one pass over + * `normalizedFileList` per path part per extension, ~39 extensions. 6448.9 µs + * per import at 2000 files and 25972.6 µs at 8000 — 4.12x the per-import cost + * for 4x the files, which is O(imports × files) — against 25.0 / 27.0 µs for + * TypeScript over the identical corpus. With the index it is 28.5 / 27.4 µs and + * the scaling factor is 1.09x. + * + * ## Why the existing guards were blind to it + * + * `CountingSet` counts traversals of the SET, and this scan walked the array + * the adapter had already materialized from it (`test/helpers/counting-file-set.ts` + * says so under "What it does NOT see"). The pass cache was reused correctly, + * so the traversal count read 2 with the defect and reads 2 without it — the + * sixteen-language contract test scored `javascript` a clean pass throughout. + * + * So the arm that would have caught this is not a count of Set traversals but + * `resolves a repo-root module by bare specifier` below: without an index a + * repo-root file is unreachable through this leg, because the scan tests + * `endsWith('/' + suffix)` and a root-level path has no `/`. It is a behaviour + * assertion, it is deterministic, and it fails the moment `index` leaves the + * cache. The direct instrument — counting entries into `suffixResolve`'s linear + * branch, with the pre-index adapter as its control — lives beside the + * differential in `test/unit/scope-resolution/javascript-import-target-parity.test.ts`. + * + * ## What the traversal counts here do guard + * + * Resolution reaches the cache through `javascriptScopeResolver.resolveImportTarget` + * — the orchestrator adapter — which must pass the Set THROUGH: a defensive + * `new Set(allFilePaths)` hands a fresh `WeakMap` key per import and restores + * the per-import rebuild (PR #1918 review P1). Two traversals per file set, not + * one: the adapter materializes `allFileList` and then keeps one mutable copy + * of the Set, because `TsResolveContext.allFilePaths` is a `Set`, not a + * `ReadonlySet`. + * + * The counts are paired with result assertions on purpose: a count of 2 is + * equally true of an adapter that has stopped resolving anything at all. + */ +import { describe, it, expect } from 'vitest'; +import { javascriptScopeResolver } from '../../src/core/ingestion/languages/javascript/scope-resolver.js'; +import { CountingSet, expectDistinctFileSetsGetOwnIndex } from '../helpers/counting-file-set.js'; + +const { resolveImportTarget } = javascriptScopeResolver; + +const FROM_FILE = 'src/main.js'; + +/** + * A synthetic CommonJS/ESM app covering the legs the resolver takes: a relative + * import answered by exact `Set.has`, a bare specifier answered by path suffix, + * a `node_modules` package, a directory `index.js`, and `config.js` at the repo + * root — the one shape only the index can reach. + */ +function buildWorkspace(fileCount: number): CountingSet { + const files: string[] = []; + for (let i = 0; i < fileCount; i++) { + files.push(`src/components/Widget${String(i).padStart(5, '0')}.js`); + } + files.push('src/util.js'); + files.push('src/models/index.js'); + files.push('lib/esm.mjs'); + files.push('node_modules/dep/index.js'); + files.push('config.js'); + files.push('bootstrap.cjs'); + files.push(FROM_FILE); + return new CountingSet(files); +} + +describe('JavaScript import resolution — index reuse across imports (#2910)', () => { + it('builds the pass cache once for many imports over a stable file set', () => { + const files = buildWorkspace(300); + const resolved: (string | readonly string[] | null)[] = []; + + for (let i = 0; i < 200; i++) { + // A relative hit, a bare-specifier suffix hit, a repo-root hit, and a + // bare specifier that misses. The miss is the expensive case: it runs + // every path part × every extension before returning null, which is the + // loop that used to scan the whole file list each time round. + resolved.push(resolveImportTarget('./util', FROM_FILE, files, undefined)); + resolved.push(resolveImportTarget('src/models', FROM_FILE, files, undefined)); + resolved.push(resolveImportTarget('config', FROM_FILE, files, undefined)); + resolved.push(resolveImportTarget(`@vendor/ghost${i}/deep`, FROM_FILE, files, undefined)); + } + + // Two: `Array.from(allFilePaths)` and the one mutable `Set` copy the + // resolver context requires. Both happen once per file set. + expect(files.scans).toBe(2); + + // Paired result assertions — a count of 2 must not be the count of an + // adapter that resolves nothing. + expect(resolved[0]).toBe('src/util.js'); + expect(resolved[1]).toBe('src/models/index.js'); + expect(resolved[2]).toBe('config.js'); + expect(resolved[3]).toBeNull(); + }); + + it('a distinct file set gets its own index (no stale cross-run reuse)', () => { + expectDistinctFileSetsGetOwnIndex({ + resolveImportTarget, + buildWorkspace: () => buildWorkspace(20), + targetRaw: 'src/models', + fromFile: FROM_FILE, + resolutionConfig: undefined, + expected: 'src/models/index.js', + expectedScans: 2, + }); + }); + + /** + * The per-file-set index and `resolveCache` must not become global. The arm + * above cannot see that: `expectDistinctFileSetsGetOwnIndex` builds two + * IDENTICAL corpora, so a stale answer carried across them is also the right + * answer, and only its traversal counts would notice. These two workspaces + * answer the same specifier differently, and they are resolved alternately. + */ + it('two different workspaces answer the same specifier differently', () => { + const a = new Set(['src/util.js', FROM_FILE]); + const b = new Set(['vendor/util.js', FROM_FILE]); + + expect(resolveImportTarget('util', FROM_FILE, a, undefined)).toBe('src/util.js'); + expect(resolveImportTarget('util', FROM_FILE, b, undefined)).toBe('vendor/util.js'); + expect(resolveImportTarget('util', FROM_FILE, a, undefined)).toBe('src/util.js'); + expect(resolveImportTarget('util', FROM_FILE, b, undefined)).toBe('vendor/util.js'); + }); + + /** + * The arm that fails without the suffix index, and the reason it is here + * rather than in the counting arms above: a repo-root file has no `/`, so + * `suffixResolve`'s scan — which tests `endsWith('/' + suffix)` — can never + * match it, while `buildSuffixIndex` indexes the whole path and can. + * Dropping `index` from the pass cache turns every one of these back to null + * while leaving `files.scans` at 2. + */ + it('resolves a repo-root module by bare specifier — impossible without the index', () => { + const files = buildWorkspace(5); + + expect(resolveImportTarget('config', FROM_FILE, files, undefined)).toBe('config.js'); + expect(resolveImportTarget('bootstrap', FROM_FILE, files, undefined)).toBe('bootstrap.cjs'); + + // Not `'config.js'`, and that is unchanged by the index: a specifier with + // no `/` has its dots turned into slashes before the suffix cascade + // (`resolveImportPath`), so `config.js` is looked up as `config/js`. + // TypeScript answers null here too — it is the same code path. + expect(resolveImportTarget('config.js', FROM_FILE, files, undefined)).toBeNull(); + }); + + it('still resolves real imports correctly (the perf test is not vacuous)', () => { + const files = buildWorkspace(5); + + // Relative, with and without an extension. + expect(resolveImportTarget('./util', FROM_FILE, files, undefined)).toBe('src/util.js'); + expect(resolveImportTarget('./util.js', FROM_FILE, files, undefined)).toBe('src/util.js'); + // Directory index. + expect(resolveImportTarget('./models', FROM_FILE, files, undefined)).toBe( + 'src/models/index.js', + ); + // Bare specifier resolved by path suffix, and an ESM extension. + expect(resolveImportTarget('components/Widget00000', FROM_FILE, files, undefined)).toBe( + 'src/components/Widget00000.js', + ); + expect(resolveImportTarget('lib/esm', FROM_FILE, files, undefined)).toBe('lib/esm.mjs'); + // A package in node_modules. + expect(resolveImportTarget('dep', FROM_FILE, files, undefined)).toBe( + 'node_modules/dep/index.js', + ); + + // Nothing in the repo answers these. + expect(resolveImportTarget('./nowhere', FROM_FILE, files, undefined)).toBeNull(); + expect(resolveImportTarget('@vendor/ghost/deep', FROM_FILE, files, undefined)).toBeNull(); + }); +}); diff --git a/gitnexus/test/integration/kotlin-import-index-reuse.test.ts b/gitnexus/test/integration/kotlin-import-index-reuse.test.ts index 24a43f5b5..110ea80e4 100644 --- a/gitnexus/test/integration/kotlin-import-index-reuse.test.ts +++ b/gitnexus/test/integration/kotlin-import-index-reuse.test.ts @@ -10,17 +10,31 @@ * call and rebuild the index on every import, restoring the O(imports × files) * behaviour this replaced. Python hit exactly that (PR #1918 review P1). * - * The build-count assertions are the perf guard. They are paired with result - * assertions on purpose: a build count of 1 is equally true of an adapter that - * has stopped resolving anything at all, so counting alone would stay green - * while every Kotlin IMPORTS edge disappeared. + * ## Why this counts TRAVERSALS and not index builds (#2909) + * + * This guard used to read a build counter that shipped in production purely so + * a test could read it (now deleted). `CountingSet` + * (`test/helpers/counting-file-set.ts`) replaces it, and the swap is not a + * wash: + * + * - STRICTLY MORE COVERAGE. A scan added BESIDE a reused index moves no build + * count — the cache still hits, the counter still reads 1 — but it does move + * the traversal count. That mutation is the one `bench/import-target/` + * provably cannot see either: `baselines.json` `_blind_spot` records a full + * workspace scan on 1-in-32 imports passing every timing arm. + * - LESS PRODUCTION SURFACE. ~30 lines shipped in the bundle whose only caller + * outside a cache miss was this file. + * - PARALLEL-SAFE. The counter lives on the instance the test built, so there + * is no module-global to `reset()` and no ordering hazard between tests. + * + * The traversal-count assertions are the perf guard. They are paired with result + * assertions on purpose: a count of 1 is equally true of an adapter that has + * stopped resolving anything at all, so counting alone would stay green while + * every Kotlin IMPORTS edge disappeared. */ import { describe, it, expect } from 'vitest'; import { kotlinScopeResolver } from '../../src/core/ingestion/languages/kotlin/scope-resolver.js'; -import { - getKotlinFileIndexBuildCount, - resetKotlinFileIndexBuildCount, -} from '../../src/core/ingestion/languages/kotlin/index-stats.js'; +import { CountingSet, expectDistinctFileSetsGetOwnIndex } from '../helpers/counting-file-set.js'; // `resolveImportTarget` is a required member of `ScopeResolver`, so this is a // plain read — no optional call, and no `toBeDefined()` guarding a branch that @@ -32,13 +46,13 @@ const { resolveImportTarget } = kotlinScopeResolver; * over one shared package namespace, so a package is reachable only as a path * suffix and never at the workspace root. */ -function buildWorkspace(fileCount: number): Set { - const files = new Set(); +function buildWorkspace(fileCount: number): CountingSet { + const files: string[] = []; for (let i = 0; i < fileCount; i++) { - files.add(`lib${String(i).padStart(5, '0')}/src/main/kotlin/com/example/widget/Widget${i}.kt`); + files.push(`lib${String(i).padStart(5, '0')}/src/main/kotlin/com/example/widget/Widget${i}.kt`); } - files.add('common/src/main/kotlin/com/example/common/Util.kt'); - return files; + files.push('common/src/main/kotlin/com/example/common/Util.kt'); + return new CountingSet(files); } const FROM_FILE = 'common/src/main/kotlin/com/example/common/Util.kt'; @@ -46,29 +60,44 @@ const FROM_FILE = 'common/src/main/kotlin/com/example/common/Util.kt'; describe('Kotlin import resolution — index reuse across imports', () => { it('builds the file index once for many imports over a stable file set', () => { const files = buildWorkspace(300); - resetKotlinFileIndexBuildCount(); + const resolved: (string | readonly string[] | null)[] = []; - for (let i = 0; i < 200; i++) { - // Alternates the two tiers that dominate real Kotlin source: a named type - // (tier 1, reached by path suffix) and a top-level function, which has no - // file named after it and so falls through to the package fan-out + for (let i = 0; i < 100; i++) { + // Both tiers that dominate real Kotlin source, every iteration: a named + // type (tier 1, reached by path suffix) and a top-level function, which + // has no file named after it and so falls through to the package fan-out // (#1759). Driving one tier only would leave the other unmeasured here. - const target = - i % 2 === 0 ? `com.example.widget.Widget${i}` : `com.example.widget.someTopLevelFun${i}`; - resolveImportTarget(target, FROM_FILE, files); + resolved.push( + resolveImportTarget(`com.example.widget.Widget${i}`, FROM_FILE, files, undefined), + ); + resolved.push( + resolveImportTarget(`com.example.widget.someTopLevelFun${i}`, FROM_FILE, files, undefined), + ); } + // An import that matches nothing at all, which runs the whole cascade — + // every tier misses and the progressive prefix strip walks to the end. + resolved.push(resolveImportTarget('org.absent.pkg.Missing', FROM_FILE, files, undefined)); - expect(getKotlinFileIndexBuildCount()).toBe(1); + expect(files.scans).toBe(1); + + // Paired result assertions — a count of 1 must not be the count of an + // adapter that resolves nothing. Tier 1 by path suffix, tier 3 fanning out + // over the package directory, and the total miss. + expect(resolved[0]).toBe('lib00000/src/main/kotlin/com/example/widget/Widget0.kt'); + expect(resolved[1]).toHaveLength(300); + expect(resolved[200]).toBeNull(); }); - it('rebuilds when the file set is a different object', () => { - resetKotlinFileIndexBuildCount(); - - for (let i = 0; i < 3; i++) { - resolveImportTarget('com.example.common.Util', 'a/B.kt', buildWorkspace(5)); - } - - expect(getKotlinFileIndexBuildCount()).toBe(3); + it('a distinct file set gets its own index (no stale cross-run reuse)', () => { + expectDistinctFileSetsGetOwnIndex({ + resolveImportTarget, + buildWorkspace: () => buildWorkspace(5), + targetRaw: 'com.example.common.Util', + fromFile: 'a/B.kt', + resolutionConfig: undefined, + expected: 'common/src/main/kotlin/com/example/common/Util.kt', + expectedScans: 1, + }); }); it('still resolves real imports correctly (the perf test is not vacuous)', () => { @@ -76,7 +105,7 @@ describe('Kotlin import resolution — index reuse across imports', () => { // Tier 1 through the adapter. The package sits under a module source root, // so this resolves by path suffix, not by an exact workspace-rooted match. - expect(resolveImportTarget('com.example.widget.Widget7', FROM_FILE, files)).toBe( + expect(resolveImportTarget('com.example.widget.Widget7', FROM_FILE, files, undefined)).toBe( 'lib00007/src/main/kotlin/com/example/widget/Widget7.kt', ); @@ -84,11 +113,19 @@ describe('Kotlin import resolution — index reuse across imports', () => { // it, so the stripped path resolves to the package directory and fans out // to every file in it. The finalize pass then picks the one whose localDefs // export the name (#1759). - const fanOut = resolveImportTarget('com.example.widget.someTopLevelFun', FROM_FILE, files); + const fanOut = resolveImportTarget( + 'com.example.widget.someTopLevelFun', + FROM_FILE, + files, + undefined, + ); expect(fanOut).toHaveLength(20); expect(fanOut).toContain('lib00000/src/main/kotlin/com/example/widget/Widget0.kt'); // An import that matches nothing in the workspace resolves to null. - expect(resolveImportTarget('org.absent.pkg.Missing', FROM_FILE, files)).toBeNull(); + expect(resolveImportTarget('org.absent.pkg.Missing', FROM_FILE, files, undefined)).toBeNull(); + + // All of it off one traversal. + expect(files.scans).toBe(1); }); }); diff --git a/gitnexus/test/integration/php-import-index-reuse.test.ts b/gitnexus/test/integration/php-import-index-reuse.test.ts new file mode 100644 index 000000000..3f0bf0102 --- /dev/null +++ b/gitnexus/test/integration/php-import-index-reuse.test.ts @@ -0,0 +1,147 @@ +/** + * Production-path regression guard for the PHP import-resolution index (#2901). + * + * PHP was the last language resolving imports with a full workspace scan per + * import. Both adapters in `languages/php/import-target.ts` materialized + * `[...allFilePaths]` twice per import and handed `resolvePhpImportInternal` an + * `index` of `undefined`, dropping it onto `suffixResolve`'s linear `findIndex` + * — a pass over every file per path-part × per extension, 98 ms per import at + * 20k files. They now read the shared `getWorkspaceFileIndex` + * (`import-resolvers/workspace-file-index.ts`), memoized on the `allFilePaths` + * Set identity via a WeakMap, through a PHP-specific parity view that keeps the + * three index-fed fast paths answering exactly what the scans answered (see the + * `#2901` header in `import-target.ts` — passing the raw shared index straight + * through MOVES IMPORTS edges, and `test/unit/scope-resolution/ + * php-import-target-parity.test.ts` is the differential that proves this one + * does not). + * + * Resolution reaches that index through `phpScopeResolver.resolveImportTarget` + * — the orchestrator adapter — not by calling `resolvePhpImportTargetInternal` + * directly the way the unit parity test does. The adapter must therefore pass + * the Set THROUGH; a defensive copy (`new Set(allFilePaths)`) would hand a + * fresh WeakMap key per call and restore the per-import rebuild. Python hit + * exactly that (PR #1918 review P1), and the parity test cannot see it: it + * never crosses the adapter. + * + * The traversal-count assertions are the perf guard. They are paired with + * result assertions on purpose: a count of 1 is equally true of an adapter that + * has stopped resolving anything at all, so counting alone would stay green + * while every PHP IMPORTS edge disappeared. + * + * On the one traversal PHP still pays per import in a specific case — a PSR-4 + * namespace whose directory has no direct `.php` children — see the pinned + * residual arm at the bottom of the unit parity test. It lives in + * `import-resolvers/php.ts`, which #2901 does not touch, so the corpora here + * resolve through the legs that do reach the index. + */ +import { describe, it, expect } from 'vitest'; +import { phpScopeResolver } from '../../src/core/ingestion/languages/php/scope-resolver.js'; +import type { ComposerConfig } from '../../src/core/ingestion/language-config.js'; +import { CountingSet, expectDistinctFileSetsGetOwnIndex } from '../helpers/counting-file-set.js'; + +const { resolveImportTarget } = phpScopeResolver; + +const FROM_FILE = 'app/Main.php'; + +/** The `composer.json` PSR-4 map `loadPhpComposerConfig` would have produced. */ +const COMPOSER: ComposerConfig = { psr4: new Map([['App', 'app']]) }; + +/** + * A synthetic PSR-4 app: many service classes, plus the shapes the three + * index-fed legs answer — `app/Models/User.php` for the class-style whole-path + * hit, the populated `app/Models/` directory for the function-import fallback, + * and `lib/Legacy/Helper.php` for the suffix fallback that runs when no PSR-4 + * prefix matches. + */ +function buildWorkspace(fileCount: number): CountingSet { + const files: string[] = []; + for (let i = 0; i < fileCount; i++) { + files.push(`app/Services/Service${String(i).padStart(5, '0')}.php`); + } + files.push('app/Models/User.php'); + files.push('app/Models/functions.php'); + files.push('lib/Legacy/Helper.php'); + files.push('index.php'); + files.push(FROM_FILE); + return new CountingSet(files); +} + +describe('PHP import resolution — index reuse across use-statements (#2901)', () => { + it('builds the workspace index once for many imports over a stable file set', () => { + const files = buildWorkspace(300); + const resolved: (string | readonly string[] | null)[] = []; + + for (let i = 0; i < 200; i++) { + // A PSR-4 class hit, a function import that falls back to the namespace + // directory, and a third-party namespace that misses. The miss is the + // expensive case: it matches no PSR-4 prefix and so walks every suffix × + // every extension before returning null. + resolved.push(resolveImportTarget('App\\Models\\User', FROM_FILE, files, COMPOSER)); + resolved.push(resolveImportTarget('App\\Models\\getUser', FROM_FILE, files, COMPOSER)); + resolved.push(resolveImportTarget(`Psr\\Log\\Missing${i}`, FROM_FILE, files, COMPOSER)); + } + + expect(files.scans).toBe(1); + + // Paired result assertions — a count of 1 must not be the count of an + // adapter that resolves nothing. + expect(resolved[0]).toBe('app/Models/User.php'); + expect(resolved[1]).toBe('app/Models/User.php'); + expect(resolved[2]).toBeNull(); + }); + + it('builds the workspace index once with no composer.json at all', () => { + const files = buildWorkspace(300); + const resolved: (string | readonly string[] | null)[] = []; + + // `loadResolutionConfig` returns null when the repo has no composer.json, + // which skips the PSR-4 block entirely and leaves `suffixResolve` — the leg + // that used to cost a `findIndex` pass per extension — as the only path. + for (let i = 0; i < 200; i++) { + resolved.push(resolveImportTarget('Legacy\\Helper', FROM_FILE, files, null)); + resolved.push(resolveImportTarget(`Psr\\Log\\Missing${i}`, FROM_FILE, files, null)); + } + + expect(files.scans).toBe(1); + expect(resolved[0]).toBe('lib/Legacy/Helper.php'); + expect(resolved[1]).toBeNull(); + }); + + it('a distinct file set gets its own index (no stale cross-run reuse)', () => { + expectDistinctFileSetsGetOwnIndex({ + resolveImportTarget, + buildWorkspace: () => buildWorkspace(20), + targetRaw: 'App\\Models\\User', + fromFile: FROM_FILE, + resolutionConfig: COMPOSER, + expected: 'app/Models/User.php', + expectedScans: 1, + }); + }); + + it('still resolves real use-statements correctly (the perf test is not vacuous)', () => { + const files = buildWorkspace(5); + + // PSR-4 class-style: `App\Models\User` → `app/Models/User.php`. + expect(resolveImportTarget('App\\Models\\User', FROM_FILE, files, COMPOSER)).toBe( + 'app/Models/User.php', + ); + expect(resolveImportTarget('App\\Services\\Service00000', FROM_FILE, files, COMPOSER)).toBe( + 'app/Services/Service00000.php', + ); + + // Suffix fallback: no PSR-4 prefix matches `Legacy`, so `suffixResolve` + // answers from the longest matching proper path suffix. + expect(resolveImportTarget('Legacy\\Helper', FROM_FILE, files, COMPOSER)).toBe( + 'lib/Legacy/Helper.php', + ); + + // A root-level file is NOT reachable as a proper suffix — the pre-#2901 + // behaviour the parity view preserves, and the single most likely thing a + // raw `getWorkspaceFileIndex().index` hand-off would have changed. + expect(resolveImportTarget('index', FROM_FILE, files, COMPOSER)).toBeNull(); + + // Third-party namespaces have no file in the repo. + expect(resolveImportTarget('Psr\\Log\\LoggerInterface', FROM_FILE, files, COMPOSER)).toBeNull(); + }); +}); diff --git a/gitnexus/test/integration/python-import-index-reuse.test.ts b/gitnexus/test/integration/python-import-index-reuse.test.ts index 33e6aba9f..46b07afd9 100644 --- a/gitnexus/test/integration/python-import-index-reuse.test.ts +++ b/gitnexus/test/integration/python-import-index-reuse.test.ts @@ -1,7 +1,8 @@ /** * Production-path regression guard for PR #1918 review finding P1. * - * The Python file index (`getPythonFileIndex` in `import-target.ts`) is + * The Python file index (`getPythonFileIndex` in + * `import-resolvers/python-file-index.ts`) is * memoized on the `allFilePaths` Set identity via a WeakMap. The registry- * primary path reaches it through `pythonScopeResolver.resolveImportTarget` * (the orchestrator adapter) — NOT by calling `resolvePythonImportTarget` @@ -10,15 +11,38 @@ * WeakMap key per call so the index rebuilt every import (O(imports × files)). * * This test drives the adapter exactly as the orchestrator does and asserts the - * index is built ONCE across many imports on a stable set. It fails (build - * count == number of imports) if the per-import copy is reintroduced. + * file set is traversed ONCE across many imports on a stable set. It fails + * (one traversal per import) if the per-import copy is reintroduced. + * + * ## Why this counts TRAVERSALS and not index builds (#2909) + * + * This guard used to read a build counter that shipped in production purely so + * a test could read it (now deleted). `CountingSet` + * (`test/helpers/counting-file-set.ts`) replaces it, and the swap is not a + * wash: + * + * - STRICTLY MORE COVERAGE. A scan added BESIDE a reused index moves no build + * count — the cache still hits, the counter still reads 1 — but it does move + * the traversal count. That mutation is the one `bench/import-target/` + * provably cannot see either: `baselines.json` `_blind_spot` records a full + * workspace scan on 1-in-32 imports passing every timing arm. + * - LESS PRODUCTION SURFACE. ~30 lines shipped in the bundle whose only caller + * outside a cache miss was this file. + * - PARALLEL-SAFE. The counter lives on the instance the test built, so there + * is no module-global to `reset()` and no ordering hazard between tests. + * + * The traversal-count assertions are the perf guard. They are paired with result + * assertions on purpose: a count of 1 is equally true of an adapter that has + * stopped resolving anything at all, so counting alone would stay green while + * every Python IMPORTS edge disappeared. */ import { describe, it, expect } from 'vitest'; import { pythonScopeResolver } from '../../src/core/ingestion/languages/python/scope-resolver.js'; -import { - getPythonFileIndexBuildCount, - resetPythonFileIndexBuildCount, -} from '../../src/core/ingestion/languages/python/index-stats.js'; +import { CountingSet, expectDistinctFileSetsGetOwnIndex } from '../helpers/counting-file-set.js'; + +const { resolveImportTarget } = pythonScopeResolver; + +const FROM_FILE = 'app/main.py'; /** * A synthetic workspace: a real package (`realpkg/__init__.py`, so the @@ -26,62 +50,60 @@ import { * below are multi-segment and miss every fast path, so each call reaches both * `hasRepoCandidate` and `resolveAbsoluteFromFiles` — the two index consumers. */ -function buildWorkspace(fileCount: number): Set { - const files = new Set(); +function buildWorkspace(fileCount: number): CountingSet { + const files: string[] = []; for (let i = 0; i < fileCount; i++) { - files.add(`pkg/sub/mod${String(i).padStart(5, '0')}.py`); + files.push(`pkg/sub/mod${String(i).padStart(5, '0')}.py`); } - files.add('realpkg/__init__.py'); - files.add('realpkg/widget.py'); - return files; + files.push('realpkg/__init__.py'); + files.push('realpkg/widget.py'); + return new CountingSet(files); } describe('Python import resolution — index reuse across imports (PR #1918 P1)', () => { it('builds the file index once for many imports over a stable file set', () => { - const allFilePaths = buildWorkspace(300); - const fromFile = 'app/main.py'; - const importCount = 300; + const files = buildWorkspace(300); + const resolved: (string | readonly string[] | null)[] = []; - resetPythonFileIndexBuildCount(); - for (let i = 0; i < importCount; i++) { + for (let i = 0; i < 300; i++) { // Multi-segment, candidate-passing, suffix-miss → reaches the index. - pythonScopeResolver.resolveImportTarget(`realpkg.ghost${i}`, fromFile, allFilePaths); + resolved.push(resolveImportTarget(`realpkg.ghost${i}`, FROM_FILE, files, undefined)); } + resolved.push(resolveImportTarget('realpkg.widget', FROM_FILE, files, undefined)); // The whole point of PR #1918: O(imports + files), not O(imports × files). // Pre-fix this was 300 (one rebuild per import via the adapter's Set copy). - expect(getPythonFileIndexBuildCount()).toBe(1); + expect(files.scans).toBe(1); + + // Paired result assertions — a count of 1 must not be the count of an + // adapter that resolves nothing. + expect(resolved[0]).toBeNull(); + expect(resolved[300]).toBe('realpkg/widget.py'); }); - it('rebuilds once per distinct file set (per-run isolation, no stale reuse)', () => { - const fromFile = 'app/main.py'; - - resetPythonFileIndexBuildCount(); - const setA = buildWorkspace(50); - for (let i = 0; i < 20; i++) { - pythonScopeResolver.resolveImportTarget(`realpkg.ghost${i}`, fromFile, setA); - } - expect(getPythonFileIndexBuildCount()).toBe(1); - - // A different Set instance is a different logical workspace → one more build. - const setB = buildWorkspace(50); - for (let i = 0; i < 20; i++) { - pythonScopeResolver.resolveImportTarget(`realpkg.ghost${i}`, fromFile, setB); - } - expect(getPythonFileIndexBuildCount()).toBe(2); + it('a distinct file set gets its own index (per-run isolation, no stale reuse)', () => { + expectDistinctFileSetsGetOwnIndex({ + resolveImportTarget, + buildWorkspace: () => buildWorkspace(50), + targetRaw: 'realpkg.widget', + fromFile: FROM_FILE, + resolutionConfig: undefined, + expected: 'realpkg/widget.py', + expectedScans: 1, + }); }); it('still resolves real imports correctly (the perf test is not vacuous)', () => { - const allFilePaths = buildWorkspace(20); - const fromFile = 'app/main.py'; + const files = buildWorkspace(20); // Suffix-fallback hit through the adapter: realpkg.widget → realpkg/widget.py. - expect(pythonScopeResolver.resolveImportTarget('realpkg.widget', fromFile, allFilePaths)).toBe( + expect(resolveImportTarget('realpkg.widget', FROM_FILE, files, undefined)).toBe( 'realpkg/widget.py', ); // Gated-out / unresolvable import returns null. - expect( - pythonScopeResolver.resolveImportTarget('realpkg.ghost', fromFile, allFilePaths), - ).toBeNull(); + expect(resolveImportTarget('realpkg.ghost', FROM_FILE, files, undefined)).toBeNull(); + + // All of it off one traversal. + expect(files.scans).toBe(1); }); }); diff --git a/gitnexus/test/integration/typescript-import-index-reuse.test.ts b/gitnexus/test/integration/typescript-import-index-reuse.test.ts new file mode 100644 index 000000000..7e0a1157c --- /dev/null +++ b/gitnexus/test/integration/typescript-import-index-reuse.test.ts @@ -0,0 +1,184 @@ +/** + * Production-path regression guard for the TypeScript import-resolution pass + * cache (#2910). + * + * `makeTsResolveImportTarget` has carried a `SuffixIndex` since #1918, so the + * per-import rebuild this file's siblings were written for never applied here. + * What did apply is the OTHER failure mode of the memo it used: a single slot, + * invalidated by `cached.key !== allFilePaths`. One file set is memoized + * perfectly; two alternating file sets rebuild the arrays, the index and the + * `resolveCache` on every single call. Measured at 4000 files × 400 imports: + * 12.0 ms for one set, 1438.2 ms alternating between two — 120x, and the same + * O(imports × files) shape the per-file-set index hoists removed. + * + * That is why this adapter could not carry `expectDistinctFileSetsGetOwnIndex`, + * the one arm every other language's guard has: the arm alternates two sets by + * construction, and the single-slot cache posts 42 traversals against the 2 it + * posts now. The cache is a `WeakMap, PassCache>` keyed on + * the Set, like every other language's index, and the arm below is the proof. + * + * Whether the thrash was reachable in production: `pipeline/run.ts` builds one + * `allFilePaths` Set per provider pass and TypeScript, JavaScript and Vue are + * separate providers with separate caches, so within one analyze it was latent + * rather than live. It was one refactor — an interleaved or re-entrant pass, a + * second workspace, a caller resolving against a filtered file set — away from + * live, and the `WeakMap` is strictly simpler than the slot it replaces. + * + * Resolution goes through `typescriptScopeResolver.resolveImportTarget`, the + * orchestrator adapter, which must pass the Set THROUGH: a defensive + * `new Set(allFilePaths)` hands a fresh `WeakMap` key per import and restores + * the per-import rebuild (PR #1918 review P1). Two traversals per file set, not + * one: the adapter materializes `allFileList` and then keeps one mutable copy + * of the Set, because `TsResolveContext.allFilePaths` is a `Set`, not a + * `ReadonlySet`. + * + * The counts are paired with result assertions on purpose: a count of 2 is + * equally true of an adapter that has stopped resolving anything at all. + */ +import { describe, it, expect } from 'vitest'; +import { typescriptScopeResolver } from '../../src/core/ingestion/languages/typescript/scope-resolver.js'; +import type { TsconfigPaths } from '../../src/core/ingestion/language-config.js'; +import { CountingSet, expectDistinctFileSetsGetOwnIndex } from '../helpers/counting-file-set.js'; + +const { resolveImportTarget } = typescriptScopeResolver; + +const FROM_FILE = 'src/main.ts'; + +/** What `loadTsconfigPaths` produces for `"@/*": ["src/*"]` under `baseUrl: "."`. */ +const TSCONFIG_PATHS: TsconfigPaths = { aliases: new Map([['@/', 'src/']]), baseUrl: '.' }; + +/** The shape `loadResolutionConfig` returns for a non-Nuxt TypeScript repo. */ +const RESOLUTION_CONFIG = { tsconfigPaths: TSCONFIG_PATHS, nuxtAutoImports: null }; + +/** + * A synthetic TypeScript app covering the legs the resolver takes: a relative + * import answered by exact `Set.has`, an ESM `.js` specifier that must strip to + * `.ts`, a bare specifier answered by path suffix, a directory `index.ts`, and + * a `@/`-aliased path. + */ +function buildWorkspace(fileCount: number): CountingSet { + const files: string[] = []; + for (let i = 0; i < fileCount; i++) { + files.push(`src/services/Service${String(i).padStart(5, '0')}.ts`); + } + files.push('src/util.ts'); + files.push('src/models/index.ts'); + files.push('src/components/Widget.tsx'); + files.push('node_modules/dep/index.d.ts'); + files.push(FROM_FILE); + return new CountingSet(files); +} + +describe('TypeScript import resolution — index reuse across imports (#2910)', () => { + it('builds the pass cache once for many imports over a stable file set', () => { + const files = buildWorkspace(300); + const resolved: (string | readonly string[] | null)[] = []; + + for (let i = 0; i < 200; i++) { + // A relative hit, an ESM `.js` specifier stripped back to `.ts`, an + // aliased path, a bare-specifier suffix hit, and a bare specifier that + // misses — the expensive case, which runs every path part × every + // extension before returning null. + resolved.push(resolveImportTarget('./util', FROM_FILE, files, RESOLUTION_CONFIG)); + resolved.push(resolveImportTarget('./util.js', FROM_FILE, files, RESOLUTION_CONFIG)); + resolved.push(resolveImportTarget('@/models', FROM_FILE, files, RESOLUTION_CONFIG)); + resolved.push(resolveImportTarget('components/Widget', FROM_FILE, files, RESOLUTION_CONFIG)); + resolved.push( + resolveImportTarget(`@vendor/ghost${i}/deep`, FROM_FILE, files, RESOLUTION_CONFIG), + ); + } + + // Two: `Array.from(allFilePaths)` and the one mutable `Set` copy the + // resolver context requires. Both happen once per file set. + expect(files.scans).toBe(2); + + // Paired result assertions — a count of 2 must not be the count of an + // adapter that resolves nothing. + expect(resolved[0]).toBe('src/util.ts'); + expect(resolved[1]).toBe('src/util.ts'); + expect(resolved[2]).toBe('src/models/index.ts'); + expect(resolved[3]).toBe('src/components/Widget.tsx'); + expect(resolved[4]).toBeNull(); + }); + + /** + * The arm the single-slot cache could not pass. `expectDistinctFileSetsGetOwnIndex` + * alternates two file sets 20 times; the slot was invalidated on every one of + * those calls, so each set posted 42 traversals instead of 2. + */ + it('a distinct file set gets its own index (no stale cross-run reuse)', () => { + expectDistinctFileSetsGetOwnIndex({ + resolveImportTarget, + buildWorkspace: () => buildWorkspace(20), + targetRaw: '@/models', + fromFile: FROM_FILE, + resolutionConfig: RESOLUTION_CONFIG, + expected: 'src/models/index.ts', + expectedScans: 2, + }); + }); + + /** + * The per-file-set index and `resolveCache` must not become global. The arm + * above cannot see that: `expectDistinctFileSetsGetOwnIndex` builds two + * IDENTICAL corpora, so a stale answer carried across them is also the right + * answer, and only its traversal counts would notice. These two workspaces + * answer the same specifier differently, and they are resolved alternately. + */ + it('two different workspaces answer the same specifier differently', () => { + const a = new Set(['src/util.ts', FROM_FILE]); + const b = new Set(['vendor/util.ts', FROM_FILE]); + + expect(resolveImportTarget('util', FROM_FILE, a, RESOLUTION_CONFIG)).toBe('src/util.ts'); + expect(resolveImportTarget('util', FROM_FILE, b, RESOLUTION_CONFIG)).toBe('vendor/util.ts'); + expect(resolveImportTarget('util', FROM_FILE, a, RESOLUTION_CONFIG)).toBe('src/util.ts'); + expect(resolveImportTarget('util', FROM_FILE, b, RESOLUTION_CONFIG)).toBe('vendor/util.ts'); + }); + + it('still resolves real imports correctly (the perf test is not vacuous)', () => { + const files = buildWorkspace(5); + + // Relative, extensionless and with the ESM `.js` spelling. + expect(resolveImportTarget('./util', FROM_FILE, files, RESOLUTION_CONFIG)).toBe('src/util.ts'); + expect(resolveImportTarget('./util.js', FROM_FILE, files, RESOLUTION_CONFIG)).toBe( + 'src/util.ts', + ); + // Directory index. + expect(resolveImportTarget('./models', FROM_FILE, files, RESOLUTION_CONFIG)).toBe( + 'src/models/index.ts', + ); + // tsconfig alias. + expect( + resolveImportTarget('@/services/Service00000', FROM_FILE, files, RESOLUTION_CONFIG), + ).toBe('src/services/Service00000.ts'); + // Bare specifier resolved by path suffix. + expect(resolveImportTarget('components/Widget', FROM_FILE, files, RESOLUTION_CONFIG)).toBe( + 'src/components/Widget.tsx', + ); + + // Nothing in the repo answers these. + expect(resolveImportTarget('./nowhere', FROM_FILE, files, RESOLUTION_CONFIG)).toBeNull(); + expect( + resolveImportTarget('@vendor/ghost/deep', FROM_FILE, files, RESOLUTION_CONFIG), + ).toBeNull(); + }); + + /** + * No `tsconfig.json` at all, which is the config the orchestrator threads for + * a repo without one. It skips the alias branch entirely and leaves the + * suffix cascade as the only path to the index. + */ + it('builds the pass cache once with no tsconfig paths', () => { + const files = buildWorkspace(300); + const resolved: (string | readonly string[] | null)[] = []; + + for (let i = 0; i < 200; i++) { + resolved.push(resolveImportTarget('components/Widget', FROM_FILE, files, undefined)); + resolved.push(resolveImportTarget(`@vendor/ghost${i}/deep`, FROM_FILE, files, undefined)); + } + + expect(files.scans).toBe(2); + expect(resolved[0]).toBe('src/components/Widget.tsx'); + expect(resolved[1]).toBeNull(); + }); +}); diff --git a/gitnexus/test/integration/vue-import-index-reuse.test.ts b/gitnexus/test/integration/vue-import-index-reuse.test.ts new file mode 100644 index 000000000..8e5a69ac0 --- /dev/null +++ b/gitnexus/test/integration/vue-import-index-reuse.test.ts @@ -0,0 +1,168 @@ +/** + * Production-path regression guard for the Vue import-resolution pass cache + * (#2910). + * + * `makeVueResolveImportTarget` is the TypeScript adapter with the language + * pinned to TypeScript, and it inherited the same memo: a single slot, + * invalidated by `cached.key !== allFilePaths`. One file set is memoized + * perfectly; two alternating file sets rebuild the arrays, the suffix index and + * the `resolveCache` on every call. Measured on the identical TypeScript + * adapter at 4000 files × 400 imports: 12.0 ms for one set, 1438.2 ms + * alternating between two — 120x, and the same O(imports × files) shape the + * per-file-set index hoists removed. + * + * That is why this adapter could not carry `expectDistinctFileSetsGetOwnIndex`, + * the one arm every other language's guard has: the arm alternates two sets by + * construction, and the single-slot cache posts 42 traversals against the 2 it + * posts now. The cache is a `WeakMap, PassCache>` keyed on + * the Set, like every other language's index, and the arm below is the proof. + * + * Whether the thrash was reachable in production: `pipeline/run.ts` builds one + * `allFilePaths` Set per provider pass and Vue, TypeScript and JavaScript are + * separate providers with separate caches, so within one analyze it was latent + * rather than live — one refactor away from live, and the `WeakMap` is strictly + * simpler than the slot it replaces. + * + * Resolution goes through `vueScopeResolver.resolveImportTarget`, the + * orchestrator adapter, which must pass the Set THROUGH: a defensive + * `new Set(allFilePaths)` hands a fresh `WeakMap` key per import and restores + * the per-import rebuild (PR #1918 review P1). Two traversals per file set, not + * one: the adapter materializes `allFileList` and then keeps one mutable copy + * of the Set, because `TsResolveContext.allFilePaths` is a `Set`, not a + * `ReadonlySet`. + * + * The counts are paired with result assertions on purpose: a count of 2 is + * equally true of an adapter that has stopped resolving anything at all. + */ +import { describe, it, expect } from 'vitest'; +import { vueScopeResolver } from '../../src/core/ingestion/languages/vue/scope-resolver.js'; +import type { TsconfigPaths } from '../../src/core/ingestion/language-config.js'; +import { CountingSet, expectDistinctFileSetsGetOwnIndex } from '../helpers/counting-file-set.js'; + +const { resolveImportTarget } = vueScopeResolver; + +const FROM_FILE = 'src/App.vue'; + +/** What `loadTsconfigPaths` produces for `"@/*": ["src/*"]` under `baseUrl: "."`. */ +const TSCONFIG_PATHS: TsconfigPaths = { aliases: new Map([['@/', 'src/']]), baseUrl: '.' }; + +/** The shape `loadResolutionConfig` returns for a Vue repo with a tsconfig. */ +const RESOLUTION_CONFIG = { tsconfigPaths: TSCONFIG_PATHS }; + +/** + * A synthetic Vue SFC project: many components, a `.ts` composable reached by + * bare specifier, a `@/`-aliased path, and a directory `index.ts`. `.vue` + * imports carry their extension, so they land on the exact-path branch. + */ +function buildWorkspace(fileCount: number): CountingSet { + const files: string[] = []; + for (let i = 0; i < fileCount; i++) { + files.push(`src/components/Widget${String(i).padStart(5, '0')}.vue`); + } + files.push('src/composables/useUser.ts'); + files.push('src/stores/index.ts'); + files.push('src/util.ts'); + files.push(FROM_FILE); + return new CountingSet(files); +} + +describe('Vue import resolution — index reuse across imports (#2910)', () => { + it('builds the pass cache once for many imports over a stable file set', () => { + const files = buildWorkspace(300); + const resolved: (string | readonly string[] | null)[] = []; + + for (let i = 0; i < 200; i++) { + // A relative `.vue` hit, a relative `.ts` composable, an aliased path, a + // bare-specifier suffix hit, and a bare specifier that misses — the + // expensive case, which runs every path part × every extension before + // returning null. + resolved.push( + resolveImportTarget('./components/Widget00000.vue', FROM_FILE, files, RESOLUTION_CONFIG), + ); + resolved.push( + resolveImportTarget('./composables/useUser', FROM_FILE, files, RESOLUTION_CONFIG), + ); + resolved.push(resolveImportTarget('@/stores', FROM_FILE, files, RESOLUTION_CONFIG)); + resolved.push( + resolveImportTarget('composables/useUser', FROM_FILE, files, RESOLUTION_CONFIG), + ); + resolved.push( + resolveImportTarget(`@vendor/ghost${i}/deep`, FROM_FILE, files, RESOLUTION_CONFIG), + ); + } + + // Two: `Array.from(allFilePaths)` and the one mutable `Set` copy the + // resolver context requires. Both happen once per file set. + expect(files.scans).toBe(2); + + // Paired result assertions — a count of 2 must not be the count of an + // adapter that resolves nothing. + expect(resolved[0]).toBe('src/components/Widget00000.vue'); + expect(resolved[1]).toBe('src/composables/useUser.ts'); + expect(resolved[2]).toBe('src/stores/index.ts'); + expect(resolved[3]).toBe('src/composables/useUser.ts'); + expect(resolved[4]).toBeNull(); + }); + + /** + * The arm the single-slot cache could not pass. `expectDistinctFileSetsGetOwnIndex` + * alternates two file sets 20 times; the slot was invalidated on every one of + * those calls, so each set posted 42 traversals instead of 2. + */ + it('a distinct file set gets its own index (no stale cross-run reuse)', () => { + expectDistinctFileSetsGetOwnIndex({ + resolveImportTarget, + buildWorkspace: () => buildWorkspace(20), + targetRaw: '@/stores', + fromFile: FROM_FILE, + resolutionConfig: RESOLUTION_CONFIG, + expected: 'src/stores/index.ts', + expectedScans: 2, + }); + }); + + /** + * The per-file-set index and `resolveCache` must not become global. The arm + * above cannot see that: `expectDistinctFileSetsGetOwnIndex` builds two + * IDENTICAL corpora, so a stale answer carried across them is also the right + * answer, and only its traversal counts would notice. These two workspaces + * answer the same specifier differently, and they are resolved alternately. + */ + it('two different workspaces answer the same specifier differently', () => { + const a = new Set(['src/util.ts', FROM_FILE]); + const b = new Set(['vendor/util.ts', FROM_FILE]); + + expect(resolveImportTarget('util', FROM_FILE, a, RESOLUTION_CONFIG)).toBe('src/util.ts'); + expect(resolveImportTarget('util', FROM_FILE, b, RESOLUTION_CONFIG)).toBe('vendor/util.ts'); + expect(resolveImportTarget('util', FROM_FILE, a, RESOLUTION_CONFIG)).toBe('src/util.ts'); + expect(resolveImportTarget('util', FROM_FILE, b, RESOLUTION_CONFIG)).toBe('vendor/util.ts'); + }); + + it('still resolves real SFC imports correctly (the perf test is not vacuous)', () => { + const files = buildWorkspace(5); + + // `.vue` imports are written with their extension and hit the exact-path + // branch. + expect( + resolveImportTarget('./components/Widget00000.vue', FROM_FILE, files, RESOLUTION_CONFIG), + ).toBe('src/components/Widget00000.vue'); + // A composable, relative and extensionless. + expect(resolveImportTarget('./composables/useUser', FROM_FILE, files, RESOLUTION_CONFIG)).toBe( + 'src/composables/useUser.ts', + ); + // Directory index behind a tsconfig alias. + expect(resolveImportTarget('@/stores', FROM_FILE, files, RESOLUTION_CONFIG)).toBe( + 'src/stores/index.ts', + ); + // Bare specifier resolved by path suffix. + expect( + resolveImportTarget('components/Widget00000.vue', FROM_FILE, files, RESOLUTION_CONFIG), + ).toBe('src/components/Widget00000.vue'); + + // Nothing in the repo answers these. + expect(resolveImportTarget('./Missing.vue', FROM_FILE, files, RESOLUTION_CONFIG)).toBeNull(); + expect( + resolveImportTarget('@vendor/ghost/deep', FROM_FILE, files, RESOLUTION_CONFIG), + ).toBeNull(); + }); +}); diff --git a/gitnexus/test/unit/import-resolvers/csharp-csproj-parity.test.ts b/gitnexus/test/unit/import-resolvers/csharp-csproj-parity.test.ts new file mode 100644 index 000000000..3bcc17ef1 --- /dev/null +++ b/gitnexus/test/unit/import-resolvers/csharp-csproj-parity.test.ts @@ -0,0 +1,620 @@ +/** + * Differential harness for the C# **csproj** leg of `resolveCSharpImportInternal` + * (#2902). + * + * #2878 moved C#'s no-csproj leg onto memoized indexes. The csproj leg kept a + * per-import, per-matching-config Θ(files) scan — step 3, "linear scan fallback + * for directory matching" — measured at ~1.08 ms per import over 50 000 `.cs` + * files. This PR answers that leg from a per-file-list directory index instead. + * + * WHY A VERBATIM COPY AND NOT A DELETION. The obvious cleanup is "step 2 already + * asks the index the same question, so skip step 3 whenever an index exists". + * That is wrong, and this file is the proof. Step 2 filters + * `index.getFilesInDir(dirPrefix, '.cs')`, whose buckets are keyed on + * SEGMENT-aligned directory suffixes; step 3 runs an UNANCHORED + * `normalized.indexOf(dirPrefix + '/')`. Step 3 therefore answers strictly more: + * + * - `dirPrefix = 'ubModels'` matches `src/SubModels/` (character suffix of a + * segment, not a segment); + * - `dirPrefix = 'rc/Models'` matches BOTH `src/Models/` and + * `vendor/mysrc/Models/`; + * - `dirPrefix = ''` — the "the import IS the root namespace and the config + * has no projectDir" case — matches every `.cs` file exactly one directory + * deep. `buildSuffixIndex` emits an empty directory suffix only for a path + * that BEGINS with '/', so over repo-relative paths `getFilesInDir('', + * '.cs')` is always empty and step 3 is the only implementation that case + * has ever had. (The leading-slash shape has its own arm below, kept off + * the main corpus precisely because step 2 DOES answer it.) + * + * Step 3 also runs only when step 2 found nothing, so those extra hits are + * observable rather than shadowed. `skips step 3 when the index is present` + * below pins that: it drives the naive cleanup and asserts it CHANGES answers. + * + * The arms all assert the full `string[]` and its order — this resolver returns + * every match, not one, and `configs/csharp.ts` turns a multi-file result into a + * `kind: 'package'` edge set whose order reaches the graph. + */ +import { describe, expect, it } from 'vitest'; + +import { + resolveCSharpImportInternal, + resolveCSharpNamespaceDir, +} from '../../../src/core/ingestion/import-resolvers/csharp.js'; +import { + buildSuffixIndex, + suffixResolve, + type SuffixIndex, +} from '../../../src/core/ingestion/import-resolvers/utils.js'; +import { csharpSuffixFallbackAllowed } from '../../../src/core/ingestion/csharp-namespace-gate.js'; +import { CountingSet } from '../../helpers/counting-file-set.js'; +import type { + CSharpProjectConfig, + CSharpNamespaceEvidence, +} from '../../../src/core/ingestion/language-config.js'; + +// ─── verbatim pre-change implementation ────────────────────────────────────── +// `git show HEAD~:gitnexus/src/core/ingestion/import-resolvers/csharp.ts`, body +// copied unchanged. Its helpers (`suffixResolve`, `csharpSuffixFallbackAllowed`, +// `SuffixIndex`) are imported from production because this PR does not touch +// them; only the function below changed. + +function legacyResolveCSharpImportInternal( + importPath: string, + csharpConfigs: CSharpProjectConfig[], + normalizedFileList: string[], + allFileList: string[], + index?: SuffixIndex, + evidence?: CSharpNamespaceEvidence, +): string[] { + const namespacePath = importPath.replace(/\./g, '/'); + const results: string[] = []; + + for (const config of csharpConfigs) { + const nsPath = config.rootNamespace.replace(/\./g, '/'); + let relative: string; + if (namespacePath.startsWith(nsPath + '/')) { + relative = namespacePath.slice(nsPath.length + 1); + } else if (namespacePath === nsPath) { + // The import IS the root namespace — resolve to all .cs files in project root + relative = ''; + } else { + continue; + } + + const dirPrefix = config.projectDir + ? relative + ? config.projectDir + '/' + relative + : config.projectDir + : relative; + + // 1. Try as single file: relative.cs (e.g., "Models/DlqMessage.cs") + if (relative) { + const candidate = dirPrefix + '.cs'; + if (index) { + const result = index.get(candidate) || index.getInsensitive(candidate); + if (result) return [result]; + } + // Also try suffix match + const suffixResult = index?.get(relative + '.cs') || index?.getInsensitive(relative + '.cs'); + if (suffixResult) return [suffixResult]; + } + + // 2. Try as directory: all .cs files directly inside (namespace import) + if (index) { + const dirFiles = index.getFilesInDir(dirPrefix, '.cs'); + for (const f of dirFiles) { + const normalized = f.replace(/\\/g, '/'); + // Check it's a direct child by finding the dirPrefix and ensuring no deeper slashes + const prefixIdx = normalized.indexOf(dirPrefix + '/'); + if (prefixIdx < 0) continue; + const afterDir = normalized.substring(prefixIdx + dirPrefix.length + 1); + if (!afterDir.includes('/')) { + results.push(f); + } + } + if (results.length > 0) return results; + } + + // 3. Linear scan fallback for directory matching + if (results.length === 0) { + const dirTrail = dirPrefix + '/'; + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + if (!normalized.endsWith('.cs')) continue; + const prefixIdx = normalized.indexOf(dirTrail); + if (prefixIdx < 0) continue; + const afterDir = normalized.substring(prefixIdx + dirTrail.length); + if (!afterDir.includes('/')) { + results.push(allFileList[i]); + } + } + if (results.length > 0) return results; + } + } + + // Fallback: suffix matching without namespace stripping (single file). + // Gated on in-repo declared-namespace evidence (#1881). + if (!csharpSuffixFallbackAllowed(importPath, evidence)) { + return []; + } + const pathParts = namespacePath.split('/').filter(Boolean); + const fallback = suffixResolve(pathParts, normalizedFileList, allFileList, index); + return fallback ? [fallback] : []; +} + +/** + * The naive cleanup this PR deliberately did NOT do: keep step 3 only as an + * un-indexed fallback. Same body as the legacy copy with step 3 gated on + * `index === undefined`. Driven by one arm below, which asserts it diverges. + */ +function skipStep3WhenIndexed( + importPath: string, + csharpConfigs: CSharpProjectConfig[], + normalizedFileList: string[], + allFileList: string[], + index?: SuffixIndex, + evidence?: CSharpNamespaceEvidence, +): string[] { + const namespacePath = importPath.replace(/\./g, '/'); + const results: string[] = []; + + for (const config of csharpConfigs) { + const nsPath = config.rootNamespace.replace(/\./g, '/'); + let relative: string; + if (namespacePath.startsWith(nsPath + '/')) { + relative = namespacePath.slice(nsPath.length + 1); + } else if (namespacePath === nsPath) { + relative = ''; + } else { + continue; + } + + const dirPrefix = config.projectDir + ? relative + ? config.projectDir + '/' + relative + : config.projectDir + : relative; + + if (relative) { + const candidate = dirPrefix + '.cs'; + if (index) { + const result = index.get(candidate) || index.getInsensitive(candidate); + if (result) return [result]; + } + const suffixResult = index?.get(relative + '.cs') || index?.getInsensitive(relative + '.cs'); + if (suffixResult) return [suffixResult]; + } + + if (index) { + const dirFiles = index.getFilesInDir(dirPrefix, '.cs'); + for (const f of dirFiles) { + const normalized = f.replace(/\\/g, '/'); + const prefixIdx = normalized.indexOf(dirPrefix + '/'); + if (prefixIdx < 0) continue; + const afterDir = normalized.substring(prefixIdx + dirPrefix.length + 1); + if (!afterDir.includes('/')) { + results.push(f); + } + } + if (results.length > 0) return results; + continue; + } + + const dirTrail = dirPrefix + '/'; + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + if (!normalized.endsWith('.cs')) continue; + const prefixIdx = normalized.indexOf(dirTrail); + if (prefixIdx < 0) continue; + const afterDir = normalized.substring(prefixIdx + dirTrail.length); + if (!afterDir.includes('/')) { + results.push(allFileList[i]); + } + } + if (results.length > 0) return results; + } + + if (!csharpSuffixFallbackAllowed(importPath, evidence)) { + return []; + } + const pathParts = namespacePath.split('/').filter(Boolean); + const fallback = suffixResolve(pathParts, normalizedFileList, allFileList, index); + return fallback ? [fallback] : []; +} + +// ─── corpus ────────────────────────────────────────────────────────────────── + +/** + * Hand-built so every tie-break the scan expressed through `indexOf` positions + * and file-list order has a witness. Order matters: the resolver emits in + * file-list order, so the interleavings below (`src/Models/Late.cs` after + * `other/Models/Thing.cs`, `src/Extra.cs` after `Models/TopLevel.cs`) are what + * make a directory-at-a-time emit distinguishable from a merged one. + */ +const RAW_FILES: readonly string[] = [ + // Repo-root files: no directory at all, so no `dirPrefix + '/'` can ever hit. + 'Root.cs', + 'notes.txt', + // The `src` project. + 'src/Program.cs', + 'src/Startup.cs', + 'src/Models/User.cs', + 'src/Models/Order.cs', + 'src/Models/Deep/Nested.cs', + // Character suffix of a segment, NOT a segment: answers `ubModels`, and + // answers `Models` only when no segment-aligned `Models` directory does. + 'src/SubModels/Widget.cs', + 'src/Services/UserService.cs', + 'src/Services/Sub/Inner.cs', + // A second directory sharing the `Models` last segment, minted BEFORE + // `src/Models/Late.cs` so multi-directory answers have to interleave. + 'other/Models/Thing.cs', + // Character suffix across a segment boundary: answers `rc/Models`. + 'vendor/mysrc/Models/Vendored.cs', + 'src/Models/Late.cs', + // `Models` nested inside `Models`: the FIRST `indexOf` occurrence is the + // outer one, whose remainder still holds a slash, so this answers nothing. + 'nest/Models/inner/Models/Ignored.cs', + // Single-segment directory, so it answers the empty `dirPrefix`. + 'Models/TopLevel.cs', + // Backslash separators: `allFileList` keeps them, the predicate runs on the + // normalized form, and the emitted value is the RAW one. + 'win\\Models\\Win.cs', + 'win\\Deep\\Models\\Deeper\\Skip.cs', + // Second project root, plus a case-only twin for the case-insensitive leg. + 'lib/Core/Widgets/Widget.cs', + 'lib/Core/Widgets.cs', + 'lib/Core/widgets/Lower.cs', + // Second single-segment-directory file, after `Models/TopLevel.cs`. + 'src/Extra.cs', + // Non-`.cs` files INSIDE directories that answer queries, so dropping the + // extension filter is visible rather than shadowed by the root-level + // `notes.txt` (which no `dirPrefix + '/'` can reach anyway). + 'src/notes.md', + 'Models/schema.json', +]; + +const ALL_FILE_LIST: string[] = [...RAW_FILES]; +const NORMALIZED_FILE_LIST: string[] = ALL_FILE_LIST.map((f) => f.replace(/\\/g, '/')); +const SUFFIX_INDEX: SuffixIndex = buildSuffixIndex(NORMALIZED_FILE_LIST, ALL_FILE_LIST); +// One Set per corpus, built once: `resolveCSharpImportInternal` now derives its +// normalized/raw lists from `getWorkspaceFileIndex(allFilePaths)`, whose memo is +// keyed on this object's identity. `RAW_FILES` is duplicate-free, so the derived +// pair is `NORMALIZED_FILE_LIST`/`ALL_FILE_LIST` element for element — which is +// what keeps the differential below a like-for-like comparison against the +// frozen legacy implementation, which still takes the two arrays. +const ALL_FILE_PATHS: ReadonlySet = new Set(ALL_FILE_LIST); + +const CONFIG_SHAPES: ReadonlyArray = [ + ['no configs at all', []], + ['projectDir=src', [{ rootNamespace: 'App', projectDir: 'src' }]], + // `projectDir` is a required `string`, so "without projectDir" is the empty + // string the `config.projectDir ? …` ternary treats as absent. + ['no projectDir', [{ rootNamespace: 'App', projectDir: '' }]], + ['dotted root namespace', [{ rootNamespace: 'Acme.App', projectDir: 'src' }]], + ['nested projectDir', [{ rootNamespace: 'Lib', projectDir: 'lib/Core' }]], + // Unanchored projectDirs: neither is a directory in the corpus, so both fall + // through to step 3 and match by character suffix. + ['unanchored projectDir', [{ rootNamespace: 'App', projectDir: 'rc' }]], + // A projectDir that already starts with '/' makes `dirPrefix` one character + // LONGER than a directory it shares a last segment with, the one shape where + // `indexOf` and `haystack.length - needle.length` both come out -1. + ['absolute projectDir', [{ rootNamespace: 'App', projectDir: '/Models' }]], + [ + 'two configs, both match', + [ + { rootNamespace: 'App', projectDir: 'nope' }, + { rootNamespace: 'App', projectDir: 'src' }, + ], + ], + [ + 'two configs, second root namespace extends the first', + [ + { rootNamespace: 'App', projectDir: 'src' }, + { rootNamespace: 'App.Models', projectDir: 'other' }, + ], + ], + [ + 'two configs, the matching one is second and has no projectDir', + [ + { rootNamespace: 'Zzz', projectDir: 'src' }, + { rootNamespace: 'App', projectDir: '' }, + ], + ], + ['no config matches', [{ rootNamespace: 'Zzz', projectDir: 'src' }]], +]; + +const IMPORTS: readonly string[] = [ + // Root-namespace-equals-import, against every projectDir shape. + 'App', + 'Acme.App', + 'Lib', + // Directories that exist, segment-aligned. + 'App.Models', + 'App.Services', + 'App.Services.Sub', + 'App.Models.Deep', + 'App.SubModels', + 'Acme.App.Models', + 'Lib.Widgets', + // Case-only variants (step 1's `getInsensitive` legs). + 'Lib.widgets', + 'App.models', + // Character-suffix-only directories: segment-aligned lookups find nothing. + 'App.ubModels', + 'App.odels', + 'App.Models.Late', + // A namespace with no matching directory anywhere — the issue's trigger. + 'App.Missing', + 'App.Missing.Deeper', + 'Acme.App.Missing', + // Single files rather than directories. + 'App.Program', + 'App.Root', + 'Lib.Core.Widgets', + // Imports that match no configured root namespace at all (BCL usings). + 'System', + 'System.Threading.Tasks', + 'Models', + 'Models.TopLevel', +]; + +/** Every (config shape, import) pair, plus both index modes. */ +const PAIRS: ReadonlyArray<{ + readonly key: string; + readonly configs: CSharpProjectConfig[]; + readonly importPath: string; + readonly index: SuffixIndex | undefined; +}> = CONFIG_SHAPES.flatMap(([shape, configs]) => + IMPORTS.flatMap((importPath) => + [SUFFIX_INDEX, undefined].map((index) => ({ + key: `${shape} | ${importPath} | index=${index === undefined ? 'absent' : 'present'}`, + configs, + importPath, + index, + })), + ), +); + +function runCurrent(pair: (typeof PAIRS)[number]): string[] { + return resolveCSharpImportInternal(pair.importPath, pair.configs, ALL_FILE_PATHS, pair.index); +} + +function runLegacy(pair: (typeof PAIRS)[number]): string[] { + return legacyResolveCSharpImportInternal( + pair.importPath, + pair.configs, + NORMALIZED_FILE_LIST, + ALL_FILE_LIST, + pair.index, + ); +} + +/** `label -> joined result`, so a mismatch prints the pair AND both answers. */ +function table(run: (pair: (typeof PAIRS)[number]) => string[]): Record { + const out: Record = {}; + for (const pair of PAIRS) out[pair.key] = run(pair).join(' , '); + return out; +} + +describe('C# csproj leg — directory index vs the pre-change linear scan (#2902)', () => { + it('returns byte-identical results, in order, for every config shape and import', () => { + expect(table(runCurrent)).toEqual(table(runLegacy)); + }); + + it('agrees on the `#1881` evidence gate too (the suffix fallback is downstream of step 3)', () => { + const evidence: CSharpNamespaceEvidence = { + declaredNamespaces: new Set(['App', 'App.Models', 'Lib.Core']), + rootNamespaces: new Set(['App', 'Lib']), + truncated: false, + }; + const current: Record = {}; + const legacy: Record = {}; + for (const pair of PAIRS) { + current[pair.key] = resolveCSharpImportInternal( + pair.importPath, + pair.configs, + ALL_FILE_PATHS, + pair.index, + evidence, + ).join(' , '); + legacy[pair.key] = legacyResolveCSharpImportInternal( + pair.importPath, + pair.configs, + NORMALIZED_FILE_LIST, + ALL_FILE_LIST, + pair.index, + evidence, + ).join(' , '); + } + expect(current).toEqual(legacy); + }); + + it('leaves `resolveCSharpNamespaceDir` — the sibling that shares the dirPrefix maths — alone', () => { + const dirs: Record = {}; + for (const [shape, configs] of CONFIG_SHAPES) { + for (const importPath of IMPORTS) { + dirs[`${shape} | ${importPath}`] = resolveCSharpNamespaceDir(importPath, configs); + } + } + expect(dirs['projectDir=src | App.Models']).toBe('/src/Models/'); + expect(dirs['no projectDir | App']).toBeNull(); + expect(dirs['no projectDir | App.Models']).toBe('/Models/'); + expect(dirs['no config matches | App.Models']).toBeNull(); + }); +}); + +describe('C# csproj leg — the answers only step 3 can give (#2902)', () => { + const withIndex = (configs: CSharpProjectConfig[], importPath: string): string[] => + resolveCSharpImportInternal(importPath, configs, ALL_FILE_PATHS, SUFFIX_INDEX); + + it('`relative === ""` with no projectDir gives dirPrefix "" — every .cs one level deep, merged', () => { + // `getFilesInDir('', '.cs')` is empty for every file set, so step 2 cannot + // answer this at all. Two directories match (`src`, `Models`) and their + // files interleave, so a directory-at-a-time emit reorders this. + expect(withIndex([{ rootNamespace: 'App', projectDir: '' }], 'App')).toEqual([ + 'src/Program.cs', + 'src/Startup.cs', + 'Models/TopLevel.cs', + 'src/Extra.cs', + ]); + }); + + it('`relative === ""` WITH a projectDir gives dirPrefix = projectDir, answered by step 2', () => { + expect(withIndex([{ rootNamespace: 'App', projectDir: 'src' }], 'App')).toEqual([ + 'src/Program.cs', + 'src/Startup.cs', + 'src/Extra.cs', + ]); + }); + + it('matches a directory by CHARACTER suffix of a segment, which no segment bucket holds', () => { + expect(withIndex([{ rootNamespace: 'App', projectDir: '' }], 'App.ubModels')).toEqual([ + 'src/SubModels/Widget.cs', + ]); + }); + + it('matches a character suffix ACROSS a segment boundary, over several directories', () => { + expect(withIndex([{ rootNamespace: 'App', projectDir: 'rc' }], 'App.Models')).toEqual([ + 'src/Models/User.cs', + 'src/Models/Order.cs', + 'vendor/mysrc/Models/Vendored.cs', + 'src/Models/Late.cs', + ]); + }); + + it('keeps the FIRST-occurrence tie-break: a directory nested inside a same-named one loses', () => { + // `nest/Models/inner/Models/Ignored.cs` is absent: `indexOf('odels/')` finds + // the outer `Models/`, and `inner/Models/Ignored.cs` still has a slash. + expect(withIndex([{ rootNamespace: 'App', projectDir: '' }], 'App.odels')).toEqual([ + 'src/Models/User.cs', + 'src/Models/Order.cs', + 'src/SubModels/Widget.cs', + 'other/Models/Thing.cs', + 'vendor/mysrc/Models/Vendored.cs', + 'src/Models/Late.cs', + 'Models/TopLevel.cs', + 'win\\Models\\Win.cs', + ]); + }); + + it('emits the RAW path for backslash-separated files while matching on the normalized one', () => { + expect(withIndex([{ rootNamespace: 'App', projectDir: 'win' }], 'App.Models')).toEqual([ + 'win\\Models\\Win.cs', + ]); + }); + + it('a leading-slash dirPrefix cannot bogus-match a shorter directory', () => { + // `dirPrefix = '/Models'` (projectDir used verbatim, since the import IS + // the root namespace): `'Models/'` is SHORTER than `'/Models/'`, and both + // `indexOf` and `haystack.length - needle.length` come out -1 without a + // length guard, so `Models/TopLevel.cs` would join the answer. + expect(withIndex([{ rootNamespace: 'App', projectDir: '/Models' }], 'App')).toEqual([ + 'src/Models/User.cs', + 'src/Models/Order.cs', + 'other/Models/Thing.cs', + 'vendor/mysrc/Models/Vendored.cs', + 'src/Models/Late.cs', + 'win\\Models\\Win.cs', + ]); + }); + + it('an import with no matching directory resolves to nothing (the issue trigger)', () => { + expect(withIndex([{ rootNamespace: 'App', projectDir: 'src' }], 'App.Missing')).toEqual([]); + }); + + it('skipping step 3 when the index is present CHANGES answers — the fallback is load-bearing', () => { + const divergent = PAIRS.filter( + (pair) => + pair.index !== undefined && + runCurrent(pair).join(' , ') !== + skipStep3WhenIndexed( + pair.importPath, + pair.configs, + NORMALIZED_FILE_LIST, + ALL_FILE_LIST, + pair.index, + ).join(' , '), + ).map((pair) => pair.key); + expect(divergent).toContain('no projectDir | App | index=present'); + expect(divergent).toContain('no projectDir | App.ubModels | index=present'); + expect(divergent).toContain('unanchored projectDir | App.Models | index=present'); + expect(divergent.length).toBeGreaterThan(10); + }); + + it('the parity arms are not vacuous: most pairs resolve, and step 3 answers many of them', () => { + const nonEmpty = PAIRS.filter((pair) => runCurrent(pair).length > 0); + const multiFile = PAIRS.filter((pair) => runCurrent(pair).length > 1); + expect(nonEmpty.length).toBeGreaterThan(PAIRS.length / 3); + expect(multiFile.length).toBeGreaterThan(20); + }); +}); + +describe('C# csproj leg — the directory index is built once per file set (#2902)', () => { + it('resolves many imports with a single pass over the file set', () => { + // `CountingSet`, not a counting ARRAY. This arm used to proxy + // `normalizedFileList` and count reads of `[0]`, because the index was keyed + // on that array; #2911 rekeyed it onto the Set, so the file set is now the + // only thing a rebuild has to re-traverse and the one instrument every other + // import-index guard already uses covers this leg too. + const files = new CountingSet(ALL_FILE_LIST); + const index = buildSuffixIndex([...NORMALIZED_FILE_LIST], ALL_FILE_LIST); + const configs: CSharpProjectConfig[] = [{ rootNamespace: 'App', projectDir: 'src' }]; + for (let i = 0; i < 40; i++) { + // Every one of these misses steps 1 and 2 and reaches step 3, so the + // directory index is genuinely consulted 40 times. + resolveCSharpImportInternal(`App.Missing${i % 4}`, configs, files, index); + } + expect(files.scans).toBe(1); + }); + + it('a path that BEGINS with a slash keeps parity (its directory is the empty string)', () => { + // Kept off the main corpus on purpose: `buildSuffixIndex` DOES emit an + // empty directory suffix for such a path, so step 2 would answer the empty + // `dirPrefix` here and short-circuit the very leg the arms above pin. + const raw = ['/Rooted.cs', 'src/Nested.cs', '/Other.cs']; + const normalized = raw.map((f) => f.replace(/\\/g, '/')); + // Duplicate-free, so the resolver derives exactly `normalized`/`raw` from + // it and the two sides of the differential still see the same corpus. + const rootedPaths: ReadonlySet = new Set(raw); + const index = buildSuffixIndex([...normalized], [...raw]); + const shapes: CSharpProjectConfig[][] = [ + [{ rootNamespace: 'App', projectDir: '' }], + [{ rootNamespace: 'App', projectDir: 'src' }], + [{ rootNamespace: 'App', projectDir: '/' }], + ]; + const current: string[][] = []; + const legacy: string[][] = []; + for (const configs of shapes) { + for (const importPath of ['App', 'App.Nested', 'App.Rooted']) { + for (const withIndex of [index, undefined]) { + current.push(resolveCSharpImportInternal(importPath, configs, rootedPaths, withIndex)); + legacy.push( + legacyResolveCSharpImportInternal(importPath, configs, [...normalized], raw, withIndex), + ); + } + } + } + expect(current).toEqual(legacy); + // Not vacuous: the un-indexed empty-dirPrefix query reaches `dir === ''`. + expect( + resolveCSharpImportInternal( + 'App', + [{ rootNamespace: 'App', projectDir: '' }], + rootedPaths, + undefined, + ), + ).toEqual(['/Rooted.cs', 'src/Nested.cs', '/Other.cs']); + }); + + it('a distinct file set gets its own index (no stale cross-run reuse)', () => { + const other: ReadonlySet = new Set(['App2/Models/Only.cs']); + const configs: CSharpProjectConfig[] = [{ rootNamespace: 'App', projectDir: 'App2' }]; + expect(resolveCSharpImportInternal('App.Models', configs, other, undefined)).toEqual([ + 'App2/Models/Only.cs', + ]); + expect(resolveCSharpImportInternal('App.Models', configs, ALL_FILE_PATHS, undefined)).toEqual( + [], + ); + }); +}); diff --git a/gitnexus/test/unit/import-resolvers/python-importer-prefixes.test.ts b/gitnexus/test/unit/import-resolvers/python-importer-prefixes.test.ts new file mode 100644 index 000000000..c6e2fcc22 --- /dev/null +++ b/gitnexus/test/unit/import-resolvers/python-importer-prefixes.test.ts @@ -0,0 +1,426 @@ +/** + * Gate for the single-segment bare-import ancestor walk in + * `import-resolvers/python.ts`: Python bare-import resolution must not scale + * with the importer's path depth. + * + * #2913 memoized the ancestor chains inside `languages/python/import-target.ts` + * and left this one behind, because its chain is a DIFFERENT SEQUENCE (self + * excluded, workspace root included, empty components kept) and no bench arm + * can reach it — `bench/import-target/measure.mjs` spells every Python import + * with a dot, and this walk runs only for a spelling with none. So it kept + * rebuilding `dirParts.slice(0, i).join('/')` per path component per import, + * and for `from x import y` / `import x as y` it did so TWICE per import: + * `resolvePythonImportTarget` probes the package with + * `targetIncludesImportedName` first and, when that misses, falls through to + * the identical call. Measured on a 400-file corpus, 3200 named single-segment + * imports: 24 `allFilePaths.has` probes per import at four directory + * components, the second twelve byte-identical to the first. + * + * ## Why this is a count and not a timing budget + * + * `test/helpers/counting-file-set.ts` is the house instrument for import-target + * reuse guards and it cannot see this defect, for the reason the #2913 gate + * states: the chain is derived from the `fromFile` STRING, and a rebuilt prefix + * traverses the file set zero extra times and issues the same `has` probes with + * the same arguments in the same order. A hoist is invisible to any instrument + * watching the resolver's inputs. So this file watches the memo, which is the + * one place the hoist is observable. + * + * The gate: `prefixMemo(files).size` after N imports from D + * importer directories must be D, for every N. That is "the prefix work is O(1) + * amortized after the first import from a given directory", stated as a number. + * It is paired with a reference-identity assertion, because a memo that stores + * a FRESH array on every import posts the same size while doing all of the work + * again, and with a non-vacuity assertion, because a perfect count is equally + * true of a resolver that stopped resolving anything. + * + * Both production surfaces are driven, not the helper: `pythonScopeResolver + * .resolveImportTarget` (the scope-resolution orchestrator's adapter, the one + * that pays the walk twice) and `pythonImportStrategy` (the import-resolver + * pipeline's, via `ImportTargetWorkspace`'s shared `ResolveCtx`). They thread + * different objects around the same Set, and the memo is keyed on the Set. + * + * `legacyPrefixes` is a verbatim copy of the pre-change inline code, in the + * house style of `python-importer-ancestors.test.ts`: it is the specification, + * and the memo agreeing with it is what makes this a hoist rather than a + * behaviour change. + * + * The four arms themselves live in `test/helpers/counting-file-set.ts`, beside + * the other import-target scaffolding: this guard and the #2913 one are the + * same suite over the same importer corpus once four values are named (the + * memo, the drive, the legacy builder, the hit), and they were previously + * written out twice. The two chains still DIFFER — this one keeps the empty + * components an absolute path or a doubled separator produces, and #2913's + * drops them — which is why `legacyChain` is per-guard and the shared path-shape + * table names shapes rather than expectations. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedImport } from 'gitnexus-shared'; +import type { ImportResolutionContext } from '../../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; +import { pythonScopeResolver } from '../../../src/core/ingestion/languages/python/scope-resolver.js'; +import { resolvePythonImportInternal } from '../../../src/core/ingestion/import-resolvers/python.js'; +import { getPythonFileIndex } from '../../../src/core/ingestion/import-resolvers/python-file-index.js'; + +/** The bare-import prefix memo, which lives inside the shared per-file-set index. */ +const prefixMemo = (files: ReadonlySet): ReadonlyMap => + getPythonFileIndex(files).bareImportPrefixesByDir; +import { pythonImportStrategy } from '../../../src/core/ingestion/import-resolvers/configs/python.js'; +import { buildSuffixIndex } from '../../../src/core/ingestion/import-resolvers/utils.js'; +import type { + ImportResult, + ResolveCtx, +} from '../../../src/core/ingestion/import-resolvers/types.js'; +import { + IMPORTER_PATH_SHAPES, + NO_PARSED_FILES, + expectDistinctFileSetsGetOwnChainMemo, + expectMemoizedChainMatchesLegacy, + expectOneChainPerImporterDir, + expectSameChainObjectReused, + pythonNamedImport, + pythonNamespaceImport, + type ChainMemoArm, + type ChainMemoResult, +} from '../../helpers/counting-file-set.js'; + +const { resolveImportTarget } = pythonScopeResolver; + +// ─── verbatim pre-change implementation ────────────────────────────────────── + +/** The prefix sequence the inline walk materialized on every import. */ +function legacyPrefixes(currentFile: string): string[] { + const importerDir = currentFile.replace(/\\/g, '/').split('/').slice(0, -1).join('/'); + const prefixes: string[] = []; + const dirParts = importerDir.split('/'); + for (let i = dirParts.length - 1; i >= 0; i--) { + const ancestorDir = dirParts.slice(0, i).join('/'); + prefixes.push(ancestorDir ? `${ancestorDir}/` : ''); + } + return prefixes; +} + +// ─── surfaces ──────────────────────────────────────────────────────────────── + +/** + * A `ResolveCtx` for the import-resolver pipeline surface. `allFilePaths` is + * the caller's Set passed THROUGH — the memo is keyed on its identity, so a + * copy here would measure nothing. + */ +function makeResolveCtx(allFilePaths: Set): ResolveCtx { + const allFileList = [...allFilePaths]; + const normalizedFileList = allFileList.map((file) => file.replace(/\\/g, '/')); + return { + allFilePaths, + allFileList, + normalizedFileList, + index: buildSuffixIndex(normalizedFileList, allFileList), + resolveCache: new Map(), + configs: { + tsconfigPaths: null, + goModule: null, + composerConfig: null, + swiftPackageConfig: null, + csharpConfigs: [], + }, + }; +} + +const ctxFor = (parsedImport: ParsedImport): ImportResolutionContext => ({ + parsedFiles: NO_PARSED_FILES, + parsedImport, +}); + +/** + * An `ImportResult` as the arms read it: the resolved path, or `null` for a + * miss. A stop-the-chain result carrying no files joins to `''`, which is + * neither the hit nor a miss — the same distinction the pre-collapse arm drew + * with `value?.kind === 'files' && value.files.join() === HIT_RESULT`. + */ +const resolvedPath = (result: ImportResult): ChainMemoResult => + result === null ? null : result.files.join(); + +// ─── the workspace the surfaces are driven against ─────────────────────────── + +/** + * `shared.py` sits at the workspace root, which is the LAST step of the walk, + * so every importer below reaches it only by running the chain to the end — + * the most expensive path, and the one the memo has to keep correct. + * + * `elsewhere/deep/probe.py` makes `probe` a segment that SURVIVES + * `pythonSegmentAbsent` (a file with that basename exists) while sitting in no + * importer's ancestry, so the walk runs to completion and misses. That + * combination is what a memo-filling miss looks like now: a segment the + * workspace has never heard of is retired in two Map lookups and never reaches + * the walk at all, which is the point of the early-out and the reason a + * `ghost{i}` spelling can no longer drive this memo. + */ +const WORKSPACE: readonly string[] = [ + 'svc/a/one.py', + 'svc/a/two.py', + 'svc/b/one.py', + 'svc/common.py', + 'deep/x/y/z/one.py', + 'elsewhere/deep/probe.py', + 'shared.py', + 'root.py', +]; + +const HIT_TARGET = 'shared'; +const HIT_RESULT = 'shared.py'; + +/** Survives the absence proof, then walks the whole chain and misses — the + * dotted tier's suffix fallback picks it up as `elsewhere/deep/probe.py`. */ +const WALK_TARGET = 'probe'; +/** Provably absent: retired before the walk, so it never touches the memo. */ +const ABSENT_TARGET = 'ghostmod'; + +/** + * The spelling sequence BOTH surfaces are driven with, `perImporter` rounds of + * it plus the one spelling that must resolve: one target that walks the whole + * chain and misses, one that the absence proof retires before the walk, and one + * that hits at the workspace root. No spelling is varied per round — the Python + * chain keeps no per-target cache, so a repeated target really is re-resolved. + * + * `resolve` is the only thing that differs between the orchestrator adapter and + * the import-resolver pipeline; everything about the sequence is shared, which + * is why the two used to be the same loop written twice. + */ +function drive( + fromFile: string, + perImporter: number, + resolve: (targetRaw: string, fromFile: string) => T, +): T[] { + const out: T[] = []; + for (let i = 0; i < perImporter; i++) { + for (const target of [WALK_TARGET, ABSENT_TARGET]) out.push(resolve(target, fromFile)); + } + out.push(resolve(HIT_TARGET, fromFile)); + return out; +} + +/** The ORCHESTRATOR ADAPTER — the surface that pays the walk twice. */ +const adapterArm = (mkImport: (targetRaw: string) => ParsedImport): ChainMemoArm => ({ + memoOf: prefixMemo, + drive: (files, fromFile, perImporter) => + drive(fromFile, perImporter, (target, from) => + resolveImportTarget(target, from, files, undefined, ctxFor(mkImport(target))), + ), + legacyChain: legacyPrefixes, + hitResult: HIT_RESULT, +}); + +/** + * The import-resolver pipeline surface. One `ResolveCtx` per drive rather than + * one for the run, on purpose: everything the strategy reads off it is derived + * from the same Set (and `pythonImportStrategy` only ever WRITES + * `resolveCache`), so a fresh ctx around the same Set is exactly the "different + * objects, same Set" case the memo has to survive. + */ +const strategyArm: ChainMemoArm = { + memoOf: prefixMemo, + drive: (files, fromFile, perImporter) => { + const ctx = makeResolveCtx(files); + return drive(fromFile, perImporter, (target, from) => + resolvedPath(pythonImportStrategy(target, from, ctx)), + ); + }, + legacyChain: legacyPrefixes, + hitResult: HIT_RESULT, +}; + +describe('Python bare-import prefix memo', () => { + it.each([ + { perImporter: 1, kind: 'namespace', mkImport: pythonNamespaceImport }, + { perImporter: 40, kind: 'namespace', mkImport: pythonNamespaceImport }, + { perImporter: 1, kind: 'named', mkImport: pythonNamedImport }, + { perImporter: 40, kind: 'named', mkImport: pythonNamedImport }, + ])( + 'holds one chain per importer DIRECTORY, not per import — $perImporter x $kind', + ({ perImporter, mkImport }) => { + expectOneChainPerImporterDir(adapterArm(mkImport), new Set(WORKSPACE), perImporter); + }, + ); + + it.each([ + { perImporter: 1, label: 'one import per importer' }, + { perImporter: 40, label: 'forty imports per importer' }, + ])( + 'holds one chain per importer DIRECTORY on the import-resolver surface too — $label', + ({ perImporter }) => { + expectOneChainPerImporterDir(strategyArm, new Set(WORKSPACE), perImporter); + }, + ); + + it('reuses the SAME chain object, rather than rebuilding and re-storing it', () => { + expectSameChainObjectReused(adapterArm(pythonNamedImport), new Set(WORKSPACE)); + }); + + it.each(IMPORTER_PATH_SHAPES)( + 'memoizes the chain the pre-change code built — $why', + ({ fromFile }) => { + expectMemoizedChainMatchesLegacy( + adapterArm(pythonNamespaceImport), + new Set(WORKSPACE), + fromFile, + ); + }, + ); + + it.each([ + { perImporter: 2, label: 'two imports per importer' }, + { perImporter: 20, label: 'twenty imports per importer' }, + ])( + 'gives a distinct file set its own memo (no leak across passes) — $label', + ({ perImporter }) => { + expectDistinctFileSetsGetOwnChainMemo( + adapterArm(pythonNamedImport), + new Set(WORKSPACE), + new Set(WORKSPACE), + perImporter, + ); + }, + ); + + /** + * The memo is filled from the importers a pass actually resolves against, so + * it is bounded by DIRECTORIES THAT IMPORT — never by the file count and + * never by the repo's directory count, which is the bound #2649 asks for. + */ + it.each([ + { dirs: 4, importsPerDir: 1 }, + { dirs: 4, importsPerDir: 50 }, + { dirs: 30, importsPerDir: 7 }, + ])( + 'is bounded by importing directories, not by files or imports — $dirs dirs x $importsPerDir', + ({ dirs, importsPerDir }) => { + const paths: string[] = []; + for (let d = 0; d < dirs; d++) { + for (let f = 0; f < 25; f++) paths.push(`pkg${d}/nest/file${f}.py`); + } + paths.push('shared.py'); + const files = new Set(paths); + const resolved: ChainMemoResult[] = []; + + for (let d = 0; d < dirs; d++) { + for (let i = 0; i < importsPerDir; i++) { + resolved.push( + resolveImportTarget( + HIT_TARGET, + `pkg${d}/nest/file0.py`, + files, + undefined, + ctxFor(pythonNamedImport(HIT_TARGET)), + ), + ); + } + } + + expect(prefixMemo(files).size).toBe(dirs); + expect(resolved.filter((value) => value === HIT_RESULT)).toHaveLength(dirs * importsPerDir); + }, + ); +}); + +/** + * Absolute expectations for the walk itself. `test/unit/suffix-index-ambiguity + * .test.ts` covers the proximity tier and the suffix fallback around it; the + * ANCESTOR tier — the thing issue #417 added and the thing this change touches + * — had no absolute coverage at all, in particular none for the two path shapes + * where its chain differs from `ancestorsByDir`'s: absolute paths and doubled + * separators, both of which a `filter(Boolean)` would send to the wrong files. + */ +describe('Python bare-import ancestor walk — resolution', () => { + it.each([ + { + why: 'the importer own directory wins over every ancestor', + files: ['app/svc/user.py', 'app/user.py', 'user.py', 'app/svc/auth.py'], + fromFile: 'app/svc/auth.py', + expected: 'app/svc/user.py', + }, + { + why: 'a same-directory package beats a same-directory module (PEP 451 §4)', + files: ['app/svc/user/__init__.py', 'app/svc/user.py', 'app/svc/auth.py'], + fromFile: 'app/svc/auth.py', + expected: 'app/svc/user/__init__.py', + }, + { + why: 'the CLOSEST ancestor wins (#417)', + files: ['app/user.py', 'user.py', 'app/svc/auth.py'], + fromFile: 'app/svc/auth.py', + expected: 'app/user.py', + }, + { + why: 'a package beats a module at the same ancestor step', + files: ['app/user/__init__.py', 'app/user.py', 'app/svc/auth.py'], + fromFile: 'app/svc/auth.py', + expected: 'app/user/__init__.py', + }, + { + why: 'the workspace root is the last step of the walk', + files: ['user.py', 'app/svc/auth.py'], + fromFile: 'app/svc/auth.py', + expected: 'user.py', + }, + { + why: 'a root-level importer walks the root and nothing else', + files: ['user.py', 'auth.py'], + fromFile: 'auth.py', + expected: 'user.py', + }, + { + why: 'an absolute workspace keeps the leading empty component', + files: ['/repo/app/user.py', '/repo/app/svc/auth.py'], + fromFile: '/repo/app/svc/auth.py', + expected: '/repo/app/user.py', + }, + { + why: 'a doubled separator keeps the empty component', + files: ['a//user.py', 'a//b/auth.py'], + fromFile: 'a//b/auth.py', + expected: 'a//user.py', + }, + { + why: 'Windows separators in the importer normalize before the walk', + files: ['app/user.py', 'app/svc/auth.py'], + fromFile: 'app\\svc\\auth.py', + expected: 'app/user.py', + }, + ])('$why', ({ files, fromFile, expected }) => { + const set = new Set(files); + // The helper, the scope-resolution adapter and the import-resolver + // strategy must agree: all three reach the same walk. + expect(resolvePythonImportInternal(fromFile, 'user', set)).toBe(expected); + expect( + resolveImportTarget('user', fromFile, set, undefined, ctxFor(pythonNamespaceImport('user'))), + ).toBe(expected); + expect(pythonImportStrategy('user', fromFile, makeResolveCtx(set))).toEqual({ + kind: 'files', + files: [expected], + }); + }); + + it.each([ + { + why: 'a module outside the importer ancestry is not an ancestor hit (#417)', + files: ['other/branch/user.py', 'app/svc/auth.py'], + fromFile: 'app/svc/auth.py', + }, + { + why: 'a namespace package (no __init__.py) has no file to resolve to', + files: ['app/user/model.py', 'app/svc/auth.py'], + fromFile: 'app/svc/auth.py', + }, + { + why: 'a sibling directory of the importer is not an ancestor', + files: ['app/other/user.py', 'app/svc/auth.py'], + fromFile: 'app/svc/auth.py', + }, + { + why: 'an absolute workspace does not answer a de-rooted prefix', + files: ['repo/app/user.py', '/repo/app/svc/auth.py'], + fromFile: '/repo/app/svc/auth.py', + }, + ])('returns null and lets the caller fall through — $why', ({ files, fromFile }) => { + expect(resolvePythonImportInternal(fromFile, 'user', new Set(files))).toBeNull(); + }); +}); diff --git a/gitnexus/test/unit/import-resolvers/suffix-index-lazy-dir-map.test.ts b/gitnexus/test/unit/import-resolvers/suffix-index-lazy-dir-map.test.ts new file mode 100644 index 000000000..3e4188c9c --- /dev/null +++ b/gitnexus/test/unit/import-resolvers/suffix-index-lazy-dir-map.test.ts @@ -0,0 +1,910 @@ +/** + * `buildSuffixIndex` builds NOTHING at construction. Each of its three maps is + * built the first time a question needs it, and one of them is DERIVED from + * another rather than traversed for. (The file name predates the change: #2903 + * deferred `dirMap` alone, and the two suffix maps followed.) + * + * What the index now does: + * + * - `get` builds `exactMap` by one pass over the file list; + * - `getInsensitive` after `get` DERIVES `lowerMap` from `exactMap` — one pass + * over that map's distinct keys, not a second pass over the file list; + * - `getInsensitive` first builds `lowerMap` straight off the file list, so a + * case-insensitive-only consumer holds exactly one map. Asking `get` + * afterwards is the documented fallback and does cost the second traversal; + * no consumer uses that order; + * - `getFilesInDir` builds `dirMap`, unchanged since #2903; + * - and over the pre-lowercased list `pass-cache.ts` hands it (TypeScript, + * JavaScript, Vue) the derivation is the identity, so `getInsensitive` reads + * the exact map ITSELF rather than a copy of it — one map, both questions. + * + * All of it is memory. `dirMap` is the array-valued map — one entry and one + * array push per file per directory component, O(files × depth) in entries and + * in churn — and only four call sites ever read it + * (`import-resolvers/{php,csharp,jvm}.ts`, `import-resolvers/configs/ + * python.ts`), yet `workspace-file-index.ts` serving Ruby, + * `languages/typescript/scope-resolver.ts`, `languages/vue/import-target.ts` + * and `group/extractors/include-extractor.ts` all built it and never touched + * it: ~15% of the retained C# index and ~19% of the retained Ruby one on + * `bench/import-target/`'s 32k-path arms. The suffix maps are the same story + * one level down — `languages/java/import-target.ts` reads only `get` and was + * carrying 49.98 MiB of dead `lowerMap` at 32k paths, `languages/php/ + * import-target.ts` reads only `getInsensitive` and was carrying 34.49 MiB of + * dead `exactMap`. Retained index: Java 80.26 -> 25.61 MiB, PHP 60.86 -> + * 32.09, JavaScript 44.07 -> 22.65. Since #2877-#2880 these indexes live for a + * whole resolution pass rather than being rebuilt per import, so all of that is + * retained memory against the #2649 kernel-scale OOM constraint. + * + * Deferring and deriving are only free if two things hold, and this file + * asserts both: + * + * 1. **Nothing observable moved.** `eagerDirMap` and `eagerSuffixMaps` below + * are verbatim copies of the pre-change loops, and the parity arms compare + * the built-on-demand answers against them over the FULL key space each + * corpus can produce — every suffix in three spellings, every directory + * suffix crossed with every extension, hits and misses alike — plus + * hand-written arms that pin buckets, collisions and their ORDER outright, + * so a parity arm cannot pass by two implementations being wrong together. + * Order is load-bearing twice over: `php.ts` returns `candidates[0]`, and + * the derived `lowerMap` is claimed byte-equal to a freshly built one in + * keys, values AND insertion order. Insertion order is not readable through + * this API, but its one consequence is: which file a case-folded key + * resolves to when several fold together. The parity arms run in BOTH build + * orders — derived and built-direct — over corpora that include + * case-colliding twins and a context-sensitive Greek final sigma. + * 2. **Each map is built at most ONCE, and only if asked for.** The laziness + * arms count index reads of the two input arrays. Every build pass reads + * each element exactly once, so the read count IS the pass count: 0 after + * construction, 1 for a `get`-only consumer however many times it asks, 1 + * for a `getInsensitive`-only consumer, still 1 for `get` THEN + * `getInsensitive` because the derivation reads no file, and one more — + * once, forever — for `getFilesInDir`. Memoizing the DECISION rather than + * the MAP, rebuilding whenever a lookup misses, reads 3, 4, 5. This is a + * structural count, not a timing or a memory delta: exact and deterministic + * on any machine. + * + * What the counter cannot see is a map derived from another map, since that + * touches no file: it would catch a `lowerMap` rebuilt from the LIST beside a + * `get`, not one copied from `exactMap`. That is why the derivation is measured + * as costing zero passes rather than assumed absent, and why its contents are + * policed by the parity arms instead. + * + * Every count assertion is paired with a result assertion. A pass count of 0 is + * equally true of an index that has stopped answering — the pairing rule of + * `test/helpers/counting-file-set.ts` and the twelve + * `test/integration/*-import-index-reuse.test.ts` guards. + */ +import { describe, expect, it } from 'vitest'; + +import { + buildSuffixIndex, + type SuffixIndex, +} from '../../../src/core/ingestion/import-resolvers/utils.js'; + +// ─── verbatim pre-change implementations ───────────────────────────────────── + +/** + * The directory-membership half of `buildSuffixIndex` exactly as it stood + * before #2903, lifted out of the shared loop and otherwise untouched. This is + * the specification the deferred build is measured against. + */ +function eagerDirMap(normalizedFileList: string[], allFileList: string[]): Map { + const dirMap = new Map(); + + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + const original = allFileList[i]; + const parts = normalized.split('/'); + + const lastSlash = normalized.lastIndexOf('/'); + if (lastSlash >= 0) { + const dirParts = parts.slice(0, -1); + const fileName = parts[parts.length - 1]; + const ext = fileName.substring(fileName.lastIndexOf('.')); + + for (let j = dirParts.length - 1; j >= 0; j--) { + const dirSuffix = dirParts.slice(j).join('/'); + const key = `${dirSuffix}:${ext}`; + let list = dirMap.get(key); + if (!list) { + list = []; + dirMap.set(key, list); + } + list.push(original); + } + } + } + + return dirMap; +} + +/** The two suffix questions, however they happen to be answered. */ +type SuffixAnswerer = Pick; + +/** + * The two suffix maps of `buildSuffixIndex` exactly as they stood before this + * change: ONE fused traversal writing both, suffixes cut with `split('/')` plus + * `slice(j).join('/')`, first spelling winning in each map independently. This + * is the specification both the deferred exact map and the derived case-folded + * map are measured against — and it is also the reference for the suffix- + * cutting rewrite that came with them (the production loop now walks slash + * offsets and slices the original string instead of re-joining parts). + */ +function eagerSuffixMaps(normalizedFileList: string[], allFileList: string[]): SuffixAnswerer { + const exactMap = new Map(); + const lowerMap = new Map(); + + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + const original = allFileList[i]; + const parts = normalized.split('/'); + + for (let j = parts.length - 1; j >= 0; j--) { + const suffix = parts.slice(j).join('/'); + // Only store first match (longest path wins for ambiguous suffixes) + if (!exactMap.has(suffix)) { + exactMap.set(suffix, original); + } + const lower = suffix.toLowerCase(); + if (!lowerMap.has(lower)) { + lowerMap.set(lower, original); + } + } + } + + return { + get: (suffix: string) => exactMap.get(suffix), + getInsensitive: (suffix: string) => lowerMap.get(suffix.toLowerCase()), + }; +} + +// ─── corpus ────────────────────────────────────────────────────────────────── + +/** + * Raw paths, in index order. Each entry is here for a reason the parity arms + * would not otherwise reach: + * + * - `src/com/{example,other}` — the same basename under two directories that + * share a parent, so `com/example` and `example` must select differently; + * - `app/Models/Legacy/User.php` — a file one level DEEPER than the bucket + * under test, which must not appear in `Models`'s bucket (the map is keyed + * on directory SUFFIX, not prefix); + * - `Makefile` — repo root, no directory at all, and no extension: skipped + * entirely by `dirMap`'s `lastSlash >= 0` guard, while the suffix maps still + * hold it under its whole-path key, which is the one key the slash walk + * cannot emit; + * - `scripts/build` — no extension, but IN a directory. `lastIndexOf('.')` is + * -1 and `substring(-1)` clamps to 0, so the extension is the whole + * filename and the key is `scripts:build`. Odd, long-standing, and pinned + * here so deferring the build cannot quietly "fix" it; + * - `lib/vendor.min.js` before `lib/vendor.js` — multiple dots (extension is + * the LAST one), and a two-entry bucket whose order is not alphabetical, so + * an implementation that sorted or reversed would be caught; + * - `win\pkg\Thing.cs` — a backslash path, so the arms cover the raw-vs- + * normalized split: keys come off the NORMALIZED path, values are the + * ORIGINAL one. + */ +const RAW_FILES: readonly string[] = [ + 'src/com/example/Foo.java', + 'src/com/example/Bar.java', + 'src/com/other/Foo.java', + 'app/Models/User.php', + 'app/Models/Post.php', + 'app/Models/Legacy/User.php', + 'Makefile', + 'scripts/build', + 'scripts/deploy.sh', + 'lib/vendor.min.js', + 'lib/vendor.js', + 'a/b/c/d.ts', + 'b/c/d.ts', + 'win\\pkg\\Thing.cs', +]; + +const ALL_FILES: string[] = [...RAW_FILES]; +const NORMALIZED_FILES: string[] = ALL_FILES.map((f) => f.replace(/\\/g, '/')); + +/** + * Two paths that differ only in case, plus a three-way collision. Nothing about + * the exact map is exercised here; the point is the case-folded one, where all + * three spellings collapse to a single key and only ONE file can answer it. The + * file that does is decided by insertion order, so this corpus is what makes + * "the derived map has the same insertion order as a freshly built one" an + * observable claim rather than an internal one. + */ +const CASE_TWIN_FILES: readonly string[] = [ + 'src/Util/Helper.php', + 'src/util/helper.php', + 'app/README.md', + 'app/ReadMe.md', + 'app/readme.md', + 'lib/Model/User.php', + 'lib/model/USER.PHP', +]; + +/** + * Paths whose `.toLowerCase()` is not a per-character mapping. + * + * `Σ` folds to `ς` at the end of a word and to `σ` elsewhere, and JS applies + * that context rule: `'ΟΔΟΣ/x'` folds the sigma to `ς` (a slash is not a cased + * letter, so the word ends) while `'ΟΔΟΣ.ts'` folds it to `σ` (`t` is). So + * `src/ΟΔΟΣ/ΟΔΟΣ.ts` contributes the same segment spelling under two DIFFERENT + * folded keys. A derivation that folded the whole path once and sliced the + * result, or that folded per character, would answer differently here. + * `İstanbul` (one code point, two after folding) and `Gruß`/`GRUSS` (which do + * NOT collide under `toLowerCase`, unlike under full case folding) pin the two + * other classic hazards. + */ +const UNICODE_FOLDING_FILES: readonly string[] = [ + 'src/ΟΔΟΣ/Καλημέρα.ts', + 'src/ΟΔΟΣ.ts', + 'src/ΟΔΟΣ/ΟΔΟΣ.ts', + 'i18n/İstanbul/Page.tsx', + 'de/STRASSE/Gruß.ts', + 'de/strasse/GRUSS.ts', +]; + +/** + * Slash spellings where cutting a suffix by slash offsets could disagree with + * `split('/')` + `join('/')`: a leading slash (where the walk stops early and + * the whole-path key is left to the write after the loop), a doubled slash (an + * empty segment mid-path), a trailing slash (an empty final segment), and a + * path with no slash at all, whose only key is that final write. + */ +const ODD_SLASH_FILES: readonly string[] = ['/root.ts', 'a//b/c.ts', 'dir/trailing/', 'plain.ts']; + +interface Corpus { + readonly name: string; + readonly raw: readonly string[]; +} + +const PARITY_CORPORA: readonly Corpus[] = [ + { name: 'base', raw: RAW_FILES }, + { name: 'case-colliding twins', raw: CASE_TWIN_FILES }, + { name: 'unicode folding', raw: UNICODE_FOLDING_FILES }, + { name: 'slash oddities', raw: ODD_SLASH_FILES }, +]; + +function corpusLists(raw: readonly string[]): { all: string[]; normalized: string[] } { + const all = [...raw]; + return { all, normalized: all.map((f) => f.replace(/\\/g, '/')) }; +} + +// ─── probe spaces ──────────────────────────────────────────────────────────── + +/** Every directory suffix the corpus can produce, in first-seen order. */ +function corpusDirSuffixes(normalized: readonly string[]): string[] { + const suffixes: string[] = []; + for (const path of normalized) { + const parts = path.split('/'); + const dirParts = parts.slice(0, -1); + for (let j = dirParts.length - 1; j >= 0; j--) { + const suffix = dirParts.slice(j).join('/'); + if (!suffixes.includes(suffix)) suffixes.push(suffix); + } + } + return suffixes; +} + +/** Every extension the corpus can produce, in first-seen order. */ +function corpusExtensions(normalized: readonly string[]): string[] { + const extensions: string[] = []; + for (const path of normalized) { + const fileName = path.slice(path.lastIndexOf('/') + 1); + const ext = fileName.substring(fileName.lastIndexOf('.')); + if (!extensions.includes(ext)) extensions.push(ext); + } + return extensions; +} + +/** + * The full `getFilesInDir` probe space: every directory suffix crossed with + * every extension — so the parity arm asserts the misses too, not only the 19 + * populated keys — plus spellings that exist nowhere in the corpus at all. + */ +function dirProbeSpace(normalized: readonly string[]): Array { + const probes: Array = []; + for (const dirSuffix of corpusDirSuffixes(normalized)) { + for (const ext of corpusExtensions(normalized)) probes.push([dirSuffix, ext]); + } + // Absent entirely: a directory PREFIX (`src`, which no file sits directly + // in), a case variant (the map is case-sensitive, unlike `getInsensitive`), a + // trailing-slash spelling, and a bare miss. + for (const dirSuffix of ['src', 'app', 'models', 'Models/', 'nope']) { + for (const ext of ['.php', '.java', '.nope', '']) probes.push([dirSuffix, ext]); + } + return probes; +} + +function probeAllDirs( + probes: ReadonlyArray, + lookup: (dirSuffix: string, extension: string) => readonly string[], +): Record { + const answers: Record = {}; + for (const [dirSuffix, ext] of probes) + answers[`${dirSuffix}\u0000${ext}`] = lookup(dirSuffix, ext); + return answers; +} + +/** + * The full suffix probe space: every key either suffix map can hold — every + * suffix of every path, which is exactly what the build loops insert — each in + * its own spelling plus its lowercased and uppercased forms, so the folded + * lookups are driven with queries that hit, miss and collide. Plus spellings + * absent from every corpus. + */ +function suffixProbeSpace(normalized: readonly string[]): string[] { + const probes: string[] = []; + const add = (probe: string): void => { + if (!probes.includes(probe)) probes.push(probe); + }; + for (const path of normalized) { + const parts = path.split('/'); + for (let j = parts.length - 1; j >= 0; j--) { + const suffix = parts.slice(j).join('/'); + add(suffix); + add(suffix.toLowerCase()); + add(suffix.toUpperCase()); + } + } + for (const miss of ['', '/', 'nope.java', 'NOPE.JAVA', 'src', 'Foo', 'Foo.java/']) add(miss); + return probes; +} + +/** + * Ask both suffix questions about every probe, `get` FIRST — so the exact map + * exists before the first `getInsensitive` and the folded map is DERIVED. + */ +function answersGetFirst( + probes: readonly string[], + answerer: SuffixAnswerer, +): Record { + const answers: Record = {}; + for (const probe of probes) { + answers[`exact\u0000${probe}`] = answerer.get(probe) ?? null; + answers[`folded\u0000${probe}`] = answerer.getInsensitive(probe) ?? null; + } + return answers; +} + +/** + * The same probes, `getInsensitive` FIRST — PHP's order, where the folded map + * is built straight off the file list and the exact map is the later fallback. + * `undefined` is mapped to `null` in both collectors because `toEqual` treats + * an explicitly-undefined property as an absent one. + */ +function answersInsensitiveFirst( + probes: readonly string[], + answerer: SuffixAnswerer, +): Record { + const answers: Record = {}; + for (const probe of probes) { + answers[`folded\u0000${probe}`] = answerer.getInsensitive(probe) ?? null; + answers[`exact\u0000${probe}`] = answerer.get(probe) ?? null; + } + return answers; +} + +/** Every probe of every corpus, one flat table, so one `toEqual` covers all four. */ +function answersAcrossCorpora( + collect: (probes: readonly string[], answerer: SuffixAnswerer) => Record, + build: (normalized: string[], all: string[]) => SuffixAnswerer, +): Record { + const answers: Record = {}; + for (const corpus of PARITY_CORPORA) { + const { all, normalized } = corpusLists(corpus.raw); + const collected = collect(suffixProbeSpace(normalized), build(normalized, all)); + for (const [key, value] of Object.entries(collected)) { + answers[`${corpus.name}\u0000${key}`] = value; + } + } + return answers; +} + +// ─── read-counting file lists ──────────────────────────────────────────────── + +interface CountingList { + /** A real `string[]`, so `buildSuffixIndex` takes it unmodified. */ + readonly list: string[]; + /** Element reads so far. Every build pass reads each element once. */ + reads: () => number; +} + +/** + * A `string[]` whose elements are accessor properties, so every `list[i]` is + * counted. An accessor on a real array rather than a `Proxy` keeps the value a + * genuine `Array` — `.length` and every array method behave normally — and + * counts only indexed reads, never `.length`. + */ +function countingList(paths: readonly string[]): CountingList { + let reads = 0; + const list = new Array(paths.length); + paths.forEach((value, i) => { + Object.defineProperty(list, i, { + enumerable: true, + configurable: true, + get: () => { + reads += 1; + return value; + }, + }); + }); + return { list, reads: () => reads }; +} + +/** One full pass over the file list reads every element of both arrays once. */ +const ONE_PASS = RAW_FILES.length; + +/** Lookups driven bare before a count is read — the count must not move. */ +const LOOKUP_REPEATS = 20; + +/** `import-resolvers/pass-cache.ts` builds exactly this: lowercased, not slash-normalized. */ +const LOWERCASED_FILES: string[] = ALL_FILES.map((f) => f.toLowerCase()); + +// ─── parity: the directory map ─────────────────────────────────────────────── + +describe('buildSuffixIndex.getFilesInDir — the deferred build is behaviour-identical', () => { + it('answers the full probe space exactly as the eager implementation did', () => { + const index = buildSuffixIndex(NORMALIZED_FILES, ALL_FILES); + const reference = eagerDirMap(NORMALIZED_FILES, ALL_FILES); + const probes = dirProbeSpace(NORMALIZED_FILES); + + const deferred = probeAllDirs(probes, (dir, ext) => index.getFilesInDir(dir, ext)); + const eager = probeAllDirs(probes, (dir, ext) => reference.get(`${dir}:${ext}`) ?? []); + + // A guard on the instrument: an empty or collapsed probe space would make + // the comparison below vacuous. + expect(probes.length).toBe(164); + expect(Object.values(eager).filter((files) => files.length > 0).length).toBe(19); + expect(deferred).toEqual(eager); + }); + + it('pins each bucket and its order outright, not only against the old code', () => { + const index = buildSuffixIndex(NORMALIZED_FILES, ALL_FILES); + + expect({ + // Same basename under sibling directories: the deeper suffix disambiguates. + 'example:.java': index.getFilesInDir('example', '.java'), + 'com/example:.java': index.getFilesInDir('com/example', '.java'), + 'src/com/example:.java': index.getFilesInDir('src/com/example', '.java'), + 'other:.java': index.getFilesInDir('other', '.java'), + // `Legacy/User.php` is one level deeper and belongs to `Legacy`, not `Models`. + 'Models:.php': index.getFilesInDir('Models', '.php'), + 'Legacy:.php': index.getFilesInDir('Legacy', '.php'), + 'Models/Legacy:.php': index.getFilesInDir('Models/Legacy', '.php'), + // Multiple dots: the extension is the LAST one, and the bucket keeps + // index order (`vendor.min.js` was indexed first) rather than sorting. + 'lib:.js': index.getFilesInDir('lib', '.js'), + // No extension at all: `substring(-1)` clamps to 0, so the "extension" + // is the whole filename. + 'scripts:build': index.getFilesInDir('scripts', 'build'), + 'scripts:.sh': index.getFilesInDir('scripts', '.sh'), + // Keyed on the normalized path, holding the ORIGINAL raw one. + 'pkg:.cs': index.getFilesInDir('pkg', '.cs'), + 'win/pkg:.cs': index.getFilesInDir('win/pkg', '.cs'), + // Two files in same-named leaf directories at different depths. + 'c:.ts': index.getFilesInDir('c', '.ts'), + 'b/c:.ts': index.getFilesInDir('b/c', '.ts'), + 'a/b/c:.ts': index.getFilesInDir('a/b/c', '.ts'), + // Misses: a repo-root file is in no bucket; `src` is a PREFIX, never a + // directory suffix any file sits directly in; the key is case-sensitive. + ':': index.getFilesInDir('', ''), + 'src:.java': index.getFilesInDir('src', '.java'), + 'models:.php': index.getFilesInDir('models', '.php'), + 'Models:.java': index.getFilesInDir('Models', '.java'), + }).toEqual({ + 'example:.java': ['src/com/example/Foo.java', 'src/com/example/Bar.java'], + 'com/example:.java': ['src/com/example/Foo.java', 'src/com/example/Bar.java'], + 'src/com/example:.java': ['src/com/example/Foo.java', 'src/com/example/Bar.java'], + 'other:.java': ['src/com/other/Foo.java'], + 'Models:.php': ['app/Models/User.php', 'app/Models/Post.php'], + 'Legacy:.php': ['app/Models/Legacy/User.php'], + 'Models/Legacy:.php': ['app/Models/Legacy/User.php'], + 'lib:.js': ['lib/vendor.min.js', 'lib/vendor.js'], + 'scripts:build': ['scripts/build'], + 'scripts:.sh': ['scripts/deploy.sh'], + 'pkg:.cs': ['win\\pkg\\Thing.cs'], + 'win/pkg:.cs': ['win\\pkg\\Thing.cs'], + 'c:.ts': ['a/b/c/d.ts', 'b/c/d.ts'], + 'b/c:.ts': ['a/b/c/d.ts', 'b/c/d.ts'], + 'a/b/c:.ts': ['a/b/c/d.ts'], + ':': [], + 'src:.java': [], + 'models:.php': [], + 'Models:.java': [], + }); + }); + + it('returns the same answers whether or not suffix lookups came first', () => { + const warmed = buildSuffixIndex(NORMALIZED_FILES, ALL_FILES); + warmed.get('Foo.java'); + warmed.getInsensitive('USER.PHP'); + warmed.getFilesInDir('nope', '.nope'); + const cold = buildSuffixIndex(NORMALIZED_FILES, ALL_FILES); + const probes = dirProbeSpace(NORMALIZED_FILES); + + expect(probeAllDirs(probes, (d, e) => warmed.getFilesInDir(d, e))).toEqual( + probeAllDirs(probes, (d, e) => cold.getFilesInDir(d, e)), + ); + }); + + it('leaves the suffix answers untouched — building the dir map moves nothing', () => { + const index = buildSuffixIndex(NORMALIZED_FILES, ALL_FILES); + index.getFilesInDir('Models', '.php'); + + expect({ + exact: index.get('example/Foo.java'), + // First path wins for an ambiguous suffix. + ambiguous: index.get('Foo.java'), + insensitive: index.getInsensitive('APP/MODELS/USER.PHP'), + // The suffix maps are built off the normalized path and return the raw one. + backslash: index.get('pkg/Thing.cs'), + miss: index.get('nope.java'), + }).toEqual({ + exact: 'src/com/example/Foo.java', + ambiguous: 'src/com/example/Foo.java', + insensitive: 'app/Models/User.php', + backslash: 'win\\pkg\\Thing.cs', + miss: undefined, + }); + }); +}); + +// ─── parity: the derived case-folded map ───────────────────────────────────── + +describe('buildSuffixIndex suffix maps — derived answers are the eager answers', () => { + it('answers the full suffix key space as the eager fused loop did, in both build orders', () => { + const eager = answersAcrossCorpora(answersGetFirst, eagerSuffixMaps); + // `get` first: the folded map is DERIVED from the exact one. + const derived = answersAcrossCorpora(answersGetFirst, (normalized, all) => + buildSuffixIndex(normalized, all), + ); + // `getInsensitive` first: the folded map is built straight off the list, + // and the exact map is the fallback traversal behind it. + const direct = answersAcrossCorpora(answersInsensitiveFirst, (normalized, all) => + buildSuffixIndex(normalized, all), + ); + + // Guards on the instrument: a collapsed probe space, or a reference that + // answered nothing, would make both comparisons vacuous. + expect(Object.keys(eager).length).toBe(430); + expect(Object.values(eager).filter((file) => file !== null).length).toBe(268); + expect(derived).toEqual(eager); + expect(direct).toEqual(eager); + }); + + it('pins first-in-file-order for case-folded collisions, derived or built direct', () => { + const { all, normalized } = corpusLists(CASE_TWIN_FILES); + const derived = buildSuffixIndex(normalized, all); + // Exact map first, so the folded map below is derived rather than built. + const derivedExact = derived.get('Util/Helper.php'); + const direct = buildSuffixIndex(normalized, all); + + expect({ + derivedExact, + // Two spellings of one path; the FIRST indexed answers both queries and + // `src/util/helper.php` answers neither. Insertion order is the only + // thing that decides this, and it must survive the derivation. + derivedTwin: derived.getInsensitive('HELPER.PHP'), + directTwin: direct.getInsensitive('HELPER.PHP'), + derivedTwinPath: derived.getInsensitive('SRC/UTIL/HELPER.PHP'), + directTwinPath: direct.getInsensitive('SRC/UTIL/HELPER.PHP'), + // Three spellings collapse to one folded key: the first still wins. + derivedThreeWay: derived.getInsensitive('app/readme.md'), + directThreeWay: direct.getInsensitive('app/readme.md'), + // The exact map keeps them apart; only the folded one collapses. + derivedUpper: derived.get('Model/User.php'), + derivedLower: derived.get('model/USER.PHP'), + // `get` after `getInsensitive` is the fallback order: a second traversal, + // the same answers. + directUpper: direct.get('Model/User.php'), + directLower: direct.get('model/USER.PHP'), + directMiss: direct.get('model/User.php'), + }).toEqual({ + derivedExact: 'src/Util/Helper.php', + derivedTwin: 'src/Util/Helper.php', + directTwin: 'src/Util/Helper.php', + derivedTwinPath: 'src/Util/Helper.php', + directTwinPath: 'src/Util/Helper.php', + derivedThreeWay: 'app/README.md', + directThreeWay: 'app/README.md', + derivedUpper: 'lib/Model/User.php', + derivedLower: 'lib/model/USER.PHP', + directUpper: 'lib/Model/User.php', + directLower: 'lib/model/USER.PHP', + directMiss: undefined, + }); + }); + + it('pins the context-sensitive folds outright, derived or built direct', () => { + const { all, normalized } = corpusLists(UNICODE_FOLDING_FILES); + const derived = buildSuffixIndex(normalized, all); + const derivedExact = derived.get('ΟΔΟΣ.ts'); + const direct = buildSuffixIndex(normalized, all); + + expect({ + derivedExact, + // `Σ` before `.ts` is not word-final, so it folds to `σ`... + derivedNonFinal: derived.getInsensitive('ΟΔΟΣ.ts'), + directNonFinal: direct.getInsensitive('ΟΔΟΣ.ts'), + // ...and the folded key really is spelled with `σ`, not `ς`. + derivedSigmaKey: derived.getInsensitive('οδοσ.ts'), + derivedFinalSigmaKey: derived.getInsensitive('οδος.ts'), + // Before a slash it IS word-final and folds to `ς` — the same segment + // spelling, a different key, from the same path. + derivedFinal: derived.getInsensitive('ΟΔΟΣ/ΟΔΟΣ.ts'), + directFinal: direct.getInsensitive('ΟΔΟΣ/ΟΔΟΣ.ts'), + derivedFinalTyped: derived.getInsensitive('οδος/οδοσ.ts'), + // One code point folding to two: `İ` -> `i` + U+0307. + derivedDotted: derived.getInsensitive('İSTANBUL/PAGE.TSX'), + directDotted: direct.getInsensitive('İstanbul/Page.tsx'), + // `ß` and `SS` are distinct under `toLowerCase`, unlike full case folding. + derivedSharpS: derived.getInsensitive('STRASSE/GRUß.TS'), + derivedDoubleS: derived.getInsensitive('STRASSE/GRUSS.TS'), + }).toEqual({ + derivedExact: 'src/ΟΔΟΣ.ts', + derivedNonFinal: 'src/ΟΔΟΣ.ts', + directNonFinal: 'src/ΟΔΟΣ.ts', + derivedSigmaKey: 'src/ΟΔΟΣ.ts', + derivedFinalSigmaKey: undefined, + derivedFinal: 'src/ΟΔΟΣ/ΟΔΟΣ.ts', + directFinal: 'src/ΟΔΟΣ/ΟΔΟΣ.ts', + derivedFinalTyped: 'src/ΟΔΟΣ/ΟΔΟΣ.ts', + derivedDotted: 'i18n/İstanbul/Page.tsx', + directDotted: 'i18n/İstanbul/Page.tsx', + derivedSharpS: 'de/STRASSE/Gruß.ts', + derivedDoubleS: 'de/strasse/GRUSS.ts', + }); + }); +}); + +// ─── laziness ──────────────────────────────────────────────────────────────── + +describe('buildSuffixIndex — nothing is built at construction, each map once on first use', () => { + it('reads the file list zero times at construction, and one pass for all suffix lookups', () => { + const normalized = countingList(NORMALIZED_FILES); + const all = countingList(ALL_FILES); + + const index = buildSuffixIndex(normalized.list, all.list); + const afterBuild = { normalized: normalized.reads(), all: all.reads() }; + + index.get('Foo.java'); + index.get('nope.java'); + index.getInsensitive('APP/MODELS/USER.PHP'); + index.getInsensitive('nope.java'); + + expect({ + afterBuild, + afterSuffixLookups: { normalized: normalized.reads(), all: all.reads() }, + // A count of zero is equally the count of an index that answers nothing. + answer: index.get('Foo.java'), + folded: index.getInsensitive('APP/MODELS/USER.PHP'), + }).toEqual({ + afterBuild: { normalized: 0, all: 0 }, + afterSuffixLookups: { normalized: ONE_PASS, all: ONE_PASS }, + answer: 'src/com/example/Foo.java', + folded: 'app/Models/User.php', + }); + }); + + it('builds ONE map for a get-only consumer — Java never pays for the folded map', () => { + const normalized = countingList(NORMALIZED_FILES); + const all = countingList(ALL_FILES); + + const index = buildSuffixIndex(normalized.list, all.list); + const atConstruction = { normalized: normalized.reads(), all: all.reads() }; + + // Driven bare: asserting inside the loop restates one bit twenty times. + for (let i = 0; i < LOOKUP_REPEATS; i++) { + index.get('Foo.java'); + index.get('example/Foo.java'); + index.get('nope.java'); + } + + expect({ + // A second map built eagerly BESIDE the exact one shows up HERE — fused + // into the same loop, as it used to be, or in a loop of its own. Fused, + // the total below does not move at all, so this field is the only thing + // that catches it. + atConstruction, + afterManyGets: { normalized: normalized.reads(), all: all.reads() }, + exact: index.get('example/Foo.java'), + ambiguous: index.get('Foo.java'), + miss: index.get('nope.java'), + }).toEqual({ + atConstruction: { normalized: 0, all: 0 }, + afterManyGets: { normalized: ONE_PASS, all: ONE_PASS }, + exact: 'src/com/example/Foo.java', + ambiguous: 'src/com/example/Foo.java', + miss: undefined, + }); + }); + + it('builds ONE map for a getInsensitive-only consumer — PHP never pays for the exact map', () => { + const normalized = countingList(NORMALIZED_FILES); + const all = countingList(ALL_FILES); + + const index = buildSuffixIndex(normalized.list, all.list); + const atConstruction = { normalized: normalized.reads(), all: all.reads() }; + + for (let i = 0; i < LOOKUP_REPEATS; i++) { + index.getInsensitive('USER.PHP'); + index.getInsensitive('APP/MODELS/USER.PHP'); + index.getInsensitive('NOPE.PHP'); + } + + expect({ + atConstruction, + afterManyLookups: { normalized: normalized.reads(), all: all.reads() }, + basename: index.getInsensitive('USER.PHP'), + path: index.getInsensitive('APP/MODELS/USER.PHP'), + miss: index.getInsensitive('NOPE.PHP'), + }).toEqual({ + atConstruction: { normalized: 0, all: 0 }, + afterManyLookups: { normalized: ONE_PASS, all: ONE_PASS }, + basename: 'app/Models/User.php', + path: 'app/Models/User.php', + miss: undefined, + }); + }); + + it('derives the folded map from the exact one — the second question costs no pass', () => { + const normalized = countingList(NORMALIZED_FILES); + const all = countingList(ALL_FILES); + + const index = buildSuffixIndex(normalized.list, all.list); + const exact = index.get('Foo.java'); + const afterGet = { normalized: normalized.reads(), all: all.reads() }; + + for (let i = 0; i < LOOKUP_REPEATS; i++) { + index.getInsensitive('FOO.JAVA'); + index.getInsensitive('APP/MODELS/USER.PHP'); + index.getInsensitive('NOPE.JAVA'); + } + + expect({ + afterGet, + // The derivation walks the exact map's keys, never the file list, so this + // must not move. A second full traversal would read a second pass. + afterDerivation: { normalized: normalized.reads(), all: all.reads() }, + exact, + folded: index.getInsensitive('FOO.JAVA'), + foldedPath: index.getInsensitive('APP/MODELS/USER.PHP'), + foldedMiss: index.getInsensitive('NOPE.JAVA'), + }).toEqual({ + afterGet: { normalized: ONE_PASS, all: ONE_PASS }, + afterDerivation: { normalized: ONE_PASS, all: ONE_PASS }, + exact: 'src/com/example/Foo.java', + folded: 'src/com/example/Foo.java', + foldedPath: 'app/Models/User.php', + foldedMiss: undefined, + }); + }); + + it('pays the second traversal only in the order no consumer uses — folded, then exact', () => { + const normalized = countingList(NORMALIZED_FILES); + const all = countingList(ALL_FILES); + + const index = buildSuffixIndex(normalized.list, all.list); + const folded = index.getInsensitive('FOO.JAVA'); + const afterInsensitive = { normalized: normalized.reads(), all: all.reads() }; + + for (let i = 0; i < LOOKUP_REPEATS; i++) index.get('Foo.java'); + + expect({ + afterInsensitive, + // There is nothing to derive an exact map FROM, so this order costs the + // traversal the other one saves. Documented, unused, and still correct. + afterFallback: { normalized: normalized.reads(), all: all.reads() }, + folded, + exact: index.get('Foo.java'), + // Case-sensitive again, which is the point of the fallback being a real + // second map rather than an alias of the folded one. + exactMiss: index.get('FOO.JAVA'), + }).toEqual({ + afterInsensitive: { normalized: ONE_PASS, all: ONE_PASS }, + afterFallback: { normalized: ONE_PASS * 2, all: ONE_PASS * 2 }, + folded: 'src/com/example/Foo.java', + exact: 'src/com/example/Foo.java', + exactMiss: undefined, + }); + }); + + it('takes exactly one more pass on the first getFilesInDir, and none after', () => { + const normalized = countingList(NORMALIZED_FILES); + const all = countingList(ALL_FILES); + + const index = buildSuffixIndex(normalized.list, all.list); + const beforeFirst = { normalized: normalized.reads(), all: all.reads() }; + + index.getFilesInDir('Models', '.php'); + const afterFirst = { normalized: normalized.reads(), all: all.reads() }; + + // Hits, misses and repeats alike: memoizing the DECISION instead of the + // MAP would rebuild on every one of these and the count would climb. + index.getFilesInDir('Models', '.php'); + index.getFilesInDir('example', '.java'); + index.getFilesInDir('nope', '.nope'); + index.getFilesInDir('nope', '.nope'); + index.getFilesInDir('', ''); + + expect({ + beforeFirst, + afterFirst, + afterMany: { normalized: normalized.reads(), all: all.reads() }, + answer: index.getFilesInDir('Models', '.php'), + }).toEqual({ + beforeFirst: { normalized: 0, all: 0 }, + afterFirst: { normalized: ONE_PASS, all: ONE_PASS }, + afterMany: { normalized: ONE_PASS, all: ONE_PASS }, + answer: ['app/Models/User.php', 'app/Models/Post.php'], + }); + }); + + it('defers per index, not per module — a second index starts cold', () => { + const firstNormalized = countingList(NORMALIZED_FILES); + const firstAll = countingList(ALL_FILES); + const first = buildSuffixIndex(firstNormalized.list, firstAll.list); + first.getFilesInDir('Models', '.php'); + + const secondNormalized = countingList(NORMALIZED_FILES); + const secondAll = countingList(ALL_FILES); + const second = buildSuffixIndex(secondNormalized.list, secondAll.list); + + expect({ + first: { normalized: firstNormalized.reads(), all: firstAll.reads() }, + // Cold even though a fully built index of the same paths exists: the maps + // hang off the closure, not off the module. + second: { normalized: secondNormalized.reads(), all: secondAll.reads() }, + // Pairing rule: a count of zero must not be the count of an index that + // answers nothing. The second index still resolves once asked. + secondAnswer: second.getFilesInDir('Models', '.php'), + secondReadsAfterAsking: secondNormalized.reads(), + }).toEqual({ + first: { normalized: ONE_PASS, all: ONE_PASS }, + second: { normalized: 0, all: 0 }, + secondAnswer: ['app/Models/User.php', 'app/Models/Post.php'], + secondReadsAfterAsking: ONE_PASS, + }); + }); + + it('aliases one map for both questions when the caller pre-lowercased the list', () => { + // `pass-cache.ts` (TypeScript, JavaScript, Vue) passes a lowercased list and + // says so, and over such a list the derivation is the identity — so the + // folded map is the exact map, not a copy of it, in either asking order. + const getFirstNormalized = countingList(LOWERCASED_FILES); + const getFirstAll = countingList(ALL_FILES); + const getFirst = buildSuffixIndex(getFirstNormalized.list, getFirstAll.list, { + alreadyLowercased: true, + }); + const getFirstExact = getFirst.get('app/models/user.php'); + const getFirstFolded = getFirst.getInsensitive('APP/MODELS/USER.PHP'); + + const foldedFirstNormalized = countingList(LOWERCASED_FILES); + const foldedFirstAll = countingList(ALL_FILES); + const foldedFirst = buildSuffixIndex(foldedFirstNormalized.list, foldedFirstAll.list, { + alreadyLowercased: true, + }); + const foldedFirstFolded = foldedFirst.getInsensitive('APP/MODELS/USER.PHP'); + const foldedFirstExact = foldedFirst.get('app/models/user.php'); + + expect({ + getFirstReads: { normalized: getFirstNormalized.reads(), all: getFirstAll.reads() }, + foldedFirstReads: { normalized: foldedFirstNormalized.reads(), all: foldedFirstAll.reads() }, + getFirstExact, + getFirstFolded, + foldedFirstFolded, + foldedFirstExact, + // Values are the ORIGINAL paths; only the keys were lowercased. + backslash: getFirst.getInsensitive('PKG/THING.CS'), + }).toEqual({ + getFirstReads: { normalized: ONE_PASS, all: ONE_PASS }, + foldedFirstReads: { normalized: ONE_PASS, all: ONE_PASS }, + getFirstExact: 'app/Models/User.php', + getFirstFolded: 'app/Models/User.php', + foldedFirstFolded: 'app/Models/User.php', + foldedFirstExact: 'app/Models/User.php', + backslash: undefined, + }); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/cobol-import-target-parity.test.ts b/gitnexus/test/unit/scope-resolution/cobol-import-target-parity.test.ts new file mode 100644 index 000000000..b46859c68 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/cobol-import-target-parity.test.ts @@ -0,0 +1,338 @@ +/** + * Differential harness for the COBOL `COPY`-target index hoist (#2908). + * + * `cobolScopeResolver.resolveImportTarget` used to answer every `COPY` with TWO + * full `allFilePaths` scans — copybooks first, then COBOL sources — each calling + * `path.extname` + `path.basename` + `toUpperCase` on every entry, so resolution + * cost O(copies × files) and a `COPY` of a member that is not in the repo (the + * common case) ran both scans to completion. Replacing them with a per-run + * two-tier index is a pure performance change ONLY if every implicit tie-break + * survives, and none of them is visible to the type system: + * + * - TIER ORDER: a `.cpy`/`.copybook` hit beats a `.cbl`/`.cob`/`.cobol` hit + * even when the source file comes FIRST in Set-iteration order. Collapsing + * the two tiers into one first-wins map is the "obvious" rewrite and it + * silently inverts this; + * - WITHIN A TIER: the first file in Set-iteration order wins, because the + * scans returned on first match; + * - CASE: the extension is compared LOWER-cased while the basename is + * compared UPPER-cased, and `path.basename(fp, ext)` strips the suffix only + * on an exact, case-sensitive match — so `Foo.CPY` is indexed under + * `FOO.CPY`, not `FOO`, and is unreachable by a `COPY FOO`; + * - `path` SEMANTICS: Node's `path.extname`/`path.basename` are what decide + * where the stem starts, and on POSIX they do not treat `\` as a separator. + * Hand-rolled slicing on `/` would start resolving backslash paths that + * previously returned null. + * + * So this file keeps a VERBATIM copy of the pre-change resolver body + * (`git show HEAD~:…/languages/cobol/scope-resolver.ts`) and asserts the new + * implementation agrees with it on a deterministic generated corpus plus a + * hand-built layout per tie-break. The copy is the specification; if an arm here + * fails, the resolver's OUTPUT moved and COBOL's IMPORTS edges move with it. + * + * Mutation-tested against the new implementation — each of these was inserted, + * confirmed RED here, and reverted: tiers collapsed into one map; within-tier + * first-wins flipped to last-wins; `targetRaw.toUpperCase()` dropped; + * `path.extname(fp).toLowerCase()` left un-lowercased. + * + * This file calls the resolver directly, which for COBOL is also the + * orchestrator adapter — but the arms below say nothing about the Set being + * passed THROUGH, and a defensive `new Set(allFilePaths)` would leave them all + * green while restoring the per-import rebuild. That failure is guarded by + * `test/integration/cobol-import-index-reuse.test.ts`. + */ +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; + +import { cobolScopeResolver } from '../../../src/core/ingestion/languages/cobol/scope-resolver.js'; + +const { resolveImportTarget } = cobolScopeResolver; + +/** COBOL takes no `resolutionConfig` and ignores `fromFile`; both are pinned. */ +const FROM_FILE = 'src/PROG.cbl'; + +function resolve(targetRaw: string, files: ReadonlySet): string | readonly string[] | null { + return resolveImportTarget(targetRaw, FROM_FILE, files, undefined); +} + +// ─── verbatim pre-change implementation ────────────────────────────────────── + +const LEGACY_COPYBOOK_EXTENSIONS = new Set(['.cpy', '.copybook']); + +function legacyResolveCobolImportTarget( + targetRaw: string, + allFilePaths: ReadonlySet, +): string | null { + const upper = targetRaw.toUpperCase(); + // Check copybook files first + for (const fp of allFilePaths) { + const ext = path.extname(fp).toLowerCase(); + if (!LEGACY_COPYBOOK_EXTENSIONS.has(ext)) continue; + const basename = path.basename(fp, ext).toUpperCase(); + if (basename === upper) return fp; + } + // Also search COBOL source files (.cbl, .cob, .cobol) + const COBOL_SOURCE_EXTS = new Set(['.cbl', '.cob', '.cobol']); + for (const fp of allFilePaths) { + const ext = path.extname(fp).toLowerCase(); + if (!COBOL_SOURCE_EXTS.has(ext)) continue; + const basename = path.basename(fp, ext).toUpperCase(); + if (basename === upper) return fp; + } + return null; +} + +// ─── deterministic corpus ──────────────────────────────────────────────────── + +/** Murmur3 finalizer — a reproducible stand-in for `Math.random()`. */ +function mix(n: number): number { + let x = n >>> 0; + x = Math.imul(x ^ (x >>> 16), 0x85ebca6b) >>> 0; + x = Math.imul(x ^ (x >>> 13), 0xc2b2ae35) >>> 0; + return (x ^ (x >>> 16)) >>> 0; +} + +/** + * Directory shapes of a typical mainframe checkout, including one whose + * segments are separated by BACKSLASHES — on POSIX that is one long filename, + * which is precisely the `path` semantic the index must not "simplify" away. + */ +const DIRS = [ + '', + 'copybooks', + 'COPYBOOKS', + 'src', + 'src/copy', + 'legacy/cpy', + 'jcl/proclib', + 'win\\dir', +]; + +/** Member names in the case mixture a real repo has. */ +const STEMS = ['CUSTREC', 'custrec', 'AcctRec', 'PAYROLL', 'BOOK', 'COMMON', 'TAXCALC', 'ERRDEMO']; + +/** + * Both tiers, both cases, plus two extensions in NEITHER tier: `.txt` (a + * non-COBOL file that must never answer a `COPY`) and `''` (a file with no + * extension at all, which `path.extname` reports as the empty string and which + * therefore falls out of both extension sets). + */ +const EXTS = ['.cpy', '.copybook', '.CPY', '.cbl', '.cob', '.cobol', '.CBL', '.txt', '']; + +function corpus(seed: number, fileCount: number): Set { + const files = new Set(); + for (let i = 0; i < fileCount; i++) { + const a = mix(seed * 7919 + i); + const b = mix(a ^ 0x9e3779b9); + const c = mix(b ^ 0x85ebca6b); + const dir = DIRS[a % DIRS.length]; + const stem = STEMS[b % STEMS.length]; + const rel = `${stem}${EXTS[c % EXTS.length]}`; + files.add(dir === '' ? rel : `${dir}/${rel}`); + } + // Backslash-separated paths, which `path` reads differently per platform and + // a `/`-slicing rewrite would read differently from `path` on POSIX. + files.add('win\\dir\\BOOK.cpy'); + files.add('win\\dir\\PAYROLL.cbl'); + return files; +} + +/** + * `COPY` operands as they appear in source, plus the spellings that reach the + * corpus's awkward files. Lower-case entries are what breaks if the target + * stops being upper-cased; the `.CPY`/`.CBL` suffixed entries are what reaches + * a file whose mixed-case extension `path.basename` refused to strip. + */ +const TARGETS = [ + '', + 'CUSTREC', + 'custrec', + 'CustRec', + 'ACCTREC', + 'PAYROLL', + 'payroll', + 'BOOK', + 'COMMON', + 'TAXCALC', + 'ERRDEMO', + 'MISSING', + 'BOOK.CPY', + 'CUSTREC.CPY', + 'PAYROLL.CBL', + 'win\\dir\\BOOK', + 'WIN\\DIR\\BOOK', + 'win/dir/BOOK', +]; + +const REPOS = 40; + +describe('COBOL COPY-target index hoist — output parity with the pre-change scans (#2908)', () => { + it('agrees with the verbatim pre-change resolver over the generated corpus', () => { + let checked = 0; + for (let repo = 0; repo < REPOS; repo++) { + const files = corpus(repo, 6 + (repo % 25)); + for (const target of TARGETS) { + expect(resolve(target, files), `cobol "${target}" repo=${repo}`).toEqual( + legacyResolveCobolImportTarget(target, files), + ); + checked++; + } + } + expect(checked).toBe(REPOS * TARGETS.length); + }); + + it('the corpus actually resolves things (the parity arm is not vacuous)', () => { + // A corpus that resolved nothing would make the arm above pass on + // `null === null` forever. Measured on this corpus: 390 hits. + let hits = 0; + for (let repo = 0; repo < REPOS; repo++) { + const files = corpus(repo, 6 + (repo % 25)); + for (const target of TARGETS) { + hits += legacyResolveCobolImportTarget(target, files) === null ? 0 : 1; + } + } + expect(hits).toBeGreaterThan(300); + }); +}); + +// ─── hand-built tie-breaks ─────────────────────────────────────────────────── + +/** + * `path` decides where the stem of a backslash path starts: on POSIX the whole + * `dir\sub\BOOK` is the stem, on Windows only `BOOK`. Deriving the target + * through the SAME call keeps the two arms below a hit and a miss respectively + * on both platforms, so what they pin is the `path` semantics rather than the + * host — and a rewrite that replaces `path` with slicing on `/` changes what + * they resolve to on Windows. + */ +const BACKSLASH_FILE = 'dir\\sub\\BOOK.cpy'; +const BACKSLASH_STEM = path.basename(BACKSLASH_FILE, '.cpy').toUpperCase(); +/** The stem's last backslash-delimited segment — a HIT only where `path` splits on `\`. */ +const BACKSLASH_LEAF = 'BOOK'; +const BACKSLASH_LEAF_EXPECTED = BACKSLASH_STEM === BACKSLASH_LEAF ? BACKSLASH_FILE : null; + +interface HandBuilt { + readonly why: string; + /** Insertion order IS the Set-iteration order, and for most arms it IS the tie-break. */ + readonly files: readonly string[]; + readonly target: string; + /** The one path (or `null`) both implementations must return. */ + readonly expected: string | null; +} + +const HANDBUILT: readonly HandBuilt[] = [ + { + why: 'a copybook beats a COBOL source that comes FIRST in Set order (tier order)', + files: ['src/BOOK.cbl', 'copybooks/BOOK.cpy'], + target: 'BOOK', + expected: 'copybooks/BOOK.cpy', + }, + { + why: '.copybook is tier 1 too, and beats an earlier .cob', + files: ['src/BOOK.cob', 'copybooks/BOOK.copybook'], + target: 'BOOK', + expected: 'copybooks/BOOK.copybook', + }, + { + why: 'the source tier answers only when every copybook has missed', + files: ['copybooks/OTHER.cpy', 'src/BOOK.cbl'], + target: 'BOOK', + expected: 'src/BOOK.cbl', + }, + { + why: 'within the copybook tier, first in Set order wins', + files: ['a/BOOK.cpy', 'b/BOOK.cpy'], + target: 'BOOK', + expected: 'a/BOOK.cpy', + }, + { + why: 'within the source tier, first in Set order wins', + files: ['b/BOOK.cbl', 'a/BOOK.cob', 'c/BOOK.cobol'], + target: 'BOOK', + expected: 'b/BOOK.cbl', + }, + { + why: 'the basename is compared UPPER-cased, so a lower-case file answers an upper-case COPY', + files: ['copybooks/custrec.cpy'], + target: 'CUSTREC', + expected: 'copybooks/custrec.cpy', + }, + { + why: 'the TARGET is upper-cased too, so a lower-case COPY reaches an upper-case file', + files: ['copybooks/CUSTREC.cpy'], + target: 'custrec', + expected: 'copybooks/CUSTREC.cpy', + }, + { + why: 'the extension is matched LOWER-cased, so `Foo.CPY` is a copybook at all', + files: ['copybooks/Foo.CPY'], + target: 'FOO.CPY', + expected: 'copybooks/Foo.CPY', + }, + { + why: '`path.basename(fp, ext)` strips case-SENSITIVELY, so `Foo.CPY` is NOT reachable as FOO', + files: ['copybooks/Foo.CPY'], + target: 'FOO', + expected: null, + }, + { + why: 'a `.CPY` file keyed with its suffix loses `BOOK` to a `.cbl` in the later tier', + files: ['x/BOOK.cbl', 'y/BOOK.CPY'], + target: 'BOOK', + expected: 'x/BOOK.cbl', + }, + { + why: 'an uppercase source extension is a source file (`.CBL` → tier 2, keyed with its suffix)', + files: ['src/Pay.CBL'], + target: 'PAY.CBL', + expected: 'src/Pay.CBL', + }, + { + why: 'a file with NO extension never answers a COPY', + files: ['copybooks/BOOK'], + target: 'BOOK', + expected: null, + }, + { + why: 'a non-COBOL extension never answers a COPY', + files: ['copybooks/BOOK.txt', 'docs/BOOK.md'], + target: 'BOOK', + expected: null, + }, + { + why: 'a target matching nothing resolves to null', + files: ['copybooks/BOOK.cpy', 'src/PROG.cbl'], + target: 'NOSUCHBOOK', + expected: null, + }, + { + why: 'an empty target matches nothing (no file has an empty stem)', + files: ['copybooks/BOOK.cpy', 'src/PROG.cbl'], + target: '', + expected: null, + }, + { + why: 'a backslash path is addressed by the stem `path` reports for it', + files: [BACKSLASH_FILE], + target: BACKSLASH_STEM, + expected: BACKSLASH_FILE, + }, + { + why: 'its trailing segment is a hit only where `path` treats `\\` as a separator', + files: [BACKSLASH_FILE], + target: BACKSLASH_LEAF, + expected: BACKSLASH_LEAF_EXPECTED, + }, +]; + +describe('COBOL COPY-target index hoist — hand-built tie-breaks (#2908)', () => { + it.each(HANDBUILT)('$why', ({ files, target, expected }) => { + const set = new Set(files); + // Two assertions, not one: agreeing with the legacy copy proves the hoist + // preserved the behaviour, and pinning the literal proves the behaviour + // being preserved is the one the case is named for — `toEqual(null)` on + // both sides would otherwise satisfy an arm that stopped resolving. + expect(legacyResolveCobolImportTarget(target, set), `legacy: ${target}`).toBe(expected); + expect(resolve(target, set), `new: ${target}`).toBe(expected); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/import-target-index-parity.test.ts b/gitnexus/test/unit/scope-resolution/import-target-index-parity.test.ts index 9e877a5b5..d729ca314 100644 --- a/gitnexus/test/unit/scope-resolution/import-target-index-parity.test.ts +++ b/gitnexus/test/unit/scope-resolution/import-target-index-parity.test.ts @@ -30,9 +30,7 @@ * provably cannot see: a full workspace scan on 1-in-32 imports scores 1.458 * against a 1.8 scaling budget and 1.736 ms against a 4 ms ceiling — it passes * everything — while this counter reads 14 instead of 1. Timing gates catch the - * constant factor; this catches the scan. Kotlin (#2872) is covered there too, - * because its own guard counts index BUILDS and a scan beside a reused index - * moves no build count. + * constant factor; this catches the scan. Kotlin (#2872) is covered there too. * * It is NOT the guard for PR #1918 review finding P1. That failure — a * defensive `new Set(allFilePaths)` in the orchestrator ADAPTER, handing a fresh @@ -793,9 +791,9 @@ describe('import-target index hoist — built once per file set, not once per im it('kotlin builds one index for many imports (#2872)', () => { // Kotlin's own guard (`test/integration/kotlin-import-index-reuse.test.ts`) - // counts index BUILDS. That catches the per-import rebuild, but a scan added - // beside a reused index moves no build count — this arm sees it, because it - // counts iterations of the Set rather than cache misses. + // counts the same traversals one layer up, at the adapter. This arm covers + // the resolver function directly, so a rescan reintroduced inside + // `resolveKotlinImportTarget` fails here even if the adapter is untouched. const files = countingCorpus(7, '.kt'); for (let i = 0; i < 200; i++) { resolveKotlinImportTarget( diff --git a/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts b/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts new file mode 100644 index 000000000..04acd1bb4 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts @@ -0,0 +1,650 @@ +/** + * One property, asserted for EVERY registered `ScopeResolver` (#2909). + * + * Import-target resolution must not re-derive its per-pass workspace structures + * once per import. The per-language guards matching + * `test/integration/*-import-index-reuse.test.ts` say that once each, with their + * own corpus, their own expected traversal count and their own header — and + * there is one only for the languages someone wrote one for, never for the rest, + * because adding a resolver to `SCOPE_RESOLVERS` is two lines + * (`pipeline/registry.ts`), neither of which is a test. This file closes that + * gap without restating its size: the table below is keyed by + * `SupportedLanguages`, and the inventory arm diffs its keys against + * `SCOPE_RESOLVERS` so a registered resolver missing from it fails. + * + * ## Called the way the ORCHESTRATOR calls, with all five arguments + * + * `pipeline/run.ts` passes five: `(targetRaw, fromFile, allFilePaths, + * resolutionConfig, { parsedFiles, parsedImport })`. This file used to pass + * four, and a fifth argument that is never supplied is a channel that is never + * measured — `languages/php/import-target.ts` returns early on `context === + * undefined`, so everything behind that guard was ungated for every language in + * the table. Replacing PHP's `perFileSet` with an identity wrapper, which + * rebuilds its `Map` per import at O(files × depth) + * (197.0 µs → 9976.2 µs per import at 8000 files, depth 6), left every arm of + * this file green. Both call sites below now pass a `context`, and the fixtures + * name their `parsedImport` explicitly so no adapter's use of the channel can + * hide behind an omission. + * + * ## Two counters, because there are two per-file-set KEYS + * + * `perFileSet` memoizes on object identity, and the orchestrator threads two + * stable objects per pass: the `allFilePaths` Set and the `parsedFiles` array. + * A `CountingSet` sees only the first, and the readers of the second touch the + * Set nowhere while reading it, so `scans` moves by ZERO for anything that goes + * wrong on that key — measured, with the identity-wrapper mutation above: + * `scans` 1 and 1, `parsedFileReads` 9 and 603. Hence `countedParsedFiles` + * (`test/helpers/counting-file-set.ts`), and hence the same comparison asserted + * twice, once per key. + * + * Two registered resolvers read `context` — PHP (`languages/php/scope-resolver.ts` + * → `resolvePhpImportTargetInternal`) and Python + * (`languages/python/scope-resolver.ts` → `pythonFileExportsName`). Every other + * adapter declares three or four parameters and cannot observe a fifth. Which + * ones those are is not a number to maintain here: the per-language + * `minimumParsedFileReads` floor in the table IS the record, and it is what a + * new reader has to change. Both languages that read this key memoize on it, + * and the two memos fail differently: + * + * - PHP: the `filesByDirectory` memo, `perFileSet`-keyed on the `parsedFiles` + * array. Defeat it and every import rebuilds a `Map`; the arm reads 603 against 9 (proven by mutation). + * - Python: `parsedFileByPath`, keyed the same way, behind + * `pythonFileExportsName`. It is built by the FIRST import whose package + * probe resolves — one pass over the array — and every later one is a + * `Map.get`, so the floor of 1 is that single build and the equality half + * is what proves it does not repeat. Before that memo the same call was a + * `parsedFiles.find` per resolving import, which is the shape #2901 + * removed on the file-set key; this arm is why it cannot come back. + * - Every other language: `0 === 0`, recorded as a floor of 0. A new reader + * arrives with that floor already in place and is caught by the equality + * half, which needs no per-language knowledge at all. + * + * Out of reach from here, and stated so it is not mistaken for covered: + * `bench/import-target/measure.mjs` calls the resolvers with THREE arguments, + * so no timing arm in that harness enters the `context` leg either. + * + * ## The assertion is a COMPARISON, not a constant + * + * `scans(200) === scans(2)`, never `scans === 1`. Per-language counts legitimately + * differ — C# and Java each build two indexes over the same Set, TypeScript / + * JavaScript / Vue materialize an array and a copy behind their pass cache, Rust + * never traverses at all — and a table of expected constants would be one entry + * per language to get wrong. Comparing two counts against each other + * needs no per-language knowledge and states the actual property: the traversal + * count is a function of the FILE SET, not of the import count. + * + * ## Paired with non-vacuity, because the comparison alone is trivially true + * + * `scans(200) === scans(2)` holds perfectly for a resolver that returns `null` + * without ever touching the set — which is exactly what an adapter looks like + * after its context narrowing starts rejecting the workspace (`instanceof Set` + * for C# and Java, the `has`/iterator duck-type for Swift). So each case also + * asserts: + * + * - `hitTarget` still resolves to something. This is the same pairing rule the + * `test/integration/*-import-index-reuse.test.ts` guards state in their + * headers, and the reason `CountingSet` is a real `Set` subclass rather than + * a counter object. + * - the count clears `minimumScans`, which proves the counting Set is the + * object the resolver actually indexed rather than a copy made upstream. + * + * ## What it does NOT cover + * + * The fixtures are minimal by design — a handful of files and two import + * spellings per language, enough to reach the index and no more. Output parity, + * tie-breaks and iteration order are the subject of + * `import-target-index-parity.test.ts` and the per-language parity tests; the + * per-language integration guards carry the realistic corpora and the exact + * expected traversal counts. This file only answers "does the work stay flat in + * the number of imports", for every resolver `SCOPE_RESOLVERS` registers. + * + * Neither counter sees inside a structure once it has been built: a scan over + * `WorkspaceFileIndex.normalized`, or over a `ParsedFile[]` bucket in PHP's + * directory index, moves nothing (see `test/helpers/counting-file-set.ts`). + * That is not hypothetical — JavaScript's adapter had no suffix + * index at all until #2910, so every JavaScript import ran `suffixResolve`'s + * linear pass over the materialized `normalizedFileList` (6448.9 µs per import + * at 2000 files, against 25.0 µs for TypeScript), and the `javascript` case + * below scored a clean pass throughout: the pass cache WAS reused, so the + * traversal count read 2 either way. What catches that class of defect is a + * behaviour or call-count assertion, not a traversal count — see + * `test/integration/javascript-import-index-reuse.test.ts` and + * `test/unit/scope-resolution/javascript-import-target-parity.test.ts`. + */ +import { describe, expect, it } from 'vitest'; +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ParsedImport } from 'gitnexus-shared'; + +import { SCOPE_RESOLVERS } from '../../../src/core/ingestion/scope-resolution/pipeline/registry.js'; +import type { ScopeResolver } from '../../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; +import type { ComposerConfig } from '../../../src/core/ingestion/language-config.js'; +import { + CountingSet, + countedParsedFiles, + pythonNamedImport, +} from '../../helpers/counting-file-set.js'; + +/** + * The minimum a language needs for one `resolveImportTarget` call to reach + * whatever structure it derives from the file set. + */ +interface ImportTargetFixture { + /** + * The whole synthetic workspace. Small on purpose: the count being compared + * is traversals, not their cost. + * + * Also the source of `context.parsedFiles` — one minimal `ParsedFile` per + * path, built by `countedParsedFiles`. NOT a second fixture field, because + * the orchestrator derives the path Set FROM the parsed workspace + * (`new Set(parsedFiles.map((f) => f.filePath))` in `pipeline/run.ts`), so two + * independent lists here could disagree in a way no real pass can. + */ + readonly files: readonly string[]; + /** The importing file. */ + readonly fromFile: string; + /** + * The resolver's 4th argument. Not optional: the languages that take none + * pass `undefined` in the open, so no call site hides which adapters read + * this channel behind an omission. + */ + readonly resolutionConfig: unknown; + /** + * An import that resolves to NOTHING, spelled differently on every call. + * + * A miss is the expensive case in every resolver here — it runs the cascade to + * completion instead of returning on the first hit — and the distinct spelling + * defeats the per-target `resolveCache` that TypeScript, JavaScript and Vue + * keep, so the resolution path is really re-entered per import rather than + * answered from a memo. + */ + readonly missTarget: (i: number) => string; + /** An import that MUST resolve. The non-vacuity half of the assertion. */ + readonly hitTarget: string; + /** + * The `parsedImport` half of the resolver's 5th argument, for the spelling + * being resolved. A function of the spelling, not a constant: PHP reaches its + * `parsedFiles` leg only for `kind: 'named' | 'alias'` carrying an + * `importedSymbolKind` of `function` or `const`, and Python resolves + * `parsedImport.targetRaw` in preference to the `targetRaw` argument — so one + * fixed import would resolve a single spelling 201 times and be answered from + * the per-target memo that the distinct `missTarget` spellings exist to + * defeat. + * + * `undefined` wherever the adapter ignores `context`; that is a statement + * about the resolver, made in the open, for the same reason + * `resolutionConfig` is never omitted. + */ + readonly parsedImport: (targetRaw: string) => ParsedImport | undefined; + /** + * Traversals of one file set that the property permits, as a floor. + * + * One for every language that derives an index from the set. ZERO for Rust, + * which is not an exemption: `resolveRustImportTarget` answers every leg with + * `allFilePaths.has(candidate)` membership probes and never iterates, so there + * is no traversal to hoist and nothing for the counter to see. (Rust's one + * workspace index, `buildRustModuleIndex`, is memoized in + * `qualified-call.ts::moduleIndexFor` and hangs off `resolveQualifiedFreeCall` + * — a different hook, not this one.) + */ + readonly minimumScans: number; + /** + * Element reads of one `context.parsedFiles` array that the property permits, + * as a floor — the `minimumScans` of the second key. + * + * ZERO wherever the adapter never reads `context`, and that zero is a fact + * about the adapter rather than an exemption: the equality half still holds, + * so a resolver that starts reading `parsedFiles` per import fails here with a + * floor of 0 in place. ONE for PHP and Python, which is what proves the leg + * behind `context` was entered at all — an early `return` on + * `context === undefined` posts a perfect zero otherwise, which is precisely + * how every arm of this file passed while measuring nothing on that channel. + */ + readonly minimumParsedFileReads: number; +} + +/** The `composer.json` PSR-4 map `loadPhpComposerConfig` would have produced. */ +const PHP_COMPOSER: ComposerConfig = { psr4: new Map([['App', 'app']]) }; + +/** The value `loadGoModulePath` produces for a repo with a `go.mod`. */ +const GO_MODULE = { modulePath: 'example.com/mod' }; + +/** + * The `parsedImport` of an adapter that takes three or four parameters and so + * cannot observe one. Named rather than inlined so a reader scanning the table + * sees at a glance which languages differ. + */ +const IGNORES_CONTEXT = (): undefined => undefined; + +/** + * `use function Vendor\Ghost\missing;` — the one PHP import shape that reaches + * `filesByDirectory`. A `type` import (the default for `use X;`) returns before + * the `parsedFiles` leg, so the class-style spelling the other arms use would + * leave `parsedFileReads` at 0. + */ +const PHP_FUNCTION_IMPORT = (targetRaw: string): ParsedImport => ({ + kind: 'named', + localName: 'imported', + importedName: 'imported', + targetRaw, + importedSymbolKind: 'function', +}); + +/** + * `from import Widget` — a named import, which is what makes + * `resolvePythonImportTarget` run the package-attribute probe + * (`pythonFileExportsName`, the `context.parsedFiles` reader) ahead of the + * submodule fallback. The default the adapter synthesizes when `context` is + * absent is a `namespace` import, and that shape never reaches the probe. + */ +const FIXTURES: ReadonlyMap = new Map< + SupportedLanguages, + ImportTargetFixture +>([ + [ + SupportedLanguages.Python, + { + // `realpkg/__init__.py` makes the package real, so `hasRepoCandidate` + // passes and the miss reaches `resolveAbsoluteFromFiles` — both index + // consumers, not just the gate. + files: ['pkg/sub/mod.py', 'realpkg/__init__.py', 'realpkg/widget.py', 'app/main.py'], + fromFile: 'app/main.py', + resolutionConfig: undefined, + missTarget: (i) => `realpkg.ghost${i}`, + hitTarget: 'realpkg.widget', + parsedImport: pythonNamedImport, + minimumScans: 1, + // The one build of `parsedFileByPath`, triggered by the single import + // whose package probe resolves — the misses never get that far, and + // every later resolver is a `Map.get` rather than another pass. + minimumParsedFileReads: 1, + }, + ], + [ + SupportedLanguages.CSharp, + { + // No `.csproj` in the config, which is the leg that reads both the shared + // workspace index and the namespace-directory index. + files: ['App/Models/User.cs', 'App/Services/Service.cs', 'App/Program.cs'], + fromFile: 'App/Program.cs', + resolutionConfig: undefined, + missTarget: (i) => `Vendor${i}.Ghost.Deep.Missing`, + hitTarget: 'App.Models.User', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.TypeScript, + { + files: ['src/util.ts', 'src/models/user.ts', 'src/main.ts'], + fromFile: 'src/main.ts', + resolutionConfig: undefined, + missTarget: (i) => `./ghost${i}`, + hitTarget: './util', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.Go, + { + files: ['internal/models/user.go', 'internal/models/user_test.go', 'main.go'], + fromFile: 'main.go', + resolutionConfig: GO_MODULE, + // Third-party: misses the module leg and runs the whole GOPATH suffix + // cascade, which used to cost one full scan per path segment. + missTarget: (i) => `github.com/vendor/dep${i}/sub`, + hitTarget: 'example.com/mod/internal/models', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.Java, + { + files: ['com/example/model/User.java', 'src/main/java/com/example/App.java'], + fromFile: 'src/main/java/com/example/App.java', + resolutionConfig: undefined, + // Four segments and no hit: the progressive-stripping loop runs to the + // end, which is what every JDK and third-party import does. + missTarget: (i) => `vendor${i}.ghost.deep.Missing`, + hitTarget: 'com.example.model.User', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.C, + { + // `resolutionConfig` is the header set from `loadResolutionConfig`. Left + // undefined so the resolver indexes THIS set: with headers present the + // adapter hands the resolver a memoized union instead, and the union's + // own scan is the only one this counter would see. + files: ['include/util.h', 'src/helper.h', 'src/main.c'], + fromFile: 'src/main.c', + resolutionConfig: undefined, + missTarget: (i) => `ghost${i}.h`, + hitTarget: 'util.h', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.CPlusPlus, + { + // Same accounting as C: `resolveCppImportTarget` delegates to the C + // resolver's basename index, keyed on the same Set. + files: ['include/util.hpp', 'src/helper.hpp', 'src/main.cpp'], + fromFile: 'src/main.cpp', + resolutionConfig: undefined, + missTarget: (i) => `ghost${i}.hpp`, + hitTarget: 'util.hpp', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.PHP, + { + files: ['app/Models/User.php', 'lib/Legacy/Helper.php', 'app/Main.php'], + fromFile: 'app/Main.php', + resolutionConfig: PHP_COMPOSER, + // Deliberately matches NO PSR-4 prefix. `resolvePhpImportInternal` runs + // its namespace-directory fallback scan unconditionally when + // `getFilesInDir` comes back empty, so a miss UNDER `App\` — say + // `App\Legacy\Ghost`, whose directory does not exist — costs one + // traversal per import: swapping this fixture onto that spelling posts + // 201 traversals for 200 imports against 3 for two (measured). The + // residual is real and is out of this file's reach — it lives in + // `import-resolvers/php.ts`, which the #2901 hoist does not touch — so it + // is pinned by name in `php-import-target-parity.test.ts` and this + // fixture takes the leg that IS indexed rather than restating it. + missTarget: (i) => `Vendor${i}\\Ghost\\Missing`, + hitTarget: 'App\\Models\\User', + parsedImport: PHP_FUNCTION_IMPORT, + minimumScans: 1, + // `filesByDirectory`'s one pass over the parsed workspace, memoized on it. + minimumParsedFileReads: 1, + }, + ], + [ + SupportedLanguages.Rust, + { + files: ['src/lib.rs', 'src/models.rs', 'src/main.rs'], + fromFile: 'src/main.rs', + resolutionConfig: undefined, + missTarget: (i) => `ghost${i}::deep::Missing`, + hitTarget: 'crate::models', + parsedImport: IGNORES_CONTEXT, + // See `minimumScans` on the interface: membership probes only. + minimumScans: 0, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.JavaScript, + { + files: ['src/util.js', 'src/models/user.js', 'src/main.js'], + fromFile: 'src/main.js', + resolutionConfig: undefined, + missTarget: (i) => `./ghost${i}`, + hitTarget: './util', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.Kotlin, + { + files: [ + 'lib/src/main/kotlin/com/example/widget/Widget.kt', + 'common/src/main/kotlin/com/example/common/Util.kt', + ], + fromFile: 'common/src/main/kotlin/com/example/common/Util.kt', + resolutionConfig: undefined, + missTarget: (i) => `org.ghost${i}.deep.Missing`, + // Under a module source root, so this resolves by path suffix rather than + // by a workspace-rooted exact match. + hitTarget: 'com.example.widget.Widget', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.Ruby, + { + files: ['lib/app/models/user.rb', 'lib/util.rb', 'lib/main.rb'], + fromFile: 'lib/main.rb', + resolutionConfig: undefined, + // A bare `require`, not a `require_relative`: the relative leg answers + // from `Set.has` and never reaches the index. + missTarget: (i) => `gem${i}/missing`, + hitTarget: 'app/models/user', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.Cobol, + { + files: ['copybooks/CUSTREC.cpy', 'src/PAYROLL.cbl', 'src/PROG.cbl'], + fromFile: 'src/PROG.cbl', + resolutionConfig: undefined, + // Vendor and system copybooks live outside the repo, so the common case + // misses both tiers — two full scans per `COPY` before the index. + missTarget: (i) => `VENDOR${i}`, + hitTarget: 'CUSTREC', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.Swift, + { + files: ['Sources/Models/User.swift', 'Sources/App/main.swift'], + fromFile: 'Sources/App/main.swift', + resolutionConfig: undefined, + missTarget: (i) => `Ghost${i}`, + hitTarget: 'Models', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.Dart, + { + files: ['lib/models.dart', 'tool/generate.dart', 'lib/main.dart'], + fromFile: 'lib/main.dart', + resolutionConfig: undefined, + // An external package: both `lib/` and bare `` miss, which is + // the two-scan case. + missTarget: (i) => `package:vendor${i}/ghost.dart`, + hitTarget: 'package:app/models.dart', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], + [ + SupportedLanguages.Vue, + { + files: ['src/components/Widget.vue', 'src/util.ts', 'src/App.vue'], + fromFile: 'src/App.vue', + resolutionConfig: undefined, + missTarget: (i) => `./ghost${i}.vue`, + hitTarget: './components/Widget.vue', + parsedImport: IGNORES_CONTEXT, + minimumScans: 1, + minimumParsedFileReads: 0, + }, + ], +]); + +/** + * Registered resolvers exempted from the property, each with the open issue + * that will remove the exemption. + * + * EMPTY, and that is the result rather than the starting state: every resolver + * in `SCOPE_RESOLVERS` either memoizes its index on the `allFilePaths` Set + * identity or never traverses the Set at all (#2872, #2877, #2878, #2879, #2880, + * #2901, #2902, #2908 closed the last of them). The map stays because the + * mechanism is the point — the next language must not be able to opt out of the + * property by quietly not appearing in `FIXTURES`. An entry here must + * cite an open issue (`#NNNN`); the arm below enforces the citation, and the + * pinned empty key list means adding one is a visible, reviewed edit rather + * than a line in a table nobody reads. + */ +const KNOWN_UNINDEXED: ReadonlyMap = new Map< + SupportedLanguages, + string +>(); + +interface ContractCase { + readonly language: SupportedLanguages; + readonly resolver: ScopeResolver; + readonly fixture: ImportTargetFixture; +} + +const CASES: readonly ContractCase[] = [...SCOPE_RESOLVERS.entries()].flatMap( + ([language, resolver]) => { + const fixture = FIXTURES.get(language); + return fixture === undefined ? [] : [{ language, resolver, fixture }]; + }, +); + +/** Imports driven in the baseline run — the smallest count above one. */ +const BASELINE_IMPORTS = 2; +/** Imports driven in the comparison run. A per-import scan shows up as a 100x. */ +const MANY_IMPORTS = 200; + +interface ImportRun { + /** Full traversals of the run's own file set. */ + readonly scans: number; + /** Element reads of the run's own `context.parsedFiles` array. */ + readonly parsedFileReads: number; + /** What `hitTarget` resolved to, read after the misses. */ + readonly hit: string | readonly string[] | null; +} + +/** + * Drive `importCount` missing imports and then one resolvable import through + * the orchestrator ADAPTER — `ScopeResolver.resolveImportTarget`, the + * surface a defensive `new Set(allFilePaths)` copy breaks and the per-language + * unit parity tests never cross. + * + * Five arguments, the shape `pipeline/run.ts` uses. One `context` object for + * the whole run, because that is what the orchestrator threads: it builds + * `parsedFiles` once per pass, so the array identity PHP's `filesByDirectory` + * memoizes on is stable across every import. Rebuilding it here would hand each + * import a fresh key and turn the fixture itself into the defect. + * + * Fresh instruments per run, for the same reason on both keys: the indexes hang + * off object identity, so two runs sharing a Set or a `parsedFiles` array would + * have the second read the first's index and report zero. + */ +function driveImports( + resolver: ScopeResolver, + fixture: ImportTargetFixture, + importCount: number, +): ImportRun { + const files = new CountingSet(fixture.files); + const workspace = countedParsedFiles(fixture.files); + const contextFor = (targetRaw: string) => ({ + parsedFiles: workspace.parsedFiles, + parsedImport: fixture.parsedImport(targetRaw), + }); + + for (let i = 0; i < importCount; i++) { + const target = fixture.missTarget(i); + resolver.resolveImportTarget( + target, + fixture.fromFile, + files, + fixture.resolutionConfig, + contextFor(target), + ); + } + + const hit = resolver.resolveImportTarget( + fixture.hitTarget, + fixture.fromFile, + files, + fixture.resolutionConfig, + contextFor(fixture.hitTarget), + ); + return { scans: files.scans, parsedFileReads: workspace.reads(), hit }; +} + +describe('import-target index reuse — the contract every registered resolver holds', () => { + it.each(CASES)( + '$language traverses the file set no more times for many imports than for two', + ({ language, resolver, fixture }) => { + const few = driveImports(resolver, fixture, BASELINE_IMPORTS); + const many = driveImports(resolver, fixture, MANY_IMPORTS); + + // The property. A per-import scan makes `many` ~100x `few`; a scan + // reintroduced beside a reused index moves both by the same constant and + // is caught instead by the per-language guards' exact counts. + expect( + many.scans, + `${language}: ${MANY_IMPORTS} imports cost ${many.scans} traversals, ${BASELINE_IMPORTS} cost ${few.scans} — the file set is being re-read per import`, + ).toBe(few.scans); + + // The same property on the other per-file-set key. PHP's + // `filesByDirectory` and Python's `pythonFileExportsName` read + // `context.parsedFiles` and never touch the Set, so the arm above is + // blind to both — measured, not assumed: defeating PHP's `perFileSet` + // leaves `scans` unmoved and takes this count from 9 to 603. + expect( + many.parsedFileReads, + `${language}: ${MANY_IMPORTS} imports read context.parsedFiles ${many.parsedFileReads} times, ${BASELINE_IMPORTS} read it ${few.parsedFileReads} — the parsed workspace is being re-derived per import`, + ).toBe(few.parsedFileReads); + + // Non-vacuity, one arm per thing the counts could be measuring nothing + // about. Without them a resolver that resolves nothing, or a leg that is + // never entered, posts a perfect score. + expect( + many.scans, + `${language}: the counting file set was never reached — is the adapter copying it?`, + ).toBeGreaterThanOrEqual(fixture.minimumScans); + expect( + many.parsedFileReads, + `${language}: context.parsedFiles was never read — did the leg behind it stop being entered?`, + ).toBeGreaterThanOrEqual(fixture.minimumParsedFileReads); + expect( + many.hit, + `${language}: '${fixture.hitTarget}' no longer resolves, so the counts above measure nothing`, + ).not.toBeNull(); + }, + ); + + it('covers every registered scope resolver', () => { + const registered = [...SCOPE_RESOLVERS.keys()].sort(); + const accountedFor = [...FIXTURES.keys(), ...KNOWN_UNINDEXED.keys()].sort(); + + // A new language in `pipeline/registry.ts` lands here first: it is either + // given a fixture in `FIXTURES` or an entry in `KNOWN_UNINDEXED`, and both + // are edits someone has to justify. + expect(accountedFor).toEqual(registered); + }); + + it('exempts nothing, and would make an exemption cite an issue', () => { + for (const [language, reason] of KNOWN_UNINDEXED) { + expect(reason, `${language}'s exemption must cite an open issue`).toMatch(/#\d+/); + } + + expect([...KNOWN_UNINDEXED.keys()]).toEqual([]); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/java-import-target-parity.test.ts b/gitnexus/test/unit/scope-resolution/java-import-target-parity.test.ts new file mode 100644 index 000000000..739f245d7 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/java-import-target-parity.test.ts @@ -0,0 +1,634 @@ +/** + * Differential harness for the Java import-target index hoist (#2908). + * + * `resolveJavaImportTarget` answered its three-tier cascade with a full + * `allFilePaths` scan, and ran that scan AGAIN inside the progressive + * prefix-stripping loop — once per stripped segment. Replacing those scans with + * the per-file-set indexes (`getWorkspaceFileIndex` + + * `buildPackageDirIndex`/`firstFileDirectlyInPkgDir`) is a pure performance + * change ONLY if every implicit tie-break survives, and those tie-breaks are + * expressed through Set-iteration order and `indexOf` positions rather than + * through anything the type system or the existing Java tests can see: + * + * - the first pass `break`s on an exact whole-path hit, so an exact match wins + * over a suffix OR directory-child match found EARLIER in iteration order; + * - the stripping loop instead returns mid-scan at the first hit of + * `f === tailFile || f.endsWith('/' + tailFile)` — no exact-wins rule there + * — while its directory child is collected and returned only after the scan + * completes, so file/suffix beats directory child within one `skip` level + * regardless of order; + * - the directory-child leg takes the FIRST `'/' + pathLike + '/'` occurrence, + * so `com/example/com/example/Deep.java` does NOT answer `com.example`; + * - a wildcard import drops its trailing `.*` before any of that runs; + * - paths are compared normalized (`\` → `/`) but returned RAW. + * + * So this file keeps a VERBATIM copy of the pre-change implementation — the + * `resolveJavaImportTarget` that shipped before #2908, scans and all — and + * asserts the new one agrees with it, both on hand-built corpora built to force + * exactly those cases and on a generated corpus replayed under three insertion + * orders — order being the only channel most of these tie-breaks travel on. + * The copy is the specification; if a future change makes an arm here fail, the + * resolver's OUTPUT moved and Java's IMPORTS edges move with it. + * + * The hand-built arm additionally pins ABSOLUTE expectations. A pure + * differential goes green when old and new agree on `null` everywhere, which is + * also what a corpus that has quietly stopped matching anything looks like. + * + * The last arm counts how often the file Set is iterated, as the deterministic + * guard against a scan reintroduced BESIDE the reused index. It is not the + * guard for a defensive `new Set(allFilePaths)` copy in the orchestrator + * ADAPTER — that lives one layer above every call here, and is guarded by + * `test/integration/java-import-index-reuse.test.ts`. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedImport, WorkspaceIndex } from 'gitnexus-shared'; + +import { resolveJavaImportTarget } from '../../../src/core/ingestion/languages/java/import-target.js'; +import { CountingSet } from '../../helpers/counting-file-set.js'; + +// ─── verbatim pre-change implementation ────────────────────────────────────── + +interface LegacyJavaResolveContext { + readonly fromFile: string; + readonly allFilePaths: ReadonlySet; +} + +function legacyResolveJavaImportTarget( + parsedImport: ParsedImport, + workspaceIndex: WorkspaceIndex, +): string | null { + const ctx = workspaceIndex as LegacyJavaResolveContext | undefined; + if ( + ctx === undefined || + typeof (ctx as { fromFile?: unknown }).fromFile !== 'string' || + !((ctx as { allFilePaths?: unknown }).allFilePaths instanceof Set) + ) { + return null; + } + if (parsedImport.kind === 'dynamic-unresolved') return null; + if (parsedImport.targetRaw === null || parsedImport.targetRaw === '') return null; + + // Strip trailing `.*` for wildcard imports: `com.example.*` → `com.example` + let target = parsedImport.targetRaw; + if (target.endsWith('.*')) { + target = target.slice(0, -2); + } + + // Package path: `com.example.User` → `com/example/User` + const pathLike = target.replace(/\./g, '/'); + const suffix = `/${pathLike}`; + + let exactFile: string | null = null; + let suffixFile: string | null = null; + let directoryChild: string | null = null; + const dirPrefix = `${pathLike}/`; + const suffixDirPrefix = `/${dirPrefix}`; + + for (const raw of ctx.allFilePaths) { + const f = raw.replace(/\\/g, '/'); + if (!f.endsWith('.java')) continue; + if (f === `${pathLike}.java`) { + exactFile = raw; + break; + } + if (suffixFile === null && f.endsWith(`${suffix}.java`)) { + suffixFile = raw; + } + if (directoryChild === null) { + const atRoot = f.startsWith(dirPrefix); + const atNested = f.includes(suffixDirPrefix); + if (atRoot || atNested) { + const idx = atRoot ? 0 : f.indexOf(suffixDirPrefix) + 1; + const after = f.slice(idx + dirPrefix.length); + if (after.length > 0 && !after.includes('/')) { + directoryChild = raw; + } + } + } + } + + if (exactFile !== null) return exactFile; + if (suffixFile !== null) return suffixFile; + if (directoryChild !== null) return directoryChild; + + // Progressive prefix stripping — handles `import com.example.User;` + // in a repo laid out `User.java` (no `com/example/` prefix). + const segments = pathLike.split('/').filter(Boolean); + for (let skip = 1; skip < segments.length; skip++) { + const tail = segments.slice(skip).join('/'); + if (tail === '') continue; + const tailFile = `${tail}.java`; + const tailSuffix = `/${tailFile}`; + const tailDir = `${tail}/`; + const tailSuffixDir = `/${tailDir}`; + let tailDirectChild: string | null = null; + for (const raw of ctx.allFilePaths) { + const f = raw.replace(/\\/g, '/'); + if (!f.endsWith('.java')) continue; + if (f === tailFile) return raw; + if (f.endsWith(tailSuffix)) return raw; + if (tailDirectChild === null) { + const atRoot = f.startsWith(tailDir); + const atNested = f.includes(tailSuffixDir); + if (atRoot || atNested) { + const idx = atRoot ? 0 : f.indexOf(tailSuffixDir) + 1; + const after = f.slice(idx + tailDir.length); + if (after.length > 0 && !after.includes('/')) tailDirectChild = raw; + } + } + } + if (tailDirectChild !== null) return tailDirectChild; + } + + return null; +} + +// ─── harness ───────────────────────────────────────────────────────────────── + +const FROM_FILE = 'src/main/java/com/example/App.java'; + +function javaImport(targetRaw: string): ParsedImport { + return { kind: 'named', localName: '_', importedName: '_', targetRaw }; +} + +/** A file layout plus the import spelling resolved against it. */ +interface Case { + readonly label: string; + readonly files: readonly string[]; + readonly target: string; +} + +/** + * `label => result`, so a divergence names the case instead of printing an + * index into two long arrays. `null` is spelled, not dropped: "resolved + * nothing" is a real answer and must be diffed like any other. + */ +function runAll( + cases: readonly Case[], + resolve: (i: ParsedImport, w: WorkspaceIndex) => string | null, +): string[] { + return cases.map((c) => { + const ws = { fromFile: FROM_FILE, allFilePaths: new Set(c.files) }; + return `${c.label} => ${resolve(javaImport(c.target), ws) ?? 'null'}`; + }); +} + +// ─── hand-built tie-break corpora ──────────────────────────────────────────── + +/** + * One corpus per tie-break, each as small as the rule it pins. The expectation + * strings are the pre-change behaviour, derived by hand from the scan and + * confirmed against the verbatim copy by the first `it` below. + */ +const HAND_CASES: readonly Case[] = [ + { + // Tier 1: the scan `break`s on the exact hit, so it wins over the suffix + // match already found at position 0. `index.get` alone returns position 0. + label: 'exact-beats-earlier-suffix', + files: ['src/main/java/com/example/model/User.java', 'com/example/model/User.java'], + target: 'com.example.model.User', + }, + { + // Same rule against the directory-child leg. + label: 'exact-beats-earlier-directory-child', + files: ['com/example/util/Helper/Inner.java', 'com/example/util/Helper.java'], + target: 'com.example.util.Helper', + }, + { + label: 'suffix-beats-earlier-directory-child', + files: ['com/example/util/Helper/Inner.java', 'src/com/example/util/Helper.java'], + target: 'com.example.util.Helper', + }, + { + label: 'directory-child-is-first-in-set-order', + files: ['com/example/service/Beta.java', 'com/example/service/Alpha.java'], + target: 'com.example.service', + }, + { + label: 'directory-child-order-follows-insertion', + files: ['com/example/service/Alpha.java', 'com/example/service/Beta.java'], + target: 'com.example.service', + }, + { + // Tie-break 3: `.*` is stripped, so this is the package-directory query. + label: 'wildcard-resolves-as-package-directory', + files: ['com/example/service/Beta.java', 'com/example/service/Alpha.java'], + target: 'com.example.service.*', + }, + { + // A file named like the package beats that package's directory child. + label: 'wildcard-exact-file-beats-directory', + files: ['com/example/service/Alpha.java', 'com/example/service.java'], + target: 'com.example.service.*', + }, + { + // Tie-break 5: the FIRST `/com/example/` occurrence leaves `com/example/ + // Deep.java` after it, which still contains a slash — so no match. + label: 'self-nested-directory-does-not-match-outer', + files: ['com/example/com/example/Deep.java'], + target: 'com.example', + }, + { + label: 'self-nested-directory-matches-full-path', + files: ['com/example/com/example/Deep.java'], + target: 'com.example.com.example', + }, + { + // Tie-break 2: the directory child is seen first, the suffix hit second, + // and the suffix hit still wins because the scan returns mid-loop. + label: 'stripping-suffix-beats-earlier-directory-child', + files: ['x/models/Order/Part.java', 'y/models/Order.java'], + target: 'com.shop.models.Order', + }, + { + label: 'stripping-reaches-root-file', + files: ['Order.java'], + target: 'com.shop.Order', + }, + { + // The mirror of tie-break 1: inside the stripping loop the scan returns at + // the first hit of `f === tailFile || f.endsWith('/' + tailFile)`, so the + // suffix hit at position 0 beats the whole-path file behind it. Applying + // tier 1's exact-wins rule here would answer `Order.java`. + label: 'stripping-takes-the-first-hit-not-the-whole-path-one', + files: ['a/Order.java', 'Order.java'], + target: 'com.shop.Order', + }, + { + label: 'stripping-reaches-directory-child', + files: ['proj/models/Thing.java'], + target: 'com.shop.models', + }, + { + // Tie-break 4: matched on the normalized path, returned RAW. + label: 'backslash-paths-normalize-and-return-raw', + files: ['win\\src\\com\\example\\win\\Windows.java'], + target: 'com.example.win.Windows', + }, + { + label: 'duplicate-normalized-path-keeps-first-raw-spelling', + files: ['a/b/Dup.java', 'a\\b\\Dup.java'], + target: 'a.b.Dup', + }, + { + label: 'duplicate-normalized-path-keeps-first-raw-spelling-reversed', + files: ['a\\b\\Dup.java', 'a/b/Dup.java'], + target: 'a.b.Dup', + }, + { + // Tie-break 4: the `.java` filter, on both the file and the directory legs. + label: 'non-java-sibling-is-skipped', + files: ['com/example/model/User.kt', 'com/example/model/User.java'], + target: 'com.example.model.User', + }, + { + label: 'directory-of-non-java-files-is-not-a-package-directory', + files: ['com/example/onlytext/notes.txt', 'com/example/onlytext/README.md'], + target: 'com.example.onlytext', + }, + { + label: 'several-directories-share-a-last-segment', + files: ['svc-b/shared/BShared.java', 'svc-a/shared/AShared.java'], + target: 'shared', + }, + { + label: 'several-directories-share-a-last-segment-reversed', + files: ['svc-a/shared/AShared.java', 'svc-b/shared/BShared.java'], + target: 'shared', + }, + { + label: 'root-file-exact-match', + files: ['src/Loose.java', 'Loose.java'], + target: 'Loose', + }, + { + label: 'single-segment-target-has-no-stripping-pass', + files: ['deep/pkg/Loose.java'], + target: 'Loose', + }, + { + // `.*` alone strips to the empty package path — the degenerate query the + // directory index answers through its empty-last-segment bucket. + label: 'bare-wildcard-over-absolute-paths', + files: ['/abs/Root.java', '/Top.java'], + target: '.*', + }, + { + label: 'bare-wildcard-over-relative-paths', + files: ['pkg/Root.java', 'Top.java'], + target: '.*', + }, + { + label: 'empty-segments-are-not-collapsed-in-the-first-pass', + files: ['com/example/User.java'], + target: 'com..example.User', + }, + { + // Every tier is case-sensitive, so the package path never matches — but the + // stripping loop reaches the bare basename, which does. + label: 'case-mismatch-falls-through-to-basename-stripping', + files: ['com/Example/Model/Cased.java'], + target: 'com.example.model.Cased', + }, + { + label: 'directory-named-like-a-java-file', + files: ['com/example/weird.java/Inside.java'], + target: 'com.example.weird', + }, + { + // Java has no in-repo-namespace gate (C#'s #1881), so a JDK import whose + // tail happens to exist locally resolves to it. Pinned, not endorsed. + label: 'jdk-import-strips-into-a-local-lookalike', + files: ['com/example/model/User.java', 'src/main/java/util/List.java'], + target: 'java.util.List', + }, + { + label: 'jdk-import-with-no-lookalike-resolves-to-nothing', + files: ['com/example/model/User.java'], + target: 'java.util.List', + }, +]; + +/** Absolute pre-change behaviour, so the differential cannot pass vacuously. */ +const HAND_EXPECTED: readonly string[] = [ + 'exact-beats-earlier-suffix => com/example/model/User.java', + 'exact-beats-earlier-directory-child => com/example/util/Helper.java', + 'suffix-beats-earlier-directory-child => src/com/example/util/Helper.java', + 'directory-child-is-first-in-set-order => com/example/service/Beta.java', + 'directory-child-order-follows-insertion => com/example/service/Alpha.java', + 'wildcard-resolves-as-package-directory => com/example/service/Beta.java', + 'wildcard-exact-file-beats-directory => com/example/service.java', + 'self-nested-directory-does-not-match-outer => null', + 'self-nested-directory-matches-full-path => com/example/com/example/Deep.java', + 'stripping-suffix-beats-earlier-directory-child => y/models/Order.java', + 'stripping-reaches-root-file => Order.java', + 'stripping-takes-the-first-hit-not-the-whole-path-one => a/Order.java', + 'stripping-reaches-directory-child => proj/models/Thing.java', + 'backslash-paths-normalize-and-return-raw => win\\src\\com\\example\\win\\Windows.java', + 'duplicate-normalized-path-keeps-first-raw-spelling => a/b/Dup.java', + 'duplicate-normalized-path-keeps-first-raw-spelling-reversed => a\\b\\Dup.java', + 'non-java-sibling-is-skipped => com/example/model/User.java', + 'directory-of-non-java-files-is-not-a-package-directory => null', + 'several-directories-share-a-last-segment => svc-b/shared/BShared.java', + 'several-directories-share-a-last-segment-reversed => svc-a/shared/AShared.java', + 'root-file-exact-match => Loose.java', + 'single-segment-target-has-no-stripping-pass => deep/pkg/Loose.java', + 'bare-wildcard-over-absolute-paths => /Top.java', + // A relative root file has no leading slash, so the empty package path finds + // nothing — unlike the absolute case above. + 'bare-wildcard-over-relative-paths => null', + // `filter(Boolean)` drops the empty segment, so stripping recovers the file + // the first pass could not see. + 'empty-segments-are-not-collapsed-in-the-first-pass => com/example/User.java', + 'case-mismatch-falls-through-to-basename-stripping => com/Example/Model/Cased.java', + 'directory-named-like-a-java-file => null', + 'jdk-import-strips-into-a-local-lookalike => src/main/java/util/List.java', + 'jdk-import-with-no-lookalike-resolves-to-nothing => null', +]; + +// ─── generated corpus ──────────────────────────────────────────────────────── + +const SOURCE_ROOTS = ['src/main/java', 'src/test/java', '', 'legacy', 'modules/core/src/main/java']; +const PACKAGE_DIRS = [ + 'com/example/model', + 'com/example/service', + 'com/example/util', + 'org/acme/api', + 'io/gn/core', +]; +const TYPE_NAMES = ['User', 'Order', 'Helper', 'Client', 'Registry']; + +/** + * A layout where the same package path exists under several source roots AND + * root-relative, so most lookups have a whole-path candidate and one or more + * earlier suffix candidates — the collision tier 1 turns on. The tail adds the + * shapes a regular layout never produces: self-nested packages, directories + * sharing a last segment, non-`.java` neighbours, root files, backslash paths, + * and the stripping-only targets. + */ +function generatedFiles(): string[] { + const files: string[] = []; + for (const pkg of PACKAGE_DIRS) { + for (const type of TYPE_NAMES) { + for (const root of SOURCE_ROOTS) { + files.push(root === '' ? `${pkg}/${type}.java` : `${root}/${pkg}/${type}.java`); + } + } + } + // A file whose whole path IS a package directory used elsewhere. + files.push('com/example/service.java'); + files.push('src/main/java/com/example/model.java'); + // Packages nested inside themselves. + files.push('com/example/model/com/example/model/Nested.java'); + files.push('legacy/io/gn/core/io/gn/core/Legacy.java'); + // Directories sharing a last segment across trees. + for (let i = 0; i < 4; i++) { + files.push(`svc${i}/shared/Shared${i}.java`); + files.push(`svc${i}/shared/internal/Deep${i}.java`); + } + // Non-`.java` neighbours, including a directory with none of them accepted. + files.push('com/example/model/User.kt'); + files.push('com/example/model/package-info.txt'); + files.push('com/example/resources/application.yaml'); + files.push('com/example/weird.java/Inside.java'); + // Root files and a deep chain. + files.push('Loose.java'); + files.push('Order.java'); + files.push('a/b/c/d/e/f/Deep6.java'); + // Backslash spellings, one of them a duplicate of a forward-slash entry. + files.push('win\\src\\main\\java\\com\\example\\win\\WinUser.java'); + files.push('a/b/Dup.java'); + files.push('a\\b\\Dup.java'); + // Reachable only after progressive prefix stripping, with a directory child + // planted ahead of the suffix hit at the same `skip` level. + files.push('x/models/Order/Part.java'); + files.push('bare/models/Order.java'); + files.push('bare/models/Invoice.java'); + return files; +} + +function generatedTargets(): string[] { + const targets: string[] = []; + for (const pkg of PACKAGE_DIRS) { + const dotted = pkg.replace(/\//g, '.'); + targets.push(dotted); + targets.push(`${dotted}.*`); + for (const type of TYPE_NAMES) targets.push(`${dotted}.${type}`); + } + targets.push( + // Package prefixes: partial paths that are directories but not packages. + 'com', + 'com.example', + 'com.*', + 'org', + 'org.acme', + 'io', + 'io.gn', + 'src.main.java.com.example.model.User', + 'legacy.com.example.util.Helper', + 'modules.core.src.main.java.io.gn.core.Client', + // Self-nesting. + 'com.example.model.com.example.model', + 'com.example.model.com.example.model.Nested', + 'io.gn.core.io.gn.core.Legacy', + // Shared last segments. + 'shared', + 'shared.*', + 'svc0.shared', + 'svc2.shared.internal', + 'internal', + // Stripping-only. + 'com.shop.models.Order', + 'com.shop.models', + 'com.shop.models.*', + 'whatever.bare.models.Invoice', + 'nowhere.Loose', + 'nowhere.deeply.nested.Order', + // Non-`.java` and odd shapes. + 'com.example.resources', + 'com.example.weird', + 'com.example.model.User.kt', + 'a.b.Dup', + 'a.b.c.d.e.f.Deep6', + 'com.example.win.WinUser', + '.*', + '*', + 'com..example.User', + 'Loose', + 'Order', + // Unresolvable: JDK and third-party, the majority case in real source. + 'java.util.List', + 'java.util.*', + 'java.io.File', + 'javax.annotation.Nullable', + 'org.junit.jupiter.api.Test', + 'org.springframework.boot.SpringApplication', + 'com.google.common.collect.ImmutableList', + 'com.example.missing.Absent', + ); + return targets; +} + +/** + * Three insertion orders over the same paths. Set-iteration order IS the + * tie-break channel for every "first match wins" rule here, so replaying the + * same targets under a reversal and a rotation exercises each collision from + * both sides — the as-built order alone would leave half of them one-sided. + */ +function orderedCorpora(): ReadonlyMap { + const base = generatedFiles(); + const reversed = [...base].reverse(); + const rotation = 7; + const rotated = [...base.slice(rotation), ...base.slice(0, rotation)]; + return new Map([ + ['as-built', base], + ['reversed', reversed], + ['rotated', rotated], + ]); +} + +function generatedCases(): Case[] { + const cases: Case[] = []; + for (const [order, files] of orderedCorpora()) { + for (const target of generatedTargets()) { + cases.push({ label: `${order}|${target}`, files, target }); + } + } + return cases; +} + +const GENERATED_CASES = generatedCases(); + +// ─── arms ──────────────────────────────────────────────────────────────────── + +describe('Java import target — index hoist parity (#2908)', () => { + it('reproduces the pre-change results on the hand-built tie-break corpora', () => { + expect(runAll(HAND_CASES, legacyResolveJavaImportTarget)).toEqual(HAND_EXPECTED); + expect(runAll(HAND_CASES, resolveJavaImportTarget)).toEqual(HAND_EXPECTED); + }); + + it('reproduces the pre-change results across three insertion orders', () => { + expect(runAll(GENERATED_CASES, resolveJavaImportTarget)).toEqual( + runAll(GENERATED_CASES, legacyResolveJavaImportTarget), + ); + }); + + it('the generated corpus resolves a broad set of distinct targets', () => { + const results = runAll(GENERATED_CASES, resolveJavaImportTarget); + const resolvedFiles = new Set( + results.map((r) => r.split(' => ')[1]).filter((r) => r !== 'null'), + ); + + // Non-vacuity: a differential is worthless if both sides answer `null`. + // Sized just under the current values so ordinary corpus edits do not trip + // it, while a corpus that stops matching does. + expect(results.filter((r) => !r.endsWith('=> null')).length).toBeGreaterThan(150); + expect(resolvedFiles.size).toBeGreaterThan(40); + // ...and it must keep exercising the unresolvable majority, which is the + // only case that runs the whole cascade. + expect(results.filter((r) => r.endsWith('=> null')).length).toBeGreaterThan(50); + }); + + it('matches the pre-change guards for unusable inputs', () => { + const files = new Set(['com/example/model/User.java']); + const good = { fromFile: FROM_FILE, allFilePaths: files }; + const inputs: readonly (readonly [string, ParsedImport, WorkspaceIndex])[] = [ + ['undefined context', javaImport('com.example.model.User'), undefined], + ['missing fromFile', javaImport('com.example.model.User'), { allFilePaths: files }], + [ + 'allFilePaths is not a Set', + javaImport('com.example.model.User'), + { fromFile: FROM_FILE, allFilePaths: ['com/example/model/User.java'] }, + ], + [ + 'dynamic-unresolved import', + { kind: 'dynamic-unresolved', localName: '', targetRaw: 'com.example.model.User' }, + good, + ], + // `targetRaw: null` is reachable only on `dynamic-unresolved`, which the + // kind check above already refuses, so the resolver's null branch has no + // typeable input of its own. + ['empty target', javaImport(''), good], + ['wildcard kind', { kind: 'wildcard', targetRaw: 'com.example.model.*' }, good], + ]; + + const legacy = inputs.map(([label, imp, ws]) => { + return `${label} => ${legacyResolveJavaImportTarget(imp, ws) ?? 'null'}`; + }); + const current = inputs.map(([label, imp, ws]) => { + return `${label} => ${resolveJavaImportTarget(imp, ws) ?? 'null'}`; + }); + + expect(current).toEqual(legacy); + // Every one of them refuses, except the last — a well-formed call, so the + // arm cannot pass by refusing everything. + expect(current).toEqual([ + 'undefined context => null', + 'missing fromFile => null', + 'allFilePaths is not a Set => null', + 'dynamic-unresolved import => null', + 'empty target => null', + 'wildcard kind => com/example/model/User.java', + ]); + }); + + it('builds each index once per file set rather than once per import', () => { + const files = new CountingSet(generatedFiles()); + const ws = { fromFile: FROM_FILE, allFilePaths: files }; + const targets = generatedTargets(); + + const results = targets.map((t) => resolveJavaImportTarget(javaImport(t), ws)); + + // Two traversals for the whole run: the shared workspace/suffix index and + // the package-directory index, each memoized on this Set's identity. The + // pre-change resolver traversed once per import PLUS once per stripped + // segment. + expect(files.scans).toBe(2); + // Paired result assertion — a count of 2 is equally true of a resolver that + // has stopped resolving anything at all. + expect(results.filter((r) => r !== null).length).toBeGreaterThan(20); + expect(resolveJavaImportTarget(javaImport('com.example.model.User'), ws)).toBe( + 'com/example/model/User.java', + ); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/javascript-import-target-parity.test.ts b/gitnexus/test/unit/scope-resolution/javascript-import-target-parity.test.ts new file mode 100644 index 000000000..c162654b8 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/javascript-import-target-parity.test.ts @@ -0,0 +1,710 @@ +/** + * Differential harness for the JavaScript import-target suffix index (#2910). + * + * JavaScript's `PassCache` was the TypeScript one minus its `index` field, so + * `makeJsResolveImportTarget` handed `resolveTsTarget` a context with + * `index: undefined` and every JavaScript import fell through to + * `suffixResolve`'s linear `findIndex` — one pass over `normalizedFileList` per + * path part per extension, and `EXTENSIONS` has ~39 entries. 6448.9 µs per + * import at 2000 files and 25972.6 µs at 8000, against 25.0 / 27.0 µs for + * TypeScript over the same corpus; 28.5 / 27.4 µs with the index. + * + * Adding the field is NOT a pure hoist. `suffixResolve` answers a different + * question with an index than without: + * + * - without: `filePath.endsWith('/' + s)`, so only a PROPER suffix matches; + * - with: `index.get(s) || index.getInsensitive(s)`, and `buildSuffixIndex` + * indexes `j = 0`, so WHOLE paths match too. + * + * This file holds a verbatim copy of the pre-change adapter + * (`git show HEAD:gitnexus/src/core/ingestion/languages/javascript/import-target.ts`) + * and pins exactly what that difference does. Two classes of answer move and no + * others: + * + * A. `null → repo-root file`. A path with no `/` has no proper suffix at all, + * so the scan could never reach it: `require('config')` was unresolvable + * with `config.js` sitting in the repo root. + * B. `file → different file`, always toward a MORE specific match. The scan + * skips the whole-path candidate and falls through to a shorter path + * suffix or a later extension, where it finds something else: + * `import 'app/main'` resolved to `node_modules/dep0/lib/main.js` — the + * first `/main.js` in file order — and now resolves to `app/main.js`. + * + * Measured over 211 200 old-vs-new pairs (400 generated corpora × 3 importing + * files × 176 targets) there is no third class: the index never loses a match + * the scan found, and its answer is never matched at a less specific + * (path-part, extension) position. Both of those are asserted below as + * universal properties over this corpus rather than as a count. + * + * ## Why the moved answers are JavaScript being fixed, not the index being wrong + * + * TypeScript and Vue have run the indexed path since #1918, over an identically + * built `normalizedFileList` (`allFileList.map(f => f.toLowerCase())`), through + * the same `resolveTsTarget` — and this adapter's whole stated design is "TS + * resolver, JS extensions". So the fix makes JavaScript agree with TypeScript, + * and the arm below asserts that agreement over the entire corpus rather than + * asserting it in prose. Class B's witness settles the direction: resolving + * `'app/main'` into `node_modules` was not a behaviour worth preserving. + * + * ## The scan counter, and its control + * + * The last arm counts entries into `suffixResolve`'s linear branch. That is the + * instrument this defect needed and did not have: `CountingSet` counts + * traversals of the SET, and this scan walks the materialized array behind it, + * which is exactly why the defect survived every index-reuse guard that existed + * and the contract test over `SCOPE_RESOLVERS`. The arm reads the legacy + * adapter first, so a + * count of zero is paired with a demonstration that the counter can be nonzero. + */ +import { describe, expect, it, vi } from 'vitest'; +import { SupportedLanguages } from 'gitnexus-shared'; + +import { makeJsResolveImportTarget } from '../../../src/core/ingestion/languages/javascript/import-target.js'; +import { typescriptScopeResolver } from '../../../src/core/ingestion/languages/typescript/scope-resolver.js'; +import { + resolveTsTarget, + type TsResolveContext, +} from '../../../src/core/ingestion/languages/typescript/import-target.js'; +import { EXTENSIONS } from '../../../src/core/ingestion/import-resolvers/utils.js'; + +// ─── the linear-fallback counter ───────────────────────────────────────────── +// `suffixResolve` is reached through `import-resolvers/standard.ts`, which +// imports it by a relative specifier that resolves to this same module id. +// Everything else in the module — `buildSuffixIndex`, `EXTENSIONS`, +// `tryResolveWithExtensions` — is passed straight through. + +const linearScans = { count: 0 }; + +vi.mock('../../../src/core/ingestion/import-resolvers/utils.js', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + suffixResolve: ( + pathParts: string[], + normalizedFileList: string[], + allFileList: string[], + index?: import('../../../src/core/ingestion/import-resolvers/utils.js').SuffixIndex, + ): string | null => { + linearScans.count += index === undefined ? 1 : 0; + return actual.suffixResolve(pathParts, normalizedFileList, allFileList, index); + }, + }; +}); + +// ─── verbatim pre-change implementation ────────────────────────────────────── +// Copied from `git show HEAD:gitnexus/src/core/ingestion/languages/javascript/ +// import-target.ts`. Only the names are prefixed; the body is untouched, and in +// particular the `PassCache` below still has no `index` field and the cache is +// still the single slot the WeakMap replaced. + +type LegacyJsResolveContext = TsResolveContext; + +type LegacyPassCache = { + readonly key: ReadonlySet; + readonly allFilePaths: Set; + readonly allFileList: readonly string[]; + readonly normalizedFileList: readonly string[]; + readonly resolveCache: Map; +}; + +function legacyMakeJsResolveImportTarget(): ( + targetRaw: string, + fromFile: string, + allFilePaths: ReadonlySet, + resolutionConfig?: unknown, +) => string | readonly string[] | null { + let cached: LegacyPassCache | null = null; + + return (targetRaw, fromFile, allFilePaths) => { + if (cached === null || cached.key !== allFilePaths) { + const allFileList = Array.from(allFilePaths); + cached = { + key: allFilePaths, + allFilePaths: new Set(allFilePaths), + allFileList, + normalizedFileList: allFileList.map((f) => f.toLowerCase()), + resolveCache: new Map(), + }; + } + + const ws: LegacyJsResolveContext = { + fromFile, + language: SupportedLanguages.JavaScript, + allFilePaths: cached.allFilePaths, + allFileList: cached.allFileList, + normalizedFileList: cached.normalizedFileList, + resolveCache: cached.resolveCache, + tsconfigPaths: null, + }; + return resolveTsTarget(targetRaw, ws); + }; +} + +// ─── corpus ────────────────────────────────────────────────────────────────── + +/** One differential case. `files` is emitted in the listed order, and that + * order is the tie-break under test — nothing here is random. */ +interface Case { + readonly name: string; + readonly files: readonly string[]; + readonly target: string; + readonly fromFile: string; +} + +const FROM_FILE = 'src/main.js'; + +/** + * A deterministic multi-root workspace. Every root carries the same relative + * layout, so a suffix-keyed lookup and a proper-suffix scan disagree about + * which root wins; `node_modules/dep0/lib/main.js` exists so a short suffix has + * somewhere wrong to land; `SRC/Utils/Helper0.JS` differs from + * `src/utils/helper0.js` only in case; and `config.js`, `index.js`, `mod0.js` + * sit at the repo root, where no proper suffix can reach them. + */ +function generatedFiles(): string[] { + const files: string[] = []; + for (let i = 0; i < 10; i++) { + files.push(`src/components/Widget${i}.js`); + files.push(`src/components/widget${i}.jsx`); + files.push(`vendor/pkg${i % 3}/src/utils/helper${i}.js`); + files.push(`src/utils/helper${i}.js`); + files.push(`src/utils/helper${i}/index.js`); + files.push(`lib/mod${i}.mjs`); + files.push(`lib/legacy${i}.cjs`); + files.push(`mod${i}.js`); + files.push(`node_modules/dep${i}/index.js`); + files.push(`node_modules/dep${i}/lib/main.js`); + files.push(`SRC/Utils/Helper${i}.JS`); + files.push(`packages/app${i}/src/index.js`); + } + files.push('config.js'); + files.push('index.js'); + files.push('src/index.js'); + files.push('src/main.js'); + files.push('app/main.js'); + return files; +} + +const GENERATED_FILES = generatedFiles(); + +/** + * Targets swept across `GENERATED_FILES`: relative hits and misses, + * extensionless and explicit-extension spellings, `index.js` directories, bare + * and `node_modules` specifiers, scoped packages, case-differing paths, and + * plain misses. Most of them miss, which is both the realistic shape and the + * expensive one — a miss runs the cascade to completion. + */ +function generatedTargets(): string[] { + const targets: string[] = []; + for (let i = 0; i < 10; i++) { + targets.push(`./components/Widget${i}`); + targets.push(`./components/Widget${i}.js`); + targets.push(`../src/utils/helper${i}`); + targets.push(`src/utils/helper${i}`); + targets.push(`utils/helper${i}`); + targets.push(`helper${i}`); + targets.push(`mod${i}`); + targets.push(`lib/mod${i}.mjs`); + targets.push(`lib/legacy${i}`); + targets.push(`dep${i}`); + targets.push(`dep${i}/lib/main`); + targets.push(`SRC/Utils/Helper${i}`); + targets.push(`packages/app${i}/src`); + targets.push(`@scope/pkg${i}`); + targets.push(`ghost${i}/missing`); + targets.push(`node_modules/dep${i}`); + } + targets.push('config'); + targets.push('index'); + targets.push('src'); + targets.push('src/main'); + targets.push('app/main'); + return targets; +} + +const GENERATED_CASES: readonly Case[] = generatedTargets().map((target) => ({ + name: `generated ${target}`, + files: GENERATED_FILES, + target, + fromFile: FROM_FILE, +})); + +/** Hand-built cases, one per shape the index could have moved. */ +const HAND_CASES: readonly Case[] = [ + // ── relative specifiers: resolved by exact `Set.has`, never reach the index ── + { + name: 'relative hit', + files: ['src/util.js', 'src/main.js'], + target: './util', + fromFile: FROM_FILE, + }, + { + name: 'relative hit with an explicit extension', + files: ['src/util.js', 'src/main.js'], + target: './util.js', + fromFile: FROM_FILE, + }, + { + name: 'relative parent-directory hit', + files: ['shared/util.js', 'src/main.js'], + target: '../shared/util', + fromFile: FROM_FILE, + }, + { + name: 'relative miss', + files: ['src/util.js', 'src/main.js'], + target: './missing', + fromFile: FROM_FILE, + }, + { + name: 'relative directory index.js', + files: ['src/util/index.js', 'src/main.js'], + target: './util', + fromFile: FROM_FILE, + }, + { + name: 'relative ESM specifier written as .js against a .mjs file', + files: ['src/util.mjs', 'src/main.js'], + target: './util.js', + fromFile: FROM_FILE, + }, + // ── class A: repo-root files, unreachable as a proper suffix ──────────────── + { + name: 'root-level file by bare specifier', + files: ['config.js', 'src/main.js'], + target: 'config', + fromFile: FROM_FILE, + }, + { + name: 'root index.js by bare specifier', + files: ['index.js', 'src/main.js'], + target: 'index', + fromFile: FROM_FILE, + }, + { + name: 'root-level file with an explicit extension', + files: ['config.js', 'src/main.js'], + target: 'config.js', + fromFile: FROM_FILE, + }, + { + name: 'root-level .mjs by bare specifier', + files: ['esm.mjs'], + target: 'esm', + fromFile: FROM_FILE, + }, + { + name: 'root-level .cjs by bare specifier', + files: ['legacy.cjs'], + target: 'legacy', + fromFile: FROM_FILE, + }, + { + name: 'root-level .jsx by bare specifier', + files: ['Btn.jsx'], + target: 'Btn', + fromFile: FROM_FILE, + }, + // ── class B: whole path vs proper suffix ──────────────────────────────────── + { + name: 'whole-path candidate earlier in file order than a proper-suffix one', + files: ['src/util.js', 'vendor/src/util.js'], + target: 'src/util', + fromFile: FROM_FILE, + }, + { + name: 'whole-path candidate later in file order than a proper-suffix one', + files: ['vendor/src/util.js', 'src/util.js'], + target: 'src/util', + fromFile: FROM_FILE, + }, + { + name: 'whole-path hit at a long suffix vs proper-suffix hit at a short one', + files: ['node_modules/dep/lib/main.js', 'app/main.js'], + target: 'app/main', + fromFile: FROM_FILE, + }, + { + name: 'whole-path hit at a long suffix vs proper-suffix hit at a short one, reversed', + files: ['app/main.js', 'node_modules/dep/lib/main.js'], + target: 'app/main', + fromFile: FROM_FILE, + }, + { + name: 'whole path is the only candidate, and a proper suffix of it exists', + files: ['src/util.js', 'src/main.js'], + target: 'src/util', + fromFile: FROM_FILE, + }, + { + name: 'whole-path directory index.js', + files: ['src/util/index.js', 'src/main.js'], + target: 'src/util', + fromFile: FROM_FILE, + }, + { + name: 'whole-path candidate at an earlier extension than the proper-suffix one', + files: ['x/U.js', 'U.jsx'], + target: 'U', + fromFile: FROM_FILE, + }, + { + name: 'whole-path candidate at an earlier extension than the proper-suffix one, reversed', + files: ['U.jsx', 'x/U.js'], + target: 'U', + fromFile: FROM_FILE, + }, + { + name: 'root .js outranks a nested .mjs', + files: ['lib/mod.mjs', 'mod.js'], + target: 'mod', + fromFile: FROM_FILE, + }, + // ── case-differing paths ──────────────────────────────────────────────────── + { + name: 'case-differing whole path beats a case-exact proper suffix', + files: ['SRC/Util.js', 'other/src/util.js'], + target: 'src/util', + fromFile: FROM_FILE, + }, + { + name: 'case-differing proper suffixes only', + files: ['other/SRC/Util.js', 'zz/deep/src/util.js'], + target: 'src/util', + fromFile: FROM_FILE, + }, + { + name: 'case-exact file later in order than a case-differing one', + files: ['a/FOO.js', 'b/Foo.js'], + target: 'Foo', + fromFile: FROM_FILE, + }, + { + name: 'case-exact file earlier in order than a case-differing one', + files: ['b/Foo.js', 'a/FOO.js'], + target: 'Foo', + fromFile: FROM_FILE, + }, + // ── bare / node_modules specifiers ────────────────────────────────────────── + { + name: 'node_modules package by bare specifier', + files: ['node_modules/dep/index.js', 'src/main.js'], + target: 'dep', + fromFile: FROM_FILE, + }, + { + name: 'node_modules deep path', + files: ['node_modules/dep/lib/main.js', 'src/main.js'], + target: 'dep/lib/main', + fromFile: FROM_FILE, + }, + { + name: 'scoped package with no file anywhere', + files: ['src/util.js', 'src/main.js'], + target: '@scope/pkg', + fromFile: FROM_FILE, + }, + { + name: 'dotted specifier is split on dots', + files: ['a/b.js', 'src/main.js'], + target: 'a.b', + fromFile: FROM_FILE, + }, + // ── extension coverage ────────────────────────────────────────────────────── + { + name: 'nested .mjs by bare specifier', + files: ['lib/mod.mjs', 'src/main.js'], + target: 'lib/mod', + fromFile: FROM_FILE, + }, + { + name: 'nested .cjs by bare specifier', + files: ['lib/legacy.cjs', 'src/main.js'], + target: 'lib/legacy', + fromFile: FROM_FILE, + }, + { + name: 'nested .jsx by bare specifier', + files: ['comp/Btn.jsx', 'src/main.js'], + target: 'comp/Btn', + fromFile: FROM_FILE, + }, + // ── degenerate inputs ─────────────────────────────────────────────────────── + { name: 'empty file set', files: [], target: 'anything', fromFile: FROM_FILE }, + { name: 'empty target', files: ['src/util.js'], target: '', fromFile: FROM_FILE }, + { + name: 'plain miss', + files: ['src/util.js', 'src/main.js'], + target: 'nowhere/at/all', + fromFile: FROM_FILE, + }, + { + name: 'importing file is itself at the repo root', + files: ['config.js', 'main.js'], + target: 'config', + fromFile: 'main.js', + }, +]; + +// ─── runners ───────────────────────────────────────────────────────────────── + +type Resolved = string | readonly string[] | null; + +/** + * One legacy adapter and one current adapter per corpus, each over its own copy + * of the file set — the legacy single-slot cache and the current WeakMap are + * both keyed on the Set, so sharing one would let each observe the other's + * work. + */ +interface Runners { + readonly legacy: (target: string, fromFile: string) => Resolved; + readonly current: (target: string, fromFile: string) => Resolved; + readonly typescript: (target: string, fromFile: string) => Resolved; +} + +function runnersFor(files: readonly string[]): Runners { + const legacyAdapter = legacyMakeJsResolveImportTarget(); + const currentAdapter = makeJsResolveImportTarget(); + const legacyFiles = new Set(files); + const currentFiles = new Set(files); + const typescriptFiles = new Set(files); + return { + legacy: (target, fromFile) => legacyAdapter(target, fromFile, legacyFiles, undefined), + current: (target, fromFile) => currentAdapter(target, fromFile, currentFiles, undefined), + typescript: (target, fromFile) => + typescriptScopeResolver.resolveImportTarget(target, fromFile, typescriptFiles, undefined), + }; +} + +interface Outcome { + readonly name: string; + readonly target: string; + readonly legacy: Resolved; + readonly current: Resolved; + readonly typescript: Resolved; +} + +/** Every case, resolved once. Built lazily and shared: the generated corpus is + * one file set across 165 targets, which is the shape a real pass has. */ +const OUTCOMES: readonly Outcome[] = (() => { + const generated = runnersFor(GENERATED_FILES); + const handOutcomes = HAND_CASES.map((testCase) => { + const runners = runnersFor(testCase.files); + return { + name: testCase.name, + target: testCase.target, + legacy: runners.legacy(testCase.target, testCase.fromFile), + current: runners.current(testCase.target, testCase.fromFile), + typescript: runners.typescript(testCase.target, testCase.fromFile), + }; + }); + const generatedOutcomes = GENERATED_CASES.map((testCase) => ({ + name: testCase.name, + target: testCase.target, + legacy: generated.legacy(testCase.target, testCase.fromFile), + current: generated.current(testCase.target, testCase.fromFile), + typescript: generated.typescript(testCase.target, testCase.fromFile), + })); + return [...handOutcomes, ...generatedOutcomes]; +})(); + +const DIVERGENT: readonly Outcome[] = OUTCOMES.filter( + (outcome) => outcome.legacy !== outcome.current, +); + +function describeDivergence(outcome: Outcome): string { + return `${outcome.name} :: ${JSON.stringify(outcome.legacy)} → ${JSON.stringify(outcome.current)}`; +} + +/** + * Where in `suffixResolve`'s two nested loops a result was matched, as + * `pathPartIndex:extensionIndex`. Lower is more specific: a longer path suffix, + * or the same suffix at an earlier extension. Mirrors `resolveImportPath`'s + * own `pathParts` construction (dots become slashes only when the specifier + * carries no slash). + */ +function matchPosition(result: string, target: string): readonly [number, number] { + const pathLike = target.includes('/') ? target : target.replace(/\./g, '/'); + const parts = pathLike.split('/').filter(Boolean); + const lower = result.toLowerCase(); + const positions = parts.flatMap((_part, i) => { + const suffix = parts.slice(i).join('/').toLowerCase(); + return EXTENSIONS.flatMap((ext, e) => { + const candidate = suffix + ext.toLowerCase(); + const matches = lower === candidate || lower.endsWith(`/${candidate}`); + return matches ? [[i, e] as const] : []; + }); + }); + return positions[0] ?? [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER]; +} + +/** + * Class A — a repo-root file, which has no proper suffix and so was + * unreachable through this leg at all. Every line goes `null → `, + * and the arm below enforces that shape rather than trusting the grouping. + */ +const CLASS_A_DIVERGENCES: readonly string[] = [ + 'root-level file by bare specifier :: null → "config.js"', + 'root index.js by bare specifier :: null → "index.js"', + 'root-level .mjs by bare specifier :: null → "esm.mjs"', + 'root-level .cjs by bare specifier :: null → "legacy.cjs"', + 'root-level .jsx by bare specifier :: null → "Btn.jsx"', + 'importing file is itself at the repo root :: null → "config.js"', + 'generated config :: null → "config.js"', +]; + +/** + * Class B — the scan skipped the whole-path candidate and landed on a shorter + * path suffix or a later extension instead. Every line moves from one file to + * another, toward the more specific match; `never answers at a less specific + * path-part / extension position` is the property behind that claim. + * + * `generated src/main` and `generated app/main` are the witnesses that settle + * the direction: both used to resolve into `node_modules`, because + * `node_modules/dep0/lib/main.js` is the first file in the corpus ending in + * `/main.js` and the scan never tried the two-segment suffix as a whole path. + */ +const CLASS_B_DIVERGENCES: readonly string[] = [ + 'whole-path candidate earlier in file order than a proper-suffix one :: "vendor/src/util.js" → "src/util.js"', + 'whole-path hit at a long suffix vs proper-suffix hit at a short one :: "node_modules/dep/lib/main.js" → "app/main.js"', + 'whole-path candidate at an earlier extension than the proper-suffix one :: "x/U.js" → "U.jsx"', + 'whole-path candidate at an earlier extension than the proper-suffix one, reversed :: "x/U.js" → "U.jsx"', + 'root .js outranks a nested .mjs :: "lib/mod.mjs" → "mod.js"', + 'case-differing whole path beats a case-exact proper suffix :: "other/src/util.js" → "SRC/Util.js"', + 'generated src/main :: "node_modules/dep0/lib/main.js" → "src/main.js"', + 'generated app/main :: "node_modules/dep0/lib/main.js" → "app/main.js"', + 'generated mod0 :: "lib/mod0.mjs" → "mod0.js"', + 'generated mod1 :: "lib/mod1.mjs" → "mod1.js"', + 'generated mod2 :: "lib/mod2.mjs" → "mod2.js"', + 'generated mod3 :: "lib/mod3.mjs" → "mod3.js"', + 'generated mod4 :: "lib/mod4.mjs" → "mod4.js"', + 'generated mod5 :: "lib/mod5.mjs" → "mod5.js"', + 'generated mod6 :: "lib/mod6.mjs" → "mod6.js"', + 'generated mod7 :: "lib/mod7.mjs" → "mod7.js"', + 'generated mod8 :: "lib/mod8.mjs" → "mod8.js"', + 'generated mod9 :: "lib/mod9.mjs" → "mod9.js"', +]; + +/** + * The divergences this corpus produces, pinned old → new. A future edit that + * moves a DIFFERENT answer — or stops moving one of these — fails here rather + * than quietly shipping. + */ +const EXPECTED_DIVERGENCES: readonly string[] = [...CLASS_A_DIVERGENCES, ...CLASS_B_DIVERGENCES]; + +// ─── the differential ──────────────────────────────────────────────────────── + +describe('JavaScript import-target parity with the pre-index adapter (#2910)', () => { + it('agrees with the pre-index adapter on every case outside the pinned set', () => { + const pinned = new Set(EXPECTED_DIVERGENCES); + const unexpected = DIVERGENT.map(describeDivergence).filter( + (description) => !pinned.has(description), + ); + + expect(unexpected).toEqual([]); + }); + + it('moves exactly the pinned answers, and still moves all of them', () => { + expect(DIVERGENT.map(describeDivergence).sort()).toEqual([...EXPECTED_DIVERGENCES].sort()); + }); + + it('never loses a match the scan found', () => { + const lost = OUTCOMES.filter( + (outcome) => outcome.legacy !== null && outcome.current === null, + ).map(describeDivergence); + + expect(lost).toEqual([]); + }); + + it('never answers at a less specific path-part / extension position', () => { + const lessSpecific = DIVERGENT.filter( + (outcome) => typeof outcome.legacy === 'string' && typeof outcome.current === 'string', + ) + .map((outcome) => ({ + outcome, + was: matchPosition(String(outcome.legacy), outcome.target), + now: matchPosition(String(outcome.current), outcome.target), + })) + .filter(({ was, now }) => now[0] > was[0] || (now[0] === was[0] && now[1] > was[1])) + .map(({ outcome, was, now }) => `${describeDivergence(outcome)} (${was} → ${now})`); + + expect(lessSpecific).toEqual([]); + }); + + it('answers identically to the TypeScript adapter over the whole corpus', () => { + const disagreements = OUTCOMES.filter((outcome) => outcome.current !== outcome.typescript).map( + (outcome) => + `${outcome.name} :: js=${JSON.stringify(outcome.current)} ts=${JSON.stringify(outcome.typescript)}`, + ); + + expect(disagreements).toEqual([]); + }); + + it('both classes are witnessed, and each line has its class’s shape', () => { + // Class A is `null → `: no slash in the new answer, which + // is the whole reason the scan could not reach it. + const misfiledA = CLASS_A_DIVERGENCES.filter((line) => !/ :: null → "[^/"]+"$/.test(line)); + // Class B moves between two files; neither side is null. + const misfiledB = CLASS_B_DIVERGENCES.filter((line) => line.includes('null')); + + expect(misfiledA).toEqual([]); + expect(misfiledB).toEqual([]); + expect(CLASS_A_DIVERGENCES.length).toBeGreaterThan(0); + expect(CLASS_B_DIVERGENCES.length).toBeGreaterThan(0); + }); + + it('resolves real JavaScript imports (the differential is not vacuous)', () => { + const resolveImportTarget = makeJsResolveImportTarget(); + const files = new Set([ + 'src/main.js', + 'src/util.js', + 'src/components/Widget.jsx', + 'src/models/index.js', + 'lib/esm.mjs', + 'lib/legacy.cjs', + 'node_modules/dep/index.js', + ]); + + expect(resolveImportTarget('./util', FROM_FILE, files, undefined)).toBe('src/util.js'); + expect(resolveImportTarget('./util.js', FROM_FILE, files, undefined)).toBe('src/util.js'); + expect(resolveImportTarget('./components/Widget', FROM_FILE, files, undefined)).toBe( + 'src/components/Widget.jsx', + ); + expect(resolveImportTarget('./models', FROM_FILE, files, undefined)).toBe( + 'src/models/index.js', + ); + expect(resolveImportTarget('lib/esm', FROM_FILE, files, undefined)).toBe('lib/esm.mjs'); + expect(resolveImportTarget('lib/legacy', FROM_FILE, files, undefined)).toBe('lib/legacy.cjs'); + expect(resolveImportTarget('./nowhere', FROM_FILE, files, undefined)).toBeNull(); + expect(resolveImportTarget('@scope/absent', FROM_FILE, files, undefined)).toBeNull(); + }); +}); + +// ─── the guard the defect needed ───────────────────────────────────────────── + +describe('JavaScript import resolution never enters the linear suffix scan (#2910)', () => { + /** + * The control runs first and on purpose. `CountingSet` cannot see this defect + * — the scan walks the array the index materialized, not the Set — so an + * assertion of zero is worth nothing unless the same instrument is shown + * reading nonzero against the adapter that had the bug. + */ + it('the pre-index adapter scans linearly once per bare specifier; the current one never does', () => { + const files = new Set(GENERATED_FILES); + const bareTargets = generatedTargets().filter((target) => !target.startsWith('.')); + + const legacyAdapter = legacyMakeJsResolveImportTarget(); + linearScans.count = 0; + bareTargets.forEach((target) => legacyAdapter(target, FROM_FILE, files, undefined)); + const legacyEntries = linearScans.count; + + const currentAdapter = makeJsResolveImportTarget(); + linearScans.count = 0; + bareTargets.forEach((target) => currentAdapter(target, FROM_FILE, new Set(files), undefined)); + const currentEntries = linearScans.count; + + expect(legacyEntries).toBe(bareTargets.length); + expect(currentEntries).toBe(0); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/php-import-target-parity.test.ts b/gitnexus/test/unit/scope-resolution/php-import-target-parity.test.ts new file mode 100644 index 000000000..7b62ed1e9 --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/php-import-target-parity.test.ts @@ -0,0 +1,1223 @@ +/** + * Differential harness for the PHP import-target index (#2901). + * + * PHP was the last language resolving imports with a full workspace scan per + * import: both adapters in `languages/php/import-target.ts` materialized + * `[...allFilePaths]` twice and then passed `resolvePhpImportInternal` an + * `index` of `undefined`, dropping it onto `suffixResolve`'s linear `findIndex` + * — one pass over every file per path-part × per extension. + * + * Unlike #2877–#2880, handing that function the SHARED `SuffixIndex` is not a + * hoist. `resolvePhpImportInternal` reads the index at three sites and all + * three answer a different question than the scan they short-circuit, so the + * fix passes a PARITY view instead (see the `#2901` header in + * `import-target.ts`). This file is the proof that the view is faithful: it + * holds verbatim copies of the pre-change implementations + * (`git show HEAD~:gitnexus/src/core/ingestion/languages/php/import-target.ts`) + * and asserts the shipped ones agree with them everywhere. + * + * The corpus is built to force the three divergences, because ordinary PHP + * imports do not show them — a plain `use App\Models\User;` against one + * matching file agrees under either index: + * + * 1. PSR-4 class-style is `allFiles.has(filePath)`, an exact whole-path test, + * and the raw index would add a case-insensitive SUFFIX probe beside it — + * so the corpus contains `vendor/**` mirrors that differ only in case. + * 2. The namespace-directory scan is anchored at the repo root + * (`f.startsWith(nsDir + '/')`), the raw index's `getFilesInDir` is keyed + * on every directory SUFFIX — so the corpus contains + * `vendor/pkg/app/Models/` beside `app/Models/`. + * 3. `suffixResolve`'s scan tests `endsWith('/' + S)` and therefore can only + * match a PROPER suffix, while `buildSuffixIndex` indexes the whole path + * too — so the corpus contains root-level files and paths that are + * themselves the suffix another file carries, in BOTH iteration orders. + * + * Set iteration order is the tie-break for every one of those, which is why the + * generated corpus is emitted in a fixed order and several hand cases appear + * twice with the two files swapped. Nothing here is random. + * + * Every hand case additionally pins ABSOLUTE `expected` / + * `expectedViaWorkspace` literals, for the reason spelled out under the + * verbatim-copy banner below: the differential cannot fail on anything the two + * sides share, and they share `resolvePhpImportInternal` itself. + * + * This file calls the resolver functions directly, so it does NOT guard PR + * #1918 review finding P1 — a defensive `new Set(allFilePaths)` in + * `php/scope-resolver.ts` leaves every arm here green. That is + * `test/integration/php-import-index-reuse.test.ts`. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile, ParsedImport, SymbolDefinition, WorkspaceIndex } from 'gitnexus-shared'; + +import { + resolvePhpImportTarget, + resolvePhpImportTargetInternal, + type PhpResolveContext, +} from '../../../src/core/ingestion/languages/php/import-target.js'; +import { resolvePhpImportInternal } from '../../../src/core/ingestion/import-resolvers/php.js'; +import { buildSuffixIndex } from '../../../src/core/ingestion/import-resolvers/utils.js'; +import type { ComposerConfig } from '../../../src/core/ingestion/language-config.js'; +import type { ImportResolutionContext } from '../../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; +import { CountingSet } from '../../helpers/counting-file-set.js'; + +// ─── verbatim pre-change implementation ────────────────────────────────────── +// Copied from `git show HEAD~:gitnexus/src/core/ingestion/languages/php/ +// import-target.ts`. `resolvePhpImportInternal` is NOT copied — it is imported +// from the shipped source, and #2901 (`67307cc91`) DID change it: 41 lines, +// including the `if (index) … else` split that moved the namespace-directory +// scan out of the empty-bucket path. +// +// Passing `index: undefined` still reaches the pre-change behaviour through that +// new `else`, so the legacy side remains a faithful stand-in for the one +// function — but it is a stand-in built out of the code under test. The `..` +// guard, the PSR-4 prefix loop, `allFiles.has`, the `nsDir` computation and the +// `suffixResolve` call are LITERALLY SHARED with the current side, so an edit to +// any of them moves both sides identically and `expect(current).toBe(legacy)` +// stays green. That is a real weakness of importing rather than copying, and it +// is why every hand case below also pins absolute literals: the differential +// proves the index hoist preserved behaviour, the literals prove the behaviour +// being preserved is the one the case is named for. + +function legacyNormalizePhpPath(value: string): string { + return value.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+$/, ''); +} + +function legacyNamespaceDirectories( + targetRaw: string, + composerConfig: ComposerConfig | null, + resolved: string | null, +): string[] { + const directories = new Set(); + if (resolved !== null) { + const normalizedResolved = legacyNormalizePhpPath(resolved); + const separator = normalizedResolved.lastIndexOf('/'); + if (separator >= 0) directories.add(normalizedResolved.slice(0, separator)); + } + + if (composerConfig === null) return [...directories]; + + const normalizedTarget = legacyNormalizePhpPath(targetRaw); + const mappings = [...composerConfig.psr4.entries()].sort((left, right) => { + const lengthDifference = right[0].length - left[0].length; + return lengthDifference !== 0 ? lengthDifference : left[0].localeCompare(right[0]); + }); + for (const [namespacePrefix, directoryPrefix] of mappings) { + const normalizedPrefix = legacyNormalizePhpPath(namespacePrefix); + if ( + normalizedTarget !== normalizedPrefix && + !normalizedTarget.startsWith(`${normalizedPrefix}/`) + ) { + continue; + } + + const remainder = normalizedTarget.slice(normalizedPrefix.length).replace(/^\//, ''); + const separator = remainder.lastIndexOf('/'); + const relativeNamespace = separator >= 0 ? remainder.slice(0, separator) : ''; + directories.add( + legacyNormalizePhpPath( + relativeNamespace === '' ? directoryPrefix : `${directoryPrefix}/${relativeNamespace}`, + ), + ); + break; + } + return [...directories]; +} + +const legacyPhpDirectoryIndexCache = new WeakMap< + readonly ParsedFile[], + ReadonlyMap +>(); + +function legacyParentDirectory(filePath: string): string { + const normalizedPath = legacyNormalizePhpPath(filePath); + const separator = normalizedPath.lastIndexOf('/'); + return separator < 0 ? '' : normalizedPath.slice(0, separator); +} + +function legacyDirectoryAliases(filePath: string): string[] { + const normalizedPath = legacyNormalizePhpPath(filePath); + const separator = normalizedPath.lastIndexOf('/'); + if (separator < 0) return ['']; + + const parent = normalizedPath.slice(0, separator); + const aliases = new Set([parent]); + const segments = parent.split('/').filter(Boolean); + for (let index = 0; index < segments.length; index++) { + aliases.add(segments.slice(index).join('/')); + } + return [...aliases]; +} + +function legacyFilesByDirectory( + parsedFiles: readonly ParsedFile[], +): ReadonlyMap { + const cached = legacyPhpDirectoryIndexCache.get(parsedFiles); + if (cached) return cached; + + const mutable = new Map(); + for (const parsed of parsedFiles) { + for (const directory of legacyDirectoryAliases(parsed.filePath)) { + const files = mutable.get(directory) ?? []; + files.push(parsed); + mutable.set(directory, files); + } + } + legacyPhpDirectoryIndexCache.set(parsedFiles, mutable); + return mutable; +} + +function legacyResolvePhpImportTarget( + parsedImport: ParsedImport, + workspaceIndex: WorkspaceIndex, +): string | null { + // The shipped adapter spells this guard `ctx === undefined || ...`; CodeQL + // flags that as a comparison between inconvertible types (`WorkspaceIndex` is + // an object type, never `undefined`). Optional chaining is the same guard at + // runtime — an undefined index still fails the `typeof` test and returns null + // — so the copy stays behaviourally verbatim. + const ctx = workspaceIndex as PhpResolveContext; + if ( + typeof (workspaceIndex as { fromFile?: unknown } | undefined)?.fromFile !== 'string' || + !((workspaceIndex as { allFilePaths?: unknown } | undefined)?.allFilePaths instanceof Set) + ) { + return null; + } + if (parsedImport.kind === 'dynamic-unresolved') return null; + if (parsedImport.targetRaw === null || parsedImport.targetRaw === '') return null; + + const allFiles = ctx.allFilePaths as Set; + const normalizedFileList = [...allFiles].map((f) => f.replace(/\\/g, '/')); + const allFileList = [...allFiles]; + + return resolvePhpImportInternal( + parsedImport.targetRaw, + null, // composerConfig not available through LanguageProvider path + allFiles, + normalizedFileList, + allFileList, + undefined, + ); +} + +function legacyResolvePhpImportTargetInternal( + targetRaw: string, + _fromFile: string, + allFilePaths: ReadonlySet, + resolutionConfig?: unknown, + context?: ImportResolutionContext, +): string | null { + if (targetRaw === '') return null; + + const composerConfig = + resolutionConfig !== undefined && resolutionConfig !== null + ? (resolutionConfig as ComposerConfig) + : null; + + const allFiles = allFilePaths as Set; + const normalizedFileList = [...allFiles].map((f) => f.replace(/\\/g, '/')); + const allFileList = [...allFiles]; + + const resolved = resolvePhpImportInternal( + targetRaw, + composerConfig, + allFiles, + normalizedFileList, + allFileList, + undefined, + ); + + const parsedImport = context?.parsedImport; + const symbolKind = + parsedImport?.kind === 'named' || parsedImport?.kind === 'alias' + ? parsedImport.importedSymbolKind + : undefined; + if ( + context === undefined || + parsedImport === undefined || + (symbolKind !== 'function' && symbolKind !== 'const') + ) { + return resolved; + } + + const importedName = targetRaw.replace(/\\/g, '/').split('/').filter(Boolean).at(-1); + if (importedName === undefined) return resolved; + + const directories = legacyNamespaceDirectories(targetRaw, composerConfig, resolved); + const directoryIndex = legacyFilesByDirectory(context.parsedFiles); + const candidateFiles = [ + ...new Set( + directories.flatMap((directory) => { + const files = directoryIndex.get(legacyNormalizePhpPath(directory)) ?? []; + const distinctParents = new Set(files.map((file) => legacyParentDirectory(file.filePath))); + return distinctParents.size > 1 ? [] : files; + }), + ), + ]; + const expectedType = symbolKind === 'function' ? 'Function' : 'Variable'; + const declaringFiles = candidateFiles.filter((parsed) => + parsed.localDefs.some((def) => { + if (def.type !== expectedType) return false; + const simpleName = (def.qualifiedName ?? '').split(/[\\.]/).at(-1); + return simpleName === importedName; + }), + ); + + if (declaringFiles.length > 1) return null; + if (declaringFiles.length === 1) return declaringFiles[0].filePath; + + if (symbolKind === 'const' && candidateFiles.length === 1) return candidateFiles[0].filePath; + return resolved; +} + +// ─── fixtures ──────────────────────────────────────────────────────────────── + +/** + * One differential case. `files` is emitted in the listed order and that order + * is the tie-break under test, so cases that exist to pin a tie appear twice + * with the order reversed rather than relying on one arrangement. + */ +interface Case { + readonly name: string; + readonly files: readonly string[]; + readonly target: string; + readonly composer?: ComposerConfig; + readonly parsedImport?: ParsedImport; + readonly defs?: ReadonlyMap; +} + +/** + * A hand case plus the two literals it must produce. Required, not optional: + * the generated sweep is a pure differential by design, but a hand case exists + * to pin one named behaviour and cannot do that without saying what it is. + */ +interface HandCase extends Case { + /** + * What the ScopeResolver adapter (`resolvePhpImportTargetInternal`) returns. + * That is the path with `composerConfig` and the function/const declaration + * leg, so it is the one PSR-4 and `context` actually reach. + */ + readonly expected: string | null; + /** + * What the LanguageProvider adapter (`resolvePhpImportTarget`) returns. It + * hard-codes `composerConfig: null` and takes no `ImportResolutionContext`, + * so for every case carrying a `composer` or a `parsedImport` this is the + * plain `suffixResolve` answer and differs from `expected`. + */ + readonly expectedViaWorkspace: string | null; +} + +function composer(entries: readonly (readonly [string, string])[]): ComposerConfig { + return { psr4: new Map(entries) }; +} + +function definition( + filePath: string, + type: SymbolDefinition['type'], + name: string, +): SymbolDefinition { + return { nodeId: `def:${filePath}:${type}:${name}`, filePath, type, qualifiedName: name }; +} + +function parsedFilesFor(testCase: Case): readonly ParsedFile[] { + return testCase.files.map( + (filePath) => + ({ + filePath, + localDefs: (testCase.defs?.get(filePath) ?? []).map(([type, name]) => + definition(filePath, type, name), + ), + }) as ParsedFile, + ); +} + +function namedImport( + targetRaw: string, + kind: 'function' | 'const' | 'class', + localName: string, +): ParsedImport { + return { + kind: 'named', + localName, + importedName: localName, + targetRaw, + importedSymbolKind: kind, + } as ParsedImport; +} + +const APP_PSR4 = composer([['App', 'app']]); +const SRC_PSR4 = composer([['App', 'src']]); +const NESTED_PSR4 = composer([ + ['App', 'app'], + ['App\\Models', 'app/Domain'], +]); +const ROOT_PSR4 = composer([['App', '']]); +const TRAILING_SLASH_PSR4 = composer([['App', 'app/']]); + +/** + * A deterministic multi-root workspace. Every root carries the same relative + * layout so that a suffix-keyed lookup and a root-anchored scan disagree about + * which root wins, and `Vendor`/`vendor` differ only in case. + */ +function generatedFiles(): string[] { + const files: string[] = []; + for (let i = 0; i < 12; i++) { + files.push(`vendor/pkg${i % 3}/app/Models/Entity${i}.php`); + files.push(`app/Models/Entity${i}.php`); + files.push(`app/Services/Service${i}.php`); + files.push(`src/App/Legacy/Entity${i}.php`); + files.push(`APP/models/entity${i}.php`); + files.push(`Entity${i}.php`); + files.push(`app/Helpers/helpers${i}.phtml`); + files.push(`packages/mod${i}/src/Widget.php`); + files.push(`app\\Windows\\Entity${i}.php`); + } + files.push('index.php'); + files.push('app/Models/User.php'); + files.push('app/Models/functions.php'); + files.push('app/Config/constants.php'); + return files; +} + +const GENERATED_FILES = generatedFiles(); + +/** Targets swept across `GENERATED_FILES`: hits, near-misses and full misses. */ +function generatedTargets(): string[] { + const targets: string[] = []; + for (let i = 0; i < 12; i++) { + targets.push(`App\\Models\\Entity${i}`); + targets.push(`app\\models\\entity${i}`); + targets.push(`Entity${i}`); + targets.push(`Models\\Entity${i}`); + targets.push(`App\\Legacy\\Entity${i}`); + targets.push(`App\\Services\\Service${i}`); + targets.push(`Widget`); + targets.push(`Symfony\\Component\\Console\\Command${i}`); + targets.push(`App\\Models\\helper${i}`); + targets.push(`\\App\\Models\\Entity${i}`); + targets.push(`App/Models/Entity${i}`); + } + targets.push('index'); + targets.push('App\\Models\\User'); + targets.push('..\\App\\Models\\User'); + targets.push('App'); + return targets; +} + +const GENERATED_CASES: readonly Case[] = generatedTargets().flatMap((target) => + [undefined, APP_PSR4, SRC_PSR4, NESTED_PSR4, ROOT_PSR4, TRAILING_SLASH_PSR4].map( + (config, configIndex) => ({ + name: `generated ${target} · composer#${configIndex}`, + files: GENERATED_FILES, + target, + composer: config, + }), + ), +); + +/** + * Hand-built cases, one per tie-break the index could have moved. + * + * Every `expected` / `expectedViaWorkspace` below was derived by hand from + * `resolvePhpImportInternal` + `suffixResolve` and then confirmed against both + * implementations. Two rules do most of the work and are worth stating once: + * + * - `suffixResolve`'s PATH-PART loop is OUTER and its EXTENSION loop is inner, + * so a longer suffix always beats a shorter one no matter which extensions + * are involved; within one path-part it is first-in-Set-order and + * case-INSENSITIVE (the scan's `endsWith(p)` disjunct is subsumed by its + * `toLowerCase().endsWith(...)` one). + * - the PSR-4 namespace-directory fallback is NOT gated on the imported symbol + * kind. It fires for a class import too, whenever the class-style path + * misses, and returns the FIRST `.php` file directly in the namespace + * directory — see the "known limitation" note atop `import-resolvers/php.ts`. + * Cases that lean on it are marked; their answers are order-dependent in + * general and deterministic here only because the fixture pins the order. + */ +const HAND_CASES: readonly HandCase[] = [ + // ── divergence 3: whole-path vs proper suffix ──────────────────────────── + { + // `Foo.php` IS the suffix, not a file carrying it, and `endsWith('/Foo.php')` + // can only match a PROPER suffix. Unresolvable — the behaviour the parity + // view exists to preserve. + name: 'root-level file is not a proper suffix of itself', + files: ['Foo.php', 'src/Bar.php'], + target: 'Foo', + expected: null, + expectedViaWorkspace: null, + }, + { + // `App/Models/User.php` is invisible at path-part 0 (whole path), so both + // files compete at part 1 on `/Models/User.php` and Set order decides. + name: 'whole-path match loses to an earlier proper-suffix match', + files: ['vendor/x/Models/User.php', 'App/Models/User.php'], + target: 'App\\Models\\User', + expected: 'vendor/x/Models/User.php', + expectedViaWorkspace: 'vendor/x/Models/User.php', + }, + { + name: 'whole-path match loses to a later proper-suffix match too', + files: ['App/Models/User.php', 'vendor/x/Models/User.php'], + target: 'App\\Models\\User', + expected: 'App/Models/User.php', + expectedViaWorkspace: 'App/Models/User.php', + }, + { + // Still not found as a whole path — found at part 1, as `Models/User.php`. + name: 'whole path is the only candidate at all', + files: ['App/Models/User.php'], + target: 'App\\Models\\User', + expected: 'App/Models/User.php', + expectedViaWorkspace: 'App/Models/User.php', + }, + { + // A whole-path candidate the scan must skip, plus TWO proper-suffix + // candidates behind it — so the skip has to land on the first of them and + // not merely on "some other file". Both are hit at path-part 0. + name: 'whole-path match skipped, first of several proper-suffix matches wins', + files: ['App/Models/User.php', 'one/App/Models/User.php', 'two/App/Models/User.php'], + target: 'App\\Models\\User', + expected: 'one/App/Models/User.php', + expectedViaWorkspace: 'one/App/Models/User.php', + }, + { + name: 'whole-path match skipped, first of several proper-suffix matches wins, reversed', + files: ['two/App/Models/User.php', 'App/Models/User.php', 'one/App/Models/User.php'], + target: 'App\\Models\\User', + expected: 'two/App/Models/User.php', + expectedViaWorkspace: 'two/App/Models/User.php', + }, + { + name: 'root-level file with a namespace-shaped import', + files: ['index.php', 'Kernel.php'], + target: 'Kernel', + expected: null, + expectedViaWorkspace: null, + }, + // ── divergence 3: case-sensitive hit must not outrank an earlier ci hit ── + { + // `a/FOO.php` matches `/Foo.php` case-insensitively and comes first, so the + // exact-case `b/Foo.php` behind it never gets a turn. + name: 'lowercase file first, exact-case file second', + files: ['a/FOO.php', 'b/Foo.php'], + target: 'Foo', + expected: 'a/FOO.php', + expectedViaWorkspace: 'a/FOO.php', + }, + { + name: 'exact-case file first, lowercase file second', + files: ['b/Foo.php', 'a/FOO.php'], + target: 'Foo', + expected: 'b/Foo.php', + expectedViaWorkspace: 'b/Foo.php', + }, + { + name: 'case tie across a multi-segment suffix', + files: ['vendor/x/models/user.php', 'app/Models/User.php'], + target: 'Models\\User', + expected: 'vendor/x/models/user.php', + expectedViaWorkspace: 'vendor/x/models/user.php', + }, + { + name: 'case tie across a multi-segment suffix, reversed', + files: ['app/Models/User.php', 'vendor/x/models/user.php'], + target: 'Models\\User', + expected: 'app/Models/User.php', + expectedViaWorkspace: 'app/Models/User.php', + }, + // ── divergence 1: PSR-4 class-style is an exact whole-path test ────────── + { + // The only case in this group that the class-style `allFiles.has` leg + // actually answers: `App\Models\User` + `App => app` is exactly + // `app/Models/User.php`. Everything below it misses that leg and falls + // through, which is what makes the group interesting. + name: 'psr-4 exact hit', + files: ['app/Models/User.php'], + target: 'App\\Models\\User', + composer: APP_PSR4, + expected: 'app/Models/User.php', + // No composer on this path, so it comes from `/Models/User.php` at part 1. + expectedViaWorkspace: 'app/Models/User.php', + }, + { + // PSR-4 is case-sensitive by spec, so the exact leg correctly misses + // `app/models/user.php`, the namespace directory `app/Models` does not + // exist either, and the answer comes from the case-INSENSITIVE suffix + // fallback. Loose, but it is the shipped behaviour and predates #2901. + name: 'psr-4 target differs from the file only by case', + files: ['app/models/user.php'], + target: 'App\\Models\\User', + composer: APP_PSR4, + expected: 'app/models/user.php', + expectedViaWorkspace: 'app/models/user.php', + }, + { + // KNOWN LIMITATION: the class-style path `src/Models/User.php` misses, and + // the namespace-directory fallback then answers a CLASS import with "first + // `.php` in `src/Models/`" — here `src/Models/user.php`, which is only + // coincidentally the right file. `other/Models/User.php` is never reached. + name: 'psr-4 mapped dir differs from the namespace, file differs by case', + files: ['src/Models/user.php', 'other/Models/User.php'], + target: 'App\\Models\\User', + composer: SRC_PSR4, + expected: 'src/Models/user.php', + expectedViaWorkspace: 'src/Models/user.php', + }, + { + // `src/Models/` does not exist at the repo root, so the root-anchored + // directory bucket is empty and the suffix leg finds the vendor copy at + // path-part 1. A directory index keyed on suffixes would have answered it + // one leg earlier — same file here, different file in the case below. + name: 'psr-4 mapped dir exists only under vendor, by case-insensitive suffix', + files: ['vendor/pkg/src/Models/User.php'], + target: 'App\\Models\\User', + composer: SRC_PSR4, + expected: 'vendor/pkg/src/Models/User.php', + expectedViaWorkspace: 'vendor/pkg/src/Models/User.php', + }, + { + // The mapped directory (`src/lib`) shares no segment with the namespace + // (`App`), so the class-style probe and the `suffixResolve` fallback name + // two DIFFERENT files. Only the exact-`has` leg is supposed to see the + // first one; the raw index's case-insensitive suffix probe reaches it. + // Correct answer: the suffix leg's `other/Models/User.php`, first in order. + name: 'psr-4 mapped dir path and namespace path name different files', + files: ['other/Models/User.php', 'vendor/one/src/lib/Models/user.php'], + target: 'App\\Models\\User', + composer: composer([['App', 'src/lib']]), + expected: 'other/Models/User.php', + expectedViaWorkspace: 'other/Models/User.php', + }, + { + name: 'psr-4 mapped dir path and namespace path name different files, reversed', + files: ['vendor/one/src/lib/Models/user.php', 'other/Models/User.php'], + target: 'App\\Models\\User', + composer: composer([['App', 'src/lib']]), + expected: 'vendor/one/src/lib/Models/user.php', + expectedViaWorkspace: 'vendor/one/src/lib/Models/user.php', + }, + { + // `App\Models => app/Domain` sorts before `App => app` (longer key), so the + // class-style leg hits `app/Domain/User.php` and never considers + // `app/Models/User.php`. The two adapters legitimately disagree here: with + // no composer there is no longest-prefix rule and the suffix leg answers + // `/Models/User.php` instead. + name: 'psr-4 longest-prefix mapping wins', + files: ['app/Domain/User.php', 'app/Models/User.php'], + target: 'App\\Models\\User', + composer: NESTED_PSR4, + expected: 'app/Domain/User.php', + expectedViaWorkspace: 'app/Models/User.php', + }, + { + // KNOWN LIMITATION: an empty `dirPrefix` builds the class-style path as + // `'' + '/Models/User' + '.php'` = `/Models/User.php`, with a leading slash + // no repo-relative path has — so a root PSR-4 mapping never hits that leg, + // and `nsDir` comes out `/Models` which no directory bucket holds either. + // The answer is the suffix leg's, and only at path-part 2 (`/User.php`): + // `Models/User.php` is the whole path, invisible to `/Models/User.php`. + name: 'psr-4 mapped to the repo root', + files: ['Models/User.php'], + target: 'App\\Models\\User', + composer: ROOT_PSR4, + expected: 'Models/User.php', + expectedViaWorkspace: 'Models/User.php', + }, + { + // KNOWN LIMITATION: a mapping kept with its trailing slash concatenates to + // `app//Models/User.php`, which misses every leg. `loadPhpComposerConfig` + // strips trailing slashes, so production never builds this config — the + // arm pins what happens if one ever reaches the resolver. The answer is + // again the plain suffix leg's. + name: 'psr-4 dir prefix carries a trailing slash', + files: ['app/Models/User.php'], + target: 'App\\Models\\User', + composer: TRAILING_SLASH_PSR4, + expected: 'app/Models/User.php', + expectedViaWorkspace: 'app/Models/User.php', + }, + // ── divergence 2: namespace-directory scan is root-anchored ────────────── + { + // The witness for divergence 2: `app/Models` is also a SUFFIX of + // `vendor/pkg/app/Models`, and the vendor file comes first in Set order, so + // a suffix-keyed directory index answers `vendor/pkg/app/Models/Zed.php`. + // Root-anchored, only `app/Models/Aaa.php` is in the bucket. + name: 'namespace dir: root-anchored candidate beats a suffix-matching vendor dir', + files: ['vendor/pkg/app/Models/Zed.php', 'app/Models/Aaa.php'], + target: 'App\\Models\\getUser', + composer: APP_PSR4, + expected: 'app/Models/Aaa.php', + // Without composer there is no namespace-directory leg at all, and + // `getUser` is not a file, so nothing matches. + expectedViaWorkspace: null, + }, + { + // Same witness from the other side: with the root-anchored bucket empty the + // vendor mirror is unreachable, where a suffix-keyed one would return it. + name: 'namespace dir: only a suffix-matching vendor dir exists', + files: ['vendor/pkg/app/Models/Zed.php'], + target: 'App\\Models\\getUser', + composer: APP_PSR4, + expected: null, + expectedViaWorkspace: null, + }, + { + // KNOWN LIMITATION, pinned rather than endorsed: "first `.php` file in the + // namespace directory" is Set-iteration order, so `Bbb` beats the + // alphabetically-earlier `Aaa`. Deterministic here only because the fixture + // fixes the insertion order; in a real repo it follows the walker's. + name: 'namespace dir: several candidates, first in order wins', + files: ['app/Models/Bbb.php', 'app/Models/Aaa.php', 'app/Models/Ccc.php'], + target: 'App\\Models\\getUser', + composer: APP_PSR4, + expected: 'app/Models/Bbb.php', + expectedViaWorkspace: null, + }, + { + name: 'namespace dir: nested subdirectory is not a direct child', + files: ['app/Models/Nested/Deep.php'], + target: 'App\\Models\\getUser', + composer: APP_PSR4, + expected: null, + expectedViaWorkspace: null, + }, + { + name: 'namespace dir: non-php sibling is skipped', + files: ['app/Models/notes.md', 'app/Models/Aaa.php'], + target: 'App\\Models\\getUser', + composer: APP_PSR4, + expected: 'app/Models/Aaa.php', + expectedViaWorkspace: null, + }, + { + // KNOWN LIMITATION, the multi-segment half of the trailing-slash bug: the + // remainder `Models/getUser` has a separator, so `nsDir` is built as + // `'app/' + '/' + 'Models'` = `app//Models` and matches no directory. Not + // reachable from a parsed `composer.json` (trailing slashes are stripped). + name: 'namespace dir with a trailing-slash mapping', + files: ['app/Models/Aaa.php'], + target: 'App\\Models\\getUser', + composer: TRAILING_SLASH_PSR4, + expected: null, + expectedViaWorkspace: null, + }, + { + // `nsDir` keeps the mapping's trailing slash when the remainder has no + // separator, so the directory bucket must be keyed without it. `nsDir` is + // `app/` here, and `app/bootstrap.php` is its only direct `.php` child — + // `app/Models/User.php` lives one level down. + name: 'namespace dir IS the trailing-slash mapping', + files: ['app/bootstrap.php', 'app/Models/User.php'], + target: 'App\\getUser', + composer: TRAILING_SLASH_PSR4, + expected: 'app/bootstrap.php', + expectedViaWorkspace: null, + }, + { + // KNOWN LIMITATION: with `App => ''` the namespace directory is the repo + // root, and neither the bucket (built from `lastIndexOf('/')`, so root files + // are in no directory) nor the scan it mirrors (`startsWith('/')`) can see + // `User.php`. A root-mapped function import is unresolvable. + name: 'namespace dir at the repo root', + files: ['User.php', 'nested/Other.php'], + target: 'App\\getUser', + composer: ROOT_PSR4, + expected: null, + expectedViaWorkspace: null, + }, + // ── raw vs normalized paths ────────────────────────────────────────────── + { + // Matched on the normalized path, returned RAW. + name: 'backslash file paths', + files: ['src\\App\\Models\\User.php'], + target: 'App\\Models\\User', + expected: 'src\\App\\Models\\User.php', + expectedViaWorkspace: 'src\\App\\Models\\User.php', + }, + { + // `allFiles.has('app/Models/User.php')` is a miss (the Set holds the + // backslash spelling) and the directory bucket is keyed on raw paths, which + // have no `/` at all — so only the normalized suffix leg can answer. + name: 'backslash file paths under a psr-4 mapping', + files: ['app\\Models\\User.php'], + target: 'App\\Models\\User', + composer: APP_PSR4, + expected: 'app\\Models\\User.php', + expectedViaWorkspace: 'app\\Models\\User.php', + }, + { + // Same, minus a suffix leg that can match: `getUser` is not a file. + name: 'backslash namespace dir candidate', + files: ['app\\Models\\Aaa.php'], + target: 'App\\Models\\getUser', + composer: APP_PSR4, + expected: null, + expectedViaWorkspace: null, + }, + // ── extension order and misses ─────────────────────────────────────────── + { + name: 'extension order: .php before .phtml at the same depth', + files: ['x/User.phtml', 'y/User.php'], + target: 'User', + expected: 'y/User.php', + expectedViaWorkspace: 'y/User.php', + }, + { + // The path-part loop is OUTER, so the full `App/Models/User` + `.php` is + // tried before any extension is tried against the bare `User` — the `.ts` + // file never gets a turn even though `.ts` precedes `.php` in `EXTENSIONS`. + // (Renamed: the old name, "a .ts file shadows a deeper .php file", claimed + // the opposite of what this resolves to. Writing the literal down is what + // surfaced that.) + name: 'extension order: the outer path-part loop beats the inner extension list', + files: ['x/User.ts', 'y/App/Models/User.php'], + target: 'App\\Models\\User', + expected: 'y/App/Models/User.php', + expectedViaWorkspace: 'y/App/Models/User.php', + }, + { + name: 'plain miss', + files: ['src/App/Models/User.php'], + target: 'Other\\Thing', + expected: null, + expectedViaWorkspace: null, + }, + { + // Refused by `if (normalized.includes('..')) return null` before any index + // is consulted. Without that guard the suffix leg resolves this to + // `app/Models/User.php` at path-part 1 — on BOTH sides, so the literal is + // the only assertion here that can see the guard disappear. + name: 'path traversal is rejected', + files: ['app/Models/User.php'], + target: '..\\Models\\User', + expected: null, + expectedViaWorkspace: null, + }, + { + name: 'empty file set', + files: [], + target: 'App\\Models\\User', + composer: APP_PSR4, + expected: null, + expectedViaWorkspace: null, + }, + { + name: 'single-segment miss', + files: ['app/Models/User.php'], + target: 'Nope', + expected: null, + expectedViaWorkspace: null, + }, + // ── function / const leg (context-driven) ──────────────────────────────── + // + // Only the ScopeResolver adapter takes an `ImportResolutionContext`, so + // `expectedViaWorkspace` is the no-composer suffix answer throughout — `null` + // for every one of them, because a symbol name is not a file name. + { + // The namespace-directory leg answers `app/Models/User.php` (first in the + // bucket, and the wrong file); the declaration search then overrides it with + // the file that actually declares `getUser`. That override is the whole + // point of the leg, so pinning the literal is what proves it ran. + name: 'function import with a unique declaration', + files: ['app/Models/User.php', 'app/Models/UserFactory.php'], + target: 'App\\Models\\getUser', + composer: APP_PSR4, + parsedImport: namedImport('App\\Models\\getUser', 'function', 'getUser'), + defs: new Map([ + ['app/Models/User.php', [['Class', 'User'] as const]], + ['app/Models/UserFactory.php', [['Function', 'getUser'] as const]], + ]), + expected: 'app/Models/UserFactory.php', + expectedViaWorkspace: null, + }, + { + // Two declarations of the same name: `declaringFiles.length > 1` returns + // null outright, rather than falling back to the namespace-directory answer. + name: 'function import with duplicate declarations fails closed', + files: ['app/Models/First.php', 'app/Models/Second.php'], + target: 'App\\Models\\getUser', + composer: APP_PSR4, + parsedImport: namedImport('App\\Models\\getUser', 'function', 'getUser'), + defs: new Map([ + ['app/Models/First.php', [['Function', 'getUser'] as const]], + ['app/Models/Second.php', [['Function', 'getUser'] as const]], + ]), + expected: null, + expectedViaWorkspace: null, + }, + { + // KNOWN LIMITATION: the `app/Models` directory ALIAS spans two roots, so + // `distinctParents.size > 1` empties the candidate list and the leg falls + // back to `resolved` — `app/Models/functions.php`, which does NOT declare + // `getUser`. The file that does (`vendor/pkg/app/Models/helpers.php`) is + // never returned. Failing closed here means "keep the composer answer", + // not "return null". + name: 'function import across suffix-colliding roots', + files: ['app/Models/functions.php', 'vendor/pkg/app/Models/helpers.php'], + target: 'App\\Models\\getUser', + composer: APP_PSR4, + parsedImport: namedImport('App\\Models\\getUser', 'function', 'getUser'), + defs: new Map([['vendor/pkg/app/Models/helpers.php', [['Function', 'getUser'] as const]]]), + expected: 'app/Models/functions.php', + expectedViaWorkspace: null, + }, + { + // PHP constants are not emitted as local definitions, so `declaringFiles` is + // always empty and the single-candidate rule decides. + name: 'const import with a single candidate file', + files: ['app/Config/constants.php'], + target: 'App\\Config\\MAX_USERS', + composer: APP_PSR4, + parsedImport: namedImport('App\\Config\\MAX_USERS', 'const', 'MAX_USERS'), + defs: new Map(), + expected: 'app/Config/constants.php', + expectedViaWorkspace: null, + }, + { + // KNOWN LIMITATION: the single-candidate rule declines, but the fallback is + // `resolved` — itself "first `.php` in `app/Config/`", i.e. the same Set + // order the rule is documented as refusing to inherit. Declining changes + // which code picks the file, not whether order picks it. + name: 'const import with several candidate files', + files: ['app/Config/constants.php', 'app/Config/more.php'], + target: 'App\\Config\\MAX_USERS', + composer: APP_PSR4, + parsedImport: namedImport('App\\Config\\MAX_USERS', 'const', 'MAX_USERS'), + defs: new Map(), + expected: 'app/Config/constants.php', + expectedViaWorkspace: null, + }, + { + // `importedSymbolKind: 'class'` returns before the declaration leg, so this + // is the plain PSR-4 class-style hit — and the one context-carrying case + // whose LanguageProvider answer is not null. + name: 'class import ignores the declaration leg', + files: ['app/Models/User.php'], + target: 'App\\Models\\User', + composer: APP_PSR4, + parsedImport: namedImport('App\\Models\\User', 'class', 'User'), + defs: new Map([['app/Models/User.php', [['Class', 'User'] as const]]]), + expected: 'app/Models/User.php', + expectedViaWorkspace: 'app/Models/User.php', + }, +]; + +function runBoth(testCase: Case): { + readonly legacy: string | null; + readonly current: string | null; +} { + const legacyFiles = new Set(testCase.files); + const currentFiles = new Set(testCase.files); + const parsedFiles = parsedFilesFor(testCase); + const context: ImportResolutionContext | undefined = + testCase.parsedImport === undefined + ? undefined + : { parsedFiles, parsedImport: testCase.parsedImport }; + + return { + legacy: legacyResolvePhpImportTargetInternal( + testCase.target, + 'app/Main.php', + legacyFiles, + testCase.composer, + context, + ), + current: resolvePhpImportTargetInternal( + testCase.target, + 'app/Main.php', + currentFiles, + testCase.composer, + context, + ), + }; +} + +function runBothWorkspaceAdapter(testCase: Case): { + readonly legacy: string | null; + readonly current: string | null; +} { + const parsedImport = testCase.parsedImport ?? namedImport(testCase.target, 'class', 'Imported'); + const legacyIndex: PhpResolveContext = { + fromFile: 'app/Main.php', + allFilePaths: new Set(testCase.files), + }; + const currentIndex: PhpResolveContext = { + fromFile: 'app/Main.php', + allFilePaths: new Set(testCase.files), + }; + return { + legacy: legacyResolvePhpImportTarget(parsedImport, legacyIndex as WorkspaceIndex), + current: resolvePhpImportTarget(parsedImport, currentIndex as WorkspaceIndex), + }; +} + +// ─── the differential ──────────────────────────────────────────────────────── + +describe('PHP import-target parity with the pre-index implementation (#2901)', () => { + // Three assertions per arm, and each answers a different question. + // `current === legacy` proves the index hoist behaviour-preserving, but it + // is blind to every line the two sides SHARE — including all of + // `resolvePhpImportInternal`, which is imported rather than copied. Pinning + // the literal on both sides is what makes the arm able to fail on a change + // there, and what says the case still exercises the behaviour it is named + // for rather than having decayed into `null === null`. + it.each(HAND_CASES.map((testCase) => [testCase.name, testCase] as const))( + 'ScopeResolver adapter agrees: %s', + (_name, testCase) => { + const { legacy, current } = runBoth(testCase); + expect(legacy).toBe(testCase.expected); + expect(current).toBe(testCase.expected); + expect(current).toBe(legacy); + }, + ); + + it.each(HAND_CASES.map((testCase) => [testCase.name, testCase] as const))( + 'LanguageProvider adapter agrees: %s', + (_name, testCase) => { + const { legacy, current } = runBothWorkspaceAdapter(testCase); + expect(legacy).toBe(testCase.expectedViaWorkspace); + expect(current).toBe(testCase.expectedViaWorkspace); + expect(current).toBe(legacy); + }, + ); + + /** + * Non-vacuity, the way the Java and COBOL harnesses state it: a table of + * literals is only a specification if enough of them are real paths. 32 of + * the 86 arms above legitimately expect `null` (a resolver miss is a real + * answer and must be pinned like any other), so this fixes the balance rather + * than letting a corpus that quietly stopped matching pass as one that never + * matched. + */ + it('the hand corpus pins real paths, not only misses', () => { + const scopeHits = HAND_CASES.filter((testCase) => testCase.expected !== null); + const workspaceHits = HAND_CASES.filter((testCase) => testCase.expectedViaWorkspace !== null); + const distinct = new Set([ + ...scopeHits.map((testCase) => testCase.expected), + ...workspaceHits.map((testCase) => testCase.expectedViaWorkspace), + ]); + + expect(scopeHits.length).toBe(31); + expect(workspaceHits.length).toBe(23); + expect(distinct.size).toBeGreaterThan(20); + // The two adapters must not be the same assertion twice: `composer` and + // `context` are visible only through the ScopeResolver one. + expect( + HAND_CASES.filter((testCase) => testCase.expected !== testCase.expectedViaWorkspace).length, + ).toBe(9); + }); + + it('agrees on every generated target × composer configuration', () => { + const disagreements = GENERATED_CASES.filter((testCase) => { + const { legacy, current } = runBoth(testCase); + return legacy !== current; + }).map((testCase) => testCase.name); + + expect(disagreements).toEqual([]); + }); + + it('agrees on every generated target through the LanguageProvider adapter', () => { + const disagreements = GENERATED_CASES.filter((testCase) => { + const { legacy, current } = runBothWorkspaceAdapter(testCase); + return legacy !== current; + }).map((testCase) => testCase.name); + + expect(disagreements).toEqual([]); + }); + + /** + * The corpus is only a specification if it actually exercises the three + * divergences. Each of these resolves to a DIFFERENT file (or from null to a + * file) when `resolvePhpImportInternal` is handed the raw shared index + * instead of the parity view, so a future edit that quietly drops one of the + * corrections cannot pass the arms above by also deleting its witness. + */ + it('the corpus contains a witness for each of the three divergences', () => { + const witnesses = [ + 'root-level file is not a proper suffix of itself', + 'whole-path match loses to an earlier proper-suffix match', + 'lowercase file first, exact-case file second', + 'psr-4 mapped dir path and namespace path name different files', + 'namespace dir: root-anchored candidate beats a suffix-matching vendor dir', + 'namespace dir: only a suffix-matching vendor dir exists', + ]; + const byName = new Map(HAND_CASES.map((testCase) => [testCase.name, testCase])); + + const notWitnessed = witnesses.filter((name) => { + const testCase = byName.get(name); + if (testCase === undefined) return true; + const files = [...testCase.files]; + const normalized = files.map((file) => file.replace(/\\/g, '/')); + // The raw index — exactly what a "just pass getWorkspaceFileIndex().index + // through" fix would have handed the resolver. + const rawIndexResult = resolvePhpImportInternal( + testCase.target, + testCase.composer ?? null, + new Set(files), + normalized, + files, + buildSuffixIndex(normalized, files), + ); + return rawIndexResult === runBoth(testCase).legacy; + }); + + expect(notWitnessed).toEqual([]); + }); + + it('resolves real PHP imports (the differential is not vacuous)', () => { + const files = new Set([ + 'app/Models/User.php', + 'app/Services/UserService.php', + 'app/Models/functions.php', + ]); + + expect( + resolvePhpImportTargetInternal('App\\Models\\User', 'app/Main.php', files, APP_PSR4), + ).toBe('app/Models/User.php'); + expect( + resolvePhpImportTargetInternal('App\\Services\\UserService', 'app/Main.php', files, APP_PSR4), + ).toBe('app/Services/UserService.php'); + expect( + resolvePhpImportTargetInternal('Nope\\Missing', 'app/Main.php', files, APP_PSR4), + ).toBeNull(); + }); +}); + +// ─── index reuse at the resolver level ─────────────────────────────────────── + +describe('PHP import-target index reuse (#2901)', () => { + /** + * Counts iterations of the file-set Set. This is the resolver-level half of + * the guard — a rescan reintroduced INSIDE the resolver. The adapter-level + * copy hazard is `test/integration/php-import-index-reuse.test.ts`. + */ + it('iterates the file set once for many imports with no composer.json', () => { + const files = new CountingSet(GENERATED_FILES); + const results: (string | null)[] = []; + + for (const target of generatedTargets()) { + results.push(resolvePhpImportTargetInternal(target, 'app/Main.php', files, undefined)); + } + + expect(files.scans).toBe(1); + // No composer.json, so this is pure `suffixResolve`: the earliest file in + // Set order carrying `App/Models/Entity0.php` as a proper suffix wins, and + // the corpus deliberately puts the vendor mirror first. + expect(results[0]).toBe('vendor/pkg0/app/Models/Entity0.php'); + expect(results.some((result) => result === null)).toBe(true); + }); + + it('iterates the file set once for many PSR-4 imports', () => { + const files = new CountingSet(GENERATED_FILES); + const results: (string | null)[] = []; + + for (let i = 0; i < 12; i++) { + // Class-style hit (`allFiles.has`), namespace-directory fallback, and a + // third-party namespace that matches no PSR-4 prefix — the three legs + // that answer from the index or return before reaching one. + results.push( + resolvePhpImportTargetInternal(`App\\Models\\Entity${i}`, 'app/Main.php', files, APP_PSR4), + ); + results.push( + resolvePhpImportTargetInternal(`App\\Models\\helper${i}`, 'app/Main.php', files, APP_PSR4), + ); + results.push( + resolvePhpImportTargetInternal(`Psr\\Log\\Missing${i}`, 'app/Main.php', files, APP_PSR4), + ); + } + + expect(files.scans).toBe(1); + expect(results[0]).toBe('app/Models/Entity0.php'); + expect(results[1]).toBe('app/Models/Entity0.php'); + expect(results[2]).toBeNull(); + }); + + /** + * `nsDir` keeps a PSR-4 mapping's trailing slash, while the directory bucket + * is keyed on the raw path's own parent (no trailing slash). Getting that + * wrong is invisible to every result assertion — the empty bucket just falls + * through to the scan, which returns the same file — so only the count sees + * it. + */ + it('answers a trailing-slash PSR-4 namespace directory from the index', () => { + const files = new CountingSet(['app/bootstrap.php', 'app/Models/User.php']); + const results: (string | null)[] = []; + + for (let i = 0; i < 5; i++) { + results.push( + resolvePhpImportTargetInternal( + `App\\getUser${i}`, + 'app/Main.php', + files, + TRAILING_SLASH_PSR4, + ), + ); + } + + expect(files.scans).toBe(1); + expect(results[0]).toBe('app/bootstrap.php'); + }); + + /** + * The last per-import traversal in PHP resolution, now closed. + * + * `resolvePhpImportInternal` used to run its namespace-directory scan + * whenever `getFilesInDir` came back EMPTY, not merely when no index was + * supplied — despite the comment above it saying "only when SuffixIndex + * unavailable": + * + * if (index) { const c = index.getFilesInDir(nsDir, '.php'); + * if (c.length > 0) return c[0]; } + * for (const f of allFiles) { ... } // ran even WITH an index + * + * An empty bucket is the correct answer, so the scan could only ever confirm + * it — at the cost of one full pass for every import whose namespace matches + * a PSR-4 prefix but whose directory holds no direct `.php` child + * (`App\Legacy\…` here: `app/Legacy/` does not exist). Measured at 11 + * traversals for 10 imports. + * + * The scan is now in the `else`, which is safe because the bucket is a + * SUPERSET of what the scan can find: a root-anchored direct child + * `nsDir/.php` has its directory exactly equal to `nsDir`, and a + * directory is always one of its own suffixes — so both the shared + * suffix-keyed `dirMap` and this file's root-anchored parity index contain + * it. Empty superset implies empty scan. + * + * The results below are unchanged by that: these imports resolve through the + * later suffix leg, and the namespace-directory pass was pure waste. + */ + it('no longer scans per import when the PSR-4 namespace directory is empty', () => { + const files = new CountingSet(GENERATED_FILES); + const results: (string | null)[] = []; + + for (let i = 0; i < 10; i++) { + results.push( + resolvePhpImportTargetInternal(`App\\Legacy\\Entity${i}`, 'app/Main.php', files, APP_PSR4), + ); + } + + // One build, and nothing per import. Was `1 + 10` before the `else`. + expect(files.scans).toBe(1); + // Paired result assertion: a traversal count of 1 must not be the count of + // a resolver that stopped answering. These resolve via the suffix leg. + expect(results.every((result) => result === 'src/App/Legacy/Entity0.php')).toBe(false); + expect(results[0]).toBe('src/App/Legacy/Entity0.php'); + }); + + it('a distinct file set gets its own index', () => { + const a = new CountingSet(['app/Models/User.php']); + const b = new CountingSet(['lib/Other.php']); + + expect(resolvePhpImportTargetInternal('App\\Models\\User', 'app/Main.php', a, undefined)).toBe( + 'app/Models/User.php', + ); + expect( + resolvePhpImportTargetInternal('App\\Models\\User', 'app/Main.php', b, undefined), + ).toBeNull(); + expect(resolvePhpImportTargetInternal('Other', 'app/Main.php', b, undefined)).toBe( + 'lib/Other.php', + ); + + expect(a.scans).toBe(1); + expect(b.scans).toBe(1); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/python/python-import-probe-count.test.ts b/gitnexus/test/unit/scope-resolution/python/python-import-probe-count.test.ts new file mode 100644 index 000000000..94a22f98c --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/python/python-import-probe-count.test.ts @@ -0,0 +1,259 @@ +/** + * Gate for the two probe-count defects in Python import resolution: the + * duplicated tail in `resolvePythonImportTarget`, and the missing O(1) proof of + * absence in front of `resolvePythonImportInternal`'s bare-import walk. + * + * ## What is being counted, and why not `CountingSet` + * + * `test/helpers/counting-file-set.ts` counts full TRAVERSALS of the file set. + * Neither defect here traverses it even once: both are made of `Set.has` + * probes, so the house instrument reads the same number before and after and + * cannot see either. This file counts the probes themselves — the one quantity + * both defects move — with a local `Set` subclass. Deterministic: the count is + * a function of the corpus and the spelling, never of wall time, and the same + * run reports the same number on any machine. + * + * ## Defect 1 — the duplicated tail (`named` / `alias` paid twice) + * + * `resolvePythonImportTarget` probes the package first with + * `targetIncludesImportedName: true`. That recursion differs from the outer + * frame in exactly one field, whose only effect is to skip the branch, so it + * runs the outer frame's whole tail — `resolvePythonImportInternal`, the + * relative gate, `hasRepoCandidate`, `resolveAbsoluteFromFiles` — on identical + * inputs. When it returned null the code FELL THROUGH and ran all of it again. + * Measured at four directory components: 24 probes, of which 12 were + * byte-identical repeats. + * + * The gate is that `from x import y` and `import x as y` issue exactly the + * probes `import x` issues. Stated as absolute numbers rather than as + * `named === namespace`, because an equality alone also passes if BOTH kinds + * start paying twice. + * + * ## Defect 2 — no proof of absence in front of the walk + * + * The bare walk probed `/.py` and `//__init__.py` + * at every step from the importer's directory to the workspace root, for every + * single-segment import — including `import os`, `import sys` and every other + * distribution the repo does not vendor, where every probe is guaranteed to + * miss. `pythonSegmentAbsent` answers "no file anywhere can have either shape" + * in two Map lookups on the index the dotted tiers already build. + * + * The gate is that a provably-absent segment costs the SAME at depth 16 as at + * depth 1, paired with the control that a segment which survives the proof + * still walks and still costs more with depth — otherwise a resolver that + * simply stopped working would post a perfect flat line. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedImport } from 'gitnexus-shared'; +import { pythonScopeResolver } from '../../../../src/core/ingestion/languages/python/scope-resolver.js'; +import { resolvePythonImportInternal } from '../../../../src/core/ingestion/import-resolvers/python.js'; +import { + NO_PARSED_FILES, + pythonNamedImport, + pythonNamespaceImport, +} from '../../../helpers/counting-file-set.js'; + +const { resolveImportTarget } = pythonScopeResolver; + +/** Counts `has` probes. `instanceof Set` still holds, which the adapter's + * structural narrowing needs. */ +class ProbeCountingSet extends Set { + probes = 0; + + override has(value: string): boolean { + this.probes++; + return super.has(value); + } +} + +/** `import x as y`. The third kind, and the only one of the three that is not + * shared with the memo guards — it reaches the same package-attribute probe + * `pythonNamedImport` does, and both arms below assert they cost the same. */ +const aliasImport = (targetRaw: string): ParsedImport => ({ + kind: 'alias', + localName: 'w', + importedName: 'Widget', + alias: 'w', + targetRaw, +}); + +const DEPTHS: readonly number[] = [1, 2, 4, 8, 16]; + +/** + * An importer `depth` directory components down. + * + * `far/away/probe.py` is out of the importer's ancestry, so a `probe` walk runs + * to the end and misses — and it makes `probe` a known basename, so the absence + * proof passes it through. `vendor/thing.py` makes `vendor/` a root directory + * prefix, so `hasRepoCandidate('vendor')` passes on its check (2) and the + * dotted target below reaches `resolveAbsoluteFromFiles` instead of being + * gated out. + */ +function corpus(depth: number): { files: readonly string[]; fromFile: string } { + const fromFile = `${Array.from({ length: depth }, (_, i) => `d${i}`).join('/')}/mod.py`; + return { + files: [fromFile, 'zz/keep.py', 'far/away/probe.py', 'vendor/thing.py'], + fromFile, + }; +} + +function probeCount( + mkImport: (targetRaw: string) => ParsedImport, + depth: number, + targetRaw: string, +): { probes: number; result: string | readonly string[] | null } { + const { files, fromFile } = corpus(depth); + const set = new ProbeCountingSet(files); + const result = resolveImportTarget(targetRaw, fromFile, set, undefined, { + parsedFiles: NO_PARSED_FILES, + parsedImport: mkImport(targetRaw), + }); + return { probes: set.probes, result }; +} + +/** Exists as a basename, so the absence proof passes it through to the walk. */ +const PRESENT_TARGET = 'probe'; +const PRESENT_RESULT = 'far/away/probe.py'; +/** No file has basename `ghostmod.py` and no directory is named `ghostmod`. */ +const ABSENT_TARGET = 'ghostmod'; + +/** + * `2 + 2 x depth` probes in the bare walk (proximity, then two per ancestor + * step including the workspace root), then `2 + depth` in the dotted tier below + * it (two direct root probes, then one per ancestor — only the module form, + * because no `probe/__init__.py` exists anywhere). `4 + 3 x depth`. + */ +const PRESENT_PROBES: readonly number[] = [7, 10, 16, 28, 52]; +/** Two: the dotted tier's direct workspace-root probes. The bare walk issues + * NONE — it is retired before the proximity check. */ +const ABSENT_PROBES = 2; + +/** + * A DOTTED target that passes `hasRepoCandidate` (its leading segment `vendor` + * is a root directory prefix), reaches `resolveAbsoluteFromFiles`, walks the + * whole ancestor chain and still resolves to nothing — because the only + * `probe.py` in the workspace does not end with `/vendor/probe.py`. + * + * This is the shape the duplicated tail actually costs on, and the reason the + * single-segment arm above cannot see it: a single-segment target that survives + * the absence proof is always answered by the suffix fallback, so its + * `packageTarget` is never null and the fallthrough never fires. A dotted one + * can miss, and missing is precisely when the old code ran the tail again. + */ +const DOTTED_TARGET = 'vendor.probe'; +/** `2 + depth`: two direct root probes, then one module probe per ancestor. */ +const DOTTED_NAMESPACE_PROBES: readonly number[] = [3, 4, 6, 10, 18]; +/** + * `named`/`alias` legitimately add TWO — the submodule probe for + * `vendor.probe.Widget`, a different target with its own direct root checks. + * What they must NOT add is a third component: another whole copy of the + * package tail. With the fallthrough restored these read [8, 10, 14, 22, 38]. + */ +const DOTTED_SUBMODULE_PROBES: readonly number[] = [5, 6, 8, 12, 20]; + +describe('Python import probe count', () => { + it.each([ + { kind: 'import x (namespace)', mkImport: pythonNamespaceImport }, + { kind: 'from x import y (named)', mkImport: pythonNamedImport }, + { kind: 'import x as y (alias)', mkImport: aliasImport }, + ])('costs the same for every import KIND — single-segment, resolving — $kind', ({ mkImport }) => { + const counted = DEPTHS.map((depth) => probeCount(mkImport, depth, PRESENT_TARGET)); + expect(counted.map((c) => c.probes)).toEqual(PRESENT_PROBES); + + // Non-vacuity: a probe count is equally flattering to a resolver that + // resolves nothing. + expect(counted.map((c) => c.result)).toEqual(DEPTHS.map(() => PRESENT_RESULT)); + }); + + it.each([ + { + kind: 'import x (namespace)', + mkImport: pythonNamespaceImport, + expected: DOTTED_NAMESPACE_PROBES, + }, + { + kind: 'from x import y (named)', + mkImport: pythonNamedImport, + expected: DOTTED_SUBMODULE_PROBES, + }, + { kind: 'import x as y (alias)', mkImport: aliasImport, expected: DOTTED_SUBMODULE_PROBES }, + ])( + 'runs the package tail ONCE for a dotted target that misses — $kind', + ({ mkImport, expected }) => { + const counted = DEPTHS.map((depth) => probeCount(mkImport, depth, DOTTED_TARGET)); + + // The duplicated-tail gate. Restoring the fallthrough adds a second copy of + // the namespace column to the two submodule rows. + expect(counted.map((c) => c.probes)).toEqual(expected); + expect(counted.map((c) => c.result)).toEqual(DEPTHS.map(() => null)); + }, + ); + + it.each([ + { kind: 'import x (namespace)', mkImport: pythonNamespaceImport }, + { kind: 'from x import y (named)', mkImport: pythonNamedImport }, + { kind: 'import x as y (alias)', mkImport: aliasImport }, + ])('retires a provably absent segment in a CONSTANT probe count — $kind', ({ mkImport }) => { + const counted = DEPTHS.map((depth) => probeCount(mkImport, depth, ABSENT_TARGET)); + + // The gate: flat in depth. Without the proof of absence this is + // `4 + 4 x depth` for a miss, i.e. 8 at depth 1 and 68 at depth 16. + expect(counted.map((c) => c.probes)).toEqual(DEPTHS.map(() => ABSENT_PROBES)); + expect(counted.map((c) => c.result)).toEqual(DEPTHS.map(() => null)); + }); + + it('the counter can see depth — the flat line above is the proof, not the instrument', () => { + // Control for the arm above: the same instrument, the same corpus, the same + // depths, one different spelling — and the count triples across the range. + // So a flat line means the walk was skipped, not that nothing is counted. + const present = DEPTHS.map( + (depth) => probeCount(pythonNamedImport, depth, PRESENT_TARGET).probes, + ); + expect(present).toEqual(PRESENT_PROBES); + expect(new Set(present).size).toBe(DEPTHS.length); + }); + + /** + * The two inputs `pythonSegmentAbsent` refuses to answer for. Both must keep + * probing exactly as before; a proof of absence that fires on either would + * silently stop resolving real files. + */ + it.each([ + { + why: 'the EMPTY segment, module form — basename `.py` is indexed normally', + files: ['a/b/.py', 'a/b/mod.py'], + fromFile: 'a/b/mod.py', + importPath: '', + expected: 'a/b/.py', + }, + { + // THE reason the empty-segment carve-out exists. The probe for an empty + // segment is `/__init__.py`, whose parent directory name is empty + // — exactly the case the `byInitParent` build skips. So the bucket cannot + // witness this file, and its absence is not proof of the file's absence. + why: 'the EMPTY segment, package form under a doubled separator — `byInitParent` skips it', + files: ['a//__init__.py', 'a/b/mod.py'], + fromFile: 'a/b/mod.py', + importPath: '', + expected: 'a//__init__.py', + }, + { + why: 'the EMPTY segment, package form at the filesystem root', + files: ['/__init__.py', 'a/b/mod.py'], + fromFile: 'a/b/mod.py', + importPath: '', + expected: '/__init__.py', + }, + { + why: 'a segment carrying a BACKSLASH — the buckets are keyed on normalized paths', + files: ['a\\b.py', 'x/mod.py'], + fromFile: 'x/mod.py', + importPath: 'a\\b', + expected: 'a\\b.py', + }, + ])('still resolves what the proof of absence cannot rule out — $why', (row) => { + expect(resolvePythonImportInternal(row.fromFile, row.importPath, new Set(row.files))).toBe( + row.expected, + ); + }); +}); diff --git a/gitnexus/test/unit/scope-resolution/python/python-importer-ancestors.test.ts b/gitnexus/test/unit/scope-resolution/python/python-importer-ancestors.test.ts new file mode 100644 index 000000000..f29ac144f --- /dev/null +++ b/gitnexus/test/unit/scope-resolution/python/python-importer-ancestors.test.ts @@ -0,0 +1,291 @@ +/** + * Gate for #2913: Python import resolution must not scale with the importer's + * path depth. + * + * `hasRepoCandidate` and `resolveAbsoluteFromFiles` each rebuilt one ancestor + * prefix per component of the importer's directory, on EVERY import — a cost + * proportional to depth (quadratic in characters) on an index that is itself + * depth-free. `importerAncestors` builds that chain once per importer DIRECTORY + * and stores it in `PythonFileIndex.ancestorsByDir`, which lives inside the + * per-file-set value and so dies with the pass. + * + * ## Why this is not `CountingSet` + * + * `test/helpers/counting-file-set.ts` is the house instrument for every other + * import-target reuse guard, and it cannot see this one. It counts TRAVERSALS + * of the file set; the ancestor chain is derived from the `fromFile` STRING and + * touches the set only through `Set.has`, whose argument and count are byte-for- + * byte identical before and after the hoist. The same is true of a `has`-call + * counter: memoizing a string that is then concatenated into the same probe + * changes no probe. A pure hoist is invisible to any instrument that watches + * only the resolver's inputs — so this file watches the memo, which is the one + * place the hoist is observable, and watches it THROUGH the production adapter + * (`pythonScopeResolver.resolveImportTarget`, the surface the orchestrator + * calls) rather than through the resolver function the parity test uses. + * + * The gate is a COUNT, not a timing budget: `ancestorsByDir.size` after N + * imports from D directories must be D, for every N. That is exactly "the + * ancestor-prefix work is O(1) amortized after the first import from a given + * directory", stated as a number a test can assert. It is paired with a + * reference-identity assertion, because a memo that stores a FRESH chain on + * every import posts the same size while doing all of the work again. + * + * Nothing ships for this file to read. `getPythonFileIndex` is the pass's own + * index and `ancestorsByDir` is the memo itself; the export is visibility, not + * a counter — the surface #2909 deleted was ~30 lines of production code whose + * only caller was a test. The module barrel (`languages/python/index.ts`) is + * unchanged, so the index stays out of the package's public API. + * + * The `legacy*` helpers below are verbatim copies of the pre-#2913 inline code, + * in the house style of `import-target-index-parity.test.ts`: they are the + * specification, and the memo agreeing with them is what makes this a hoist + * rather than a behaviour change. + * + * The four memo arms live in `counting-file-set.ts` beside the other + * import-target scaffolding, because this guard and the bare-prefix one + * (`test/unit/import-resolvers/python-importer-prefixes.test.ts`) are the same + * suite over the same importer corpus once four values are named (the memo, the + * drive, the legacy builder, the hit). The two CHAINS still differ — this one + * drops the empty components an absolute path or a doubled separator produces + * and the other keeps them — so `legacyChain` stays per-guard and the shared + * path-shape table names shapes rather than expectations. + */ +import { describe, expect, it } from 'vitest'; +import type { ParsedImport } from 'gitnexus-shared'; +import { pythonScopeResolver } from '../../../../src/core/ingestion/languages/python/scope-resolver.js'; +import { getPythonFileIndex } from '../../../../src/core/ingestion/import-resolvers/python-file-index.js'; +import { + IMPORTER_PATH_SHAPES, + countedParsedFiles, + expectDistinctFileSetsGetOwnChainMemo, + expectMemoizedChainMatchesLegacy, + expectOneChainPerImporterDir, + expectSameChainObjectReused, + sortedStrings, + type ChainMemoArm, + type ChainMemoResult, +} from '../../../helpers/counting-file-set.js'; + +const { resolveImportTarget } = pythonScopeResolver; + +// ─── verbatim pre-#2913 implementations ────────────────────────────────────── + +/** The `ancestorPrefixes` array `hasRepoCandidate` used to build per import. */ +function legacyAncestorPrefixes(fromFile: string, leadingSegment: string): string[] { + const importerDir = fromFile.replace(/\\/g, '/').split('/').slice(0, -1).join('/'); + const dirParts = importerDir ? importerDir.split('/').filter(Boolean) : []; + const ancestorPrefixes: string[] = []; + for (let i = dirParts.length; i > 0; i--) { + ancestorPrefixes.push(`${dirParts.slice(0, i).join('/')}/${leadingSegment}/`); + } + return ancestorPrefixes; +} + +/** The ancestors `resolveAbsoluteFromFiles`'s walk used to build per import. */ +function legacyAncestorChain(fromFile: string): string[] { + const importerDir = fromFile.replace(/\\/g, '/').split('/').slice(0, -1).join('/'); + const chain: string[] = []; + const dirParts = importerDir ? importerDir.split('/').filter(Boolean) : []; + for (let i = dirParts.length; i > 0; i--) { + chain.push(dirParts.slice(0, i).join('/')); + } + return chain; +} + +/** The forward, no-early-exit `dirPrefixes` build. */ +function legacyDirPrefixes(files: readonly string[]): Set { + const dirPrefixes = new Set(); + for (const raw of files) { + const norm = raw.replace(/\\/g, '/'); + if (!norm.endsWith('.py')) continue; + const lastSlash = norm.lastIndexOf('/'); + for (let i = 0; i <= lastSlash; i++) { + if (norm[i] === '/') dirPrefixes.add(norm.slice(0, i + 1)); + } + } + return dirPrefixes; +} + +/** + * The specification of `nestedDirNames`, read off the legacy prefix set: the + * name of every directory prefix that has a NON-EMPTY parent, which is exactly + * the set of `${ancestor}/${segment}/` shapes the old ancestor loop could ever + * match. A segment outside it made the old loop run to completion and answer + * false; the new code answers false without running it. + */ +function specNestedDirNames(dirPrefixes: ReadonlySet): Set { + const names = new Set(); + for (const prefix of dirPrefixes) { + const dir = prefix.slice(0, -1); + const slash = dir.lastIndexOf('/'); + if (slash > 0) names.add(dir.slice(slash + 1)); + } + return names; +} + +// ─── the workspace the adapter is driven against ───────────────────────────── + +/** + * `outer/nested/` is what makes `nested` a NESTED directory name without making + * `nested/` a root prefix, so `hasRepoCandidate('nested')` has to reach the + * ancestor walk instead of answering from check (1) or (2). `one.py` repeats + * across three directories so the `outer.one` spelling reaches + * `resolveAbsoluteFromFiles`'s walk too — both memo call sites, one corpus. + */ +const WORKSPACE: readonly string[] = [ + 'outer/nested/mod.py', + 'svc/a/one.py', + 'svc/a/two.py', + 'svc/b/one.py', + 'deep/x/y/z/one.py', + 'root.py', +]; + +/** + * One import that must resolve, so a memo count is never the count of an + * adapter that has stopped resolving anything (the pairing rule every guard in + * this family states). `outer/` is a root directory prefix, so the gate passes + * on check (2) and the direct workspace-root hit answers it. + */ +const HIT_TARGET = 'outer.nested.mod'; +const HIT_RESULT = 'outer/nested/mod.py'; + +/** + * Drives the ORCHESTRATOR ADAPTER `perImporter` times from `fromFile`, with two + * spellings that between them enter the memo from both call sites: + * - `nested.ghost{i}` — reaches `hasRepoCandidate`'s ancestor walk and misses. + * Spelled differently every iteration, so nothing upstream can answer it + * from a per-target memo. + * - `outer.one` — passes the gate on check (2) (`outer/` is a root directory + * prefix), misses the direct workspace-root hit, and reaches + * `resolveAbsoluteFromFiles`'s ancestor walk. NOT varied per iteration: + * `one.py` has to be a real basename somewhere or the walk is skipped + * before it starts, and the Python chain keeps no per-target cache, so a + * repeated spelling really is re-resolved. + * …then the one spelling that must resolve. + */ +function driveImporter( + files: Set, + fromFile: string, + perImporter: number, +): ChainMemoResult[] { + const out: ChainMemoResult[] = []; + for (let i = 0; i < perImporter; i++) { + out.push(resolveImportTarget(`nested.ghost${i}`, fromFile, files, undefined, undefined)); + out.push(resolveImportTarget('outer.one', fromFile, files, undefined, undefined)); + } + out.push(resolveImportTarget(HIT_TARGET, fromFile, files, undefined, undefined)); + return out; +} + +const ancestorArm: ChainMemoArm = { + memoOf: (files) => getPythonFileIndex(files).ancestorsByDir, + drive: driveImporter, + legacyChain: legacyAncestorChain, + hitResult: HIT_RESULT, +}; + +describe('Python importer-ancestor memo (#2913)', () => { + it.each([ + { perImporter: 1, label: 'one import per importer' }, + { perImporter: 40, label: 'forty imports per importer' }, + ])('holds one chain per importer DIRECTORY, not per import — $label', ({ perImporter }) => { + expectOneChainPerImporterDir(ancestorArm, new Set(WORKSPACE), perImporter); + }); + + it('reuses the SAME chain object, rather than rebuilding and re-storing it', () => { + expectSameChainObjectReused(ancestorArm, new Set(WORKSPACE)); + }); + + it.each(IMPORTER_PATH_SHAPES)( + 'memoizes the chain the pre-#2913 code built — $why', + ({ fromFile }) => { + const chain = expectMemoizedChainMatchesLegacy(ancestorArm, new Set(WORKSPACE), fromFile); + + // Both consumers' chains, from the one memo: `resolveAbsoluteFromFiles` + // walked these directories, `hasRepoCandidate` walked the same directories + // with `//` appended. + expect(chain.map((ancestor) => `${ancestor}/nested/`)).toEqual( + legacyAncestorPrefixes(fromFile, 'nested'), + ); + }, + ); + + it.each([ + { why: 'relative paths sharing directories', files: WORKSPACE }, + { why: 'absolute paths', files: ['/repo/pkg/__init__.py', '/repo/vendor/pkg/thing.py'] }, + { why: 'a doubled separator', files: ['a//b/x.py', 'a//b/y.py'] }, + { why: 'Windows separators', files: ['a\\b\\x.py', 'a\\b\\c\\y.py'] }, + { why: 'root-level files only', files: ['x.py', 'y.py'] }, + { why: 'a polyglot corpus', files: ['a/b/x.py', 'a/b/x.ts', 'c/d/e/f/g/h.py', 'c/d/n.go'] }, + { why: 'one deep directory, many files', files: ['a/b/c/d/e/1.py', 'a/b/c/d/e/2.py'] }, + ])('builds the same prefix set as the pre-#2913 forward scan — $why', ({ files }) => { + const index = getPythonFileIndex(new Set(files)); + const legacy = legacyDirPrefixes(files); + + // The build now walks separators from the deepest outward and stops at + // the first prefix already present. Skipping the rest is only sound + // because a prefix is always stored with all of its own ancestors. + expect(sortedStrings(index.dirPrefixes)).toEqual(sortedStrings(legacy)); + expect(sortedStrings(index.nestedDirNames)).toEqual(sortedStrings(specNestedDirNames(legacy))); + }); + + /** + * The same defect on the OTHER collection the orchestrator threads. + * `pythonFileExportsName` opened with `parsedFiles.find(...)`, an O(files) + * scan run for every import whose package probe resolves — which on a repo + * where `from pkg import X` usually resolves is most imports. + * + * `import-target-index-reuse.contract.test.ts` measures this channel for + * every language, but its Python fixture has exactly ONE resolving import, so + * its equality arm passes whether the scan is memoized or not. This arm is + * the one that bites: every import resolves through the probe, so a per-import + * `find` makes the read count grow with the import count. + */ + it.each([ + { imports: 2, label: 'two imports' }, + { imports: 200, label: 'two hundred imports' }, + ])('reads the parsed workspace once per PASS, not once per import — $label', ({ imports }) => { + const modules = Array.from({ length: 30 }, (_, i) => `pkg/m${i}.py`); + const paths = ['pkg/__init__.py', ...modules, 'app/main.py']; + const workspace = countedParsedFiles(paths); + const files = new Set(paths); + const resolved: ChainMemoResult[] = []; + + for (let i = 0; i < imports; i++) { + const targetRaw = `pkg.m${i % modules.length}`; + const parsedImport: ParsedImport = { + kind: 'named', + localName: 'Widget', + importedName: 'Widget', + targetRaw, + }; + resolved.push( + resolveImportTarget(targetRaw, 'app/main.py', files, undefined, { + parsedFiles: workspace.parsedFiles, + parsedImport, + }), + ); + } + + // One pass over the parsed workspace, whatever the import count. A `find` + // per import reads 32 for two imports and thousands for two hundred. + expect(workspace.reads()).toBe(paths.length); + // ...and the leg was really entered, so the count is not a perfect zero + // posted by a resolver that returned early. + expect(workspace.reads()).toBeGreaterThan(0); + expect(resolved[0]).toBe('pkg/m0.py'); + expect(resolved[resolved.length - 1]).toBe(`pkg/m${(imports - 1) % modules.length}.py`); + }); + + it('gives a distinct file set its own memo (no leak across passes)', () => { + const a = new Set(WORKSPACE); + const b = new Set(WORKSPACE); + + expectDistinctFileSetsGetOwnChainMemo(ancestorArm, a, b, 2); + + // The whole per-file-set index, not only the memo inside it: the WeakMap is + // keyed on the Set, so two Sets can never share one index. + expect(getPythonFileIndex(a)).not.toBe(getPythonFileIndex(b)); + }); +});