diff --git a/gitnexus/bench/import-target/baselines.json b/gitnexus/bench/import-target/baselines.json index ee177c780..9dd4da253 100644 --- a/gitnexus/bench/import-target/baselines.json +++ b/gitnexus/bench/import-target/baselines.json @@ -2,7 +2,7 @@ "_what": "Baselines for bench/import-target/measure.mjs \u2014 EVERY import-target resolver registered in SCOPE_RESOLVERS, on one shared corpus, plus csharp a second time WITH csproj configs. One entry per registered language and one more for the csproj arm, no registered language ungated \u2014 and that is ASSERTED rather than asserted-in-a-comment, which is also why no roster of language names is kept in this prose to go stale: measure.mjs derives its language list from a LANG_REGISTRY table and a --check inventory arm reconciles that table against SCOPE_RESOLVERS in both directions. A C/C++ #include is an import site for this purpose and is gated like every other registered language. csharp and csharp_csproj resolve the IDENTICAL file corpus (buildFiles aliases the two) and differ in exactly one thing: whether csharpConfigs is supplied. Without that second arm the csproj namespace-directory index ships unmeasured, because every C# import in the no-csproj arm returns before reaching it. C and C++ follow that same precedent for a different context \u2014 their HEADERS arrive through resolutionConfig rather than through allFilePaths, and augmentedFilePaths unions the two once per pass, so the corpus is split at newPass rather than pre-merged. The first nine were added as their own O(imports x files) scans were indexed away (#2877/#2878/#2879/#2880, #2872, #2901, #2902, #2908) and this is the forward guard on each; the other eight were ungated until now, and PR #2911 \u2014 JavaScript reaching suffixResolve with no index at all, 25972 us per import at 8000 files \u2014 is what that costs.", "_fingerprint_note": "Per-language sha256 over every distinct fromFile|target -> resolved target. A change here is a BEHAVIOUR change: the resolver returned a different target set, and IMPORTS/CALLS edges moved. Explain it, never re-baseline to make CI green. For the languages these PRs changed, the pre-change implementations produce these same values on this corpus at both 400 and 1600 files \u2014 that is what makes the index hoist a performance change. The tie-break-level proof lives in test/unit/scope-resolution/import-target-index-parity.test.ts (verbatim copies of the pre-change code, diffed) for Kotlin in test/unit/scope-resolution/kotlin/kotlin-import-target-parity.test.ts, and for the four resolvers added there in test/unit/scope-resolution/{php,java,cobol}-import-target-parity.test.ts and test/unit/import-resolvers/csharp-csproj-parity.test.ts, and for JavaScript in test/unit/scope-resolution/javascript-import-target-parity.test.ts (a differential over 211200 old-vs-new pairs, PR #2911). The eight languages added last have no per-language parity harness against a pre-change implementation and do NOT need one: nothing about their resolution changed, so there is no before to diff against. Their fingerprints are pure forward guards, minted from the current implementations, and their adapter-boundary index reuse is covered for every registered language at once by test/unit/scope-resolution/import-target-index-reuse.contract.test.ts. NOTE for csharp_csproj: on this corpus the #2902 indexed leg (step 3 of resolveCSharpImportInternal) is reached by 2221 of the 3200 small-arm imports but answers null for every one of them \u2014 the 979 that resolve do so at step 2 \u2014 so this fingerprint pins that legs cost and its null answers, while its positive tie-breaks (first-occurrence, unanchored substring, iteration order) are pinned by csharp-csproj-parity.test.ts.", "_shape_note": "files/imports/resolved/distinct_outcomes AND the fingerprint are asserted exactly, per scale. A fingerprint alone cannot tell a legitimate resolution change from a corpus quietly shrunk below the size at which the timing arms can see anything; conversely the counts alone cannot see a defect confined to one arm, because the arms differ only in path padding and directory layout and both of those are count-neutral by design. Two cross-arm assertions close the remaining hole: the deep and collide arms must resolve exactly what small resolves (they are the same workload), and each of their fingerprints must DIFFER from small's (they are not the same corpus). Without the second, setting DEEP_PAD to 0 \u2014 which deletes the entire depth arm \u2014 moves no asserted number and prints PASS; the same is true of a collideDir that forwards to uniqueDir. THE HEAP ARM IS ASSERTED THE SAME WAY, by the same loop, and was not before: files_small, files_large, path_segments and probe decide WHAT it measures, and every one of them was reported and compared to nothing. Swapping HEAP_PROBE_TARGET.csharp_csproj for a target matching no CSPROJ_CONFIGS rootNamespace skips the whole config loop, so the getFilesInDir and getInsensitive legs never run and the arm the header calls the witness that the read pattern IS the footprint quietly becomes a two-map arm \u2014 73703384 -> 59921216 B, ratio 1.017 -> 1.011, ceiling and floor both still passing and --check still exiting 0. Setting HEAP_SMALL equal to HEAP_LARGE is the same hole from the other side: ratio goes to ~1.0 by construction and bytes_large never moves. bytes_small and bytes_large are deliberately NOT asserted for equality \u2014 heap_ceiling_bytes and the heap_reading_bytes floor bound them with ~50% either way, because heapUsed accounting moves across platforms and Node majors and an exact byte assertion would be a re-baseline per runner. THE CONTEXT ARM IS ASSERTED THE SAME WAY, by the same loop, and more strictly than either: target, with_context and without_context are exact strings with no tolerance at all, because the arm resolves one import over a three-file corpus and has no measurement noise to tolerate. A separate check requires the last two to DIFFER, for the same reason deep.fingerprint must differ from small.fingerprint \u2014 a probe on which both call shapes agree asserts one number twice. Both halves run through resolveOne, so what the arm gates is this bench threading run.ts's fifth argument, not the resolvers' behaviour.", - "_arms_note": "Five timing arms, one memory arm and one deterministic arm elsewhere, because none of them gates alone. scaling_ratio (t_large/t_small)/(1600/400) catches cost growing with FILE COUNT \u2014 the #2877-#2880, #2901, #2902 and #2908 regressions themselves; every one of those legs was Theta(files) per import, so a revert scores ~4 here by construction. depth_ratio (t_deep/t_small at a FIXED file count, ~6x the path components) catches cost growing with path DEPTH, which scaling_ratio divides out and structurally cannot see; buildSuffixIndex (C#, Ruby, PHP, Java) and Kotlin suffixByStem emit one entry per component, so they legitimately sit above 1.0 while Go, Dart and COBOL, whose indexes are depth-free, sit at ~1.0. csharp's depth_budget has now been retightened twice for the same reason, and the second time it did lock the win in. It was 5 against a then-measured 3.318; #2903 made buildSuffixIndex's dirMap lazy and it became 3.5 against 2.31, with the file stating plainly that 3.5 did NOT lock that win in because a revert to an eager dirMap scores 3.318 and passes. Extending the laziness to the two SUFFIX maps drops it again, to 1.438 (java likewise 2.214 -> 1.402), because the deep arm has ~6x the path components and an O(files x depth) build of a map the no-csproj leg never reads is exactly the cost that scales with depth. Both are now 2.2, which is this file's 1.5x convention against measurements whose own peak-to-peak over 4 runs is 1.04x and 1.07x \u2014 and 2.2 DOES lock it in: an eager rebuild scores 2.3+ and fails. The other fifteen depth budgets sit at 1.37-1.75x measured and are unchanged. collide_scaling_ratio is the same measurement on a SHARED-LEAF layout (svcN/internal, SrcN/Models, com/example/model in every service, a repeated mod0.dart/mod0.rb/Mod0.cpy basename) carrying an identical file, import and resolved count: the small/large/deep arms mint one directory name per index, so every index bucket in them holds exactly ONE entry (measured: max last-segment bucket 1 and max matching directories 1 for go and csharp at 400 and 1600 files; max basename bucket 1 for dart and ruby), and bucket cardinality is the only non-constant term the new indexes have. On the shared-leaf shape go, csharp, dart and java legitimately score 2.1-3.9 because the bucket grows with the file count BY CONSTRUCTION \u2014 this is a limit on the SCOPE of the \"independent of corpus size\" claim, not a regression (the indexed code is still faster there than the pre-change full scan); their collide budgets say so honestly instead of pretending 1.8. Ruby, Kotlin, PHP and COBOL answer from keyed maps and are collision-immune, so they keep the linear 1.8 budget and that immunity is the assertion. csharp_csproj is the one arm that runs the other way: its shared leaf collapses dirsByLastSegment to the single key Models, so the slash-free sweep (see CSPROJ_CONFIGS) is CHEAPER on the collide layout than on the unique one and its expensive scale arm is large, not collide_large. Its 1.8 collide budget is therefore the linear one, and the arm that carries its real cost is the unique one. The collide arm is also the only arm that reaches filesDirectlyInPkgDir's dirCount > 1 merge (go: 388 multi-directory calls at 400 files, up to 9 directories; 1517 at 1600 files, up to 34) and the only one that reaches COBOL's copybook-over-source tier tie-break, which needs one bookname to name two files. small_ms_ceiling and collide_ms_ceiling are ABSOLUTE (~4x the measured arm), because a constant-factor regression that grows both scale arms equally passes every ratio. The five arms added here use 4.2x, the middle of the 3.7-4.6x the original five already carry; the two COBOL arms use ~5x, the multiplier dart's sub-1 ms arm has always carried, because a fixed scheduler hiccup is a larger fraction of a smaller number \u2014 measured over 8 runs they sat at 0.25-0.37 ms and 0.18-0.30 ms, and the pre-#2908 two-scans-per-COPY implementation costs ~300 ms on the same arm, so 2.0 and 1.5 still separate fixed from broken by two orders of magnitude. NOISE, measured rather than assumed: depth_ratio divides two sub-3 ms numbers (Dart's are sub-1 ms) and is by far the noisiest arm here, so it set N for the whole file. fastest() is a min-of-N estimator, so N is the knob. Over 22 --check runs on an idle box, peak-to-peak: at N=5 go ran 0.757-1.748 (2.31x) and tripped its own 1.6 budget about 1 run in 20; at N=7 (the kotlin-import-target setting) Dart still ran 0.678-2.043 (3.01x) and tripped once; at N=15 (bench/cfg, bench/schema-pairs, bench/callable-value-flow) every language collapsed to a 1.13-1.26x swing with 22/22 passing. The budgets were NOT widened; the estimator was fixed instead, which is why the headroom above is real rather than granted. N IS NOW PER LANGUAGE, and that is a refinement of the same finding rather than a retreat from it. The overshoot of min-of-K against min-of-15 is a function of the CELL's absolute duration, not of the language: replayed against two independent runs' full sample sets, the worst overshoots at K=7 land on swift.small (0.43 ms, 31.8%) and dart.collide (1.5 ms, 37.6%), while every cell at or above 10 ms overshoots by at most 6.3%. So repsFor() keeps 15 while a language's cheapest arm is under 5 ms and otherwise spends ~150 ms per cell, floored at 7 \u2014 15 for go, csharp, dart, kotlin, java, cobol, swift, rust, python, c and cpp (every language the flakiness above was ever about, cheapest arm 0.19-3.2 ms) and 7-8 for csharp_csproj, ruby, php, javascript, typescript and vue (cheapest arm 20-28 ms). Per LANGUAGE, not per cell, so all five arms of a language share one estimator and the four ratios stay comparisons of like with like. The replay passed all 85 cells on all five gates at 0.4-0.7 of budget and saved 12.8 s and 12.4 s of a 46 s run; min-of-7 also reads slightly HIGHER than min-of-15, so the ceilings get marginally more sensitive rather than less. Confirmed on 4 fresh runs with the adaptive estimator live: every small arm inside 1.12x peak-to-peak and every collide arm inside 1.07x, with the six 7-8 rep languages at 1.008-1.071 \u2014 no worse than the 11 that kept 15. The chosen N is reported per language as `reps`. heap_ceiling_bytes bounds the retained per-pass import index, the only arm here that can see memory: buildSuffixIndex emits maps at O(files x depth), the profile package-dir-index.ts cites #2649 to avoid for itself, and csharp, ruby, php and java all retained NOTHING across imports at BASE (C#'s no-csproj leg and PHP's and Java's every leg re-scanned the raw Set; Ruby rebuilt and discarded a suffix index per require). It is measured at 8000 and 32000 files at HEAP_PAD depth rather than at the timing arms' sizes, because the finding is an ABSOLUTE footprint at repository scale. THE ARM NOW READS WHAT THE LANGUAGE READS, and that change is the whole reason this file was re-baselined. Four of these arms used to call getWorkspaceFileIndex(set) directly and then read index.all.length, which asks no suffix question at all \u2014 harmless only while buildSuffixIndex built both maps eagerly. The moment they went lazy the direct call built NO map, csharp, ruby, php and java each reported 0 B at 32000 files, and 0 B is under every ceiling: --check printed PASS over four gates that had silently become ceilings over nothing, which is precisely the failure this file's own header warns about for rust and cobol. Every arm now resolves a real MISSING import through the real resolver (HEAP_PROBE_TARGET, asserted to miss), so the maps it forces are the maps production forces, and a resolver that starts asking a new question moves the number without anyone editing the bench. That makes the READ PATTERN the dominant term, and the eight numbers say so: java 34958600 B and csharp 29862200 B ask index.get and never getInsensitive; php 37579888 B asks getInsensitive and never get, plus its own first-proper-suffix map; ruby 41025360 B and javascript 26745296 B read get(s) || getInsensitive(s) and pay for both, the second DERIVED from the first; and csharp_csproj 73705944 B additionally asks getFilesInDir. csharp_csproj IS NOW GATED, reversing the earlier decision that it would be 'a ceiling on a duplicate': at +20.8% of the C# index it was one, and at 2.47x of it \u2014 same corpus, same getWorkspaceFileIndex, three maps instead of one \u2014 it is the witness that the read pattern is the footprint. The old RESIDUAL note is superseded by that number: a dirMap-sized addition is no longer +18%, and a consumer that asks all three questions blows csharp's ceiling by 1.64x rather than sliding under it. A SECOND MEASUREMENT BIAS was removed at the same time and it moved every figure here, so do not read these against the old ones as if only the read pattern changed. buildFiles mints paths with template literals, which V8 keeps as ropes; the first traversal that slices one flattens it, allocating the flat string and dropping the rope's pieces, so a build measured over an unflattened corpus reports the index MINUS that net release \u2014 11% low, uniformly. bytes_small was read over a corpus a discarded warm-up pass had already flattened and bytes_large over a fresh one, so every ratio read ~0.85-0.89 for structures that are exactly linear in the file count. measureHeap now flattens each corpus before measuring it; all eight ratios read 0.998-1.017, and the warm-up pass is gone because with the corpus flat a language's first and second reads agree to within 0.3%. python's figure rises from 7624992 to 10362976 for this reason and not because anything regressed, and then to 10543152 (+1.7%) because #2913's nestedDirNames set is retained for the pass — the ancestorsByDir memo itself is NOT in this reading, because python's probe target misses at the nested-name rejection and never reaches the walk, so this arm does not bound that memo; measured separately with a probe that does reach it, a 32000-file corpus with every file in its own 10-deep directory retains ~19 MB, which would clear this ceiling, so repointing python's heap probe at a walking spelling means re-recording the ceiling in the same change, and c is unchanged at 10018816 because its basename map does not slice paths. Its ceiling is 1.5x the measured arm, and the DIFFERENCE FROM THE 4x TIMING CONVENTION IS DELIBERATE \u2014 do not harmonise it back. 4x exists because runner contention dominates a wall-clock number; this one has essentially no measurement noise (across 4 runs the widest spread was 0.11% on python, 0.03% on csharp_csproj and 0.00% \u2014 identical to the byte \u2014 on ruby, php, java, javascript and c, and the same holds across separate processes), so 4x would throw away almost all of the gate's power and sail straight past the regression this arm exists to catch. 1.5x still tolerates ~50% of cross-platform and Node-version drift, far more than a Node major bump plausibly moves heapUsed accounting; it catches a duplicated index (+100%) or a second exactMap-sized suffix map (+~85%). heap_floor_fraction is the arm the 0 B incident proved was missing. A ceiling can only say 'not too big'; nothing said 'still measuring something', which is why four dead arms passed. The floor is 0.5 x each language's RECORDED READING (heap_reading_bytes), which is half the measured size and says so. It used to be 0.33 x the CEILING, described the same way \u2014 true only while every ceiling stayed at exactly 1.5x its reading, a convention this file states and nothing enforces, so re-tuning one ceiling upward would have loosened that language's floor by the same factor in the one direction a floor exists to watch. The two forms agree to within 0.8% for all eight today, so this is a correction of derivation, not of strength. It sits ~400x above the readings' own reproducibility and far below any collapse. A genuine 2x memory WIN trips it too, and that is intended: like a fingerprint move, it must be explained and re-baselined rather than absorbed. COBOL is left out for the opposite reason: its index is two Map, O(files) with no depth term, and at 32000 files its retained delta does not clear the noise of the measurement itself. heap_ratio_budget, the linear-growth check across the 4x file-count gap, is the orthogonal arm: it sees per-file and per-depth growth but not a constant factor. ---- THE EIGHT LANGUAGES ADDED LAST (swift, rust, python, javascript, typescript, vue, c, cpp) ---- They carry the SAME five arms and the same gates; what differs is which arm can actually fail for each, because each resolver has a different cost axis, and the budgets below say so instead of copying a number across. Every figure quoted is the MAXIMUM over 5 full runs on an idle box, and the peak-to-peak of every one of these arms stayed inside 1.10x over those runs \u2014 tighter than the 1.13-1.26x the original nine record, because none of these arms divides two sub-1 ms numbers the way dart depth_ratio does. depth_budget is ~1.5x measured throughout: swift 2.3 (1.487), rust 2.1 (1.377), javascript 2.1 (1.376), typescript 2.1 (1.381), vue 2.3 (1.563), c 3.0 (1.990), cpp 3.0 (1.999). PYTHON WAS 11 AGAINST 7.389 AND IS NOW 2.6 AGAINST 1.872, because #2913 fixed the resolver rather than the budget. Its INDEX was always depth-free; hasRepoCandidate and resolveAbsoluteFromFiles each rebuilt one ancestor prefix per directory component of the importer on EVERY import, and the index's own dirPrefixes build inserted one entry per component per file, so the resolver was quadratic in path depth where every other language here is linear or flat. The prefixes are a pure function of the importer's DIRECTORY, so they are now memoized per directory inside getPythonFileIndex (ancestorsByDir), the leading segment is rejected up front against a set of nested directory names, the module and package buckets are consulted before the walk rather than inside it, and the dirPrefixes build stops at the first ancestor already stored. All five fingerprints are byte-identical, so it is a hoist. The budget is 2.2, and BOTH numbers behind it were re-measured on a quiet box AFTER the context leg below started being measured, because that change moved the arm: the work it adds is depth-FLAT, so python's absolute cost more than doubled while depth_ratio FELL to 1.405-1.563 over 5 serial runs (peak-to-peak 1.11x). A budget carried over from before that change would have been slack against a smaller ratio. 2.2 is 1.41x the measured maximum, inside the 1.37-1.75x band the other fifteen sit in, and it LOCKS THE WIN IN: reverting the per-directory ancestor memo alone scores 2.524 and reverting the nested-name rejection alone scores 2.553, both measured under the current call shape, so each fails at 2.2 with 13% to spare. Do not read those two figures as the pre-#2913 cost — 7.239 was that, and the gap closed because the bare-import tier stopped walking at all (see below). The other two parts of the fix are not gated by this arm and are not meant to be: reverting the bucket prune or the dirPrefixes early break lands under any budget this arm's noise supports, so they are gated deterministically instead, by the prefix-parity and package-probe arms of test/unit/scope-resolution/python/python-importer-ancestors.test.ts and python-import-target-parity.test.ts, which go red on exactly those two mutations. A timing budget catches what it can measure; the counts catch the rest. THE BARE-IMPORT TIER (`import os`, single segment, no dot) was a separate O(depth) walk in import-resolvers/python.ts that this bench cannot see at all, because every python arm here spells its imports with a dot and returns at the `pathLike.includes('/')` guard before reaching it. It ran TWICE per `from x import y` — the package probe's recursion re-ran the whole tail on identical inputs — and is now one memoized chain plus an O(1) proof-of-absence against the index's basename buckets: 12/24/72 Set probes at depth 1/4/16 became a flat 2, and 11.615 us/import at 18 path components became 0.740. Gated by probe COUNT in test/unit/scope-resolution/python/python-import-probe-count.test.ts, not here. collide_scaling_budget splits three ways. Three languages scan a bucket that grows with the corpus and get their measured value x1.5: swift 4.9 (3.279 \u2014 its bucket is the module file list it RETURNS, and its collide arm is four modules instead of dirs of them so that bucket is fileCount/4, i.e. 100 files at 400 and 400 at 1600), c 3.8 (2.535) and cpp 4.0 (2.639, the same basename bucket its suffix fallback walks). Four answer from keyed maps and keep the linear 1.8 \u2014 python 1.097, javascript 1.083, typescript 1.053, vue 1.079 \u2014 and that immunity IS the assertion, exactly as for ruby, kotlin, php and cobol. RUST IS THE ONE ARM THAT WAS REDESIGNED RATHER THAN BUDGETED. It resolves by probing candidate paths with allFilePaths.has(...) and never searches, so its cost is O(path segments) and provably flat in the file count (1.095 scaling, 1.061 collide scaling): a shared-leaf collide arm for rust would have asserted nothing, which is worse than no arm. Its collide corpus is instead a deep module tree (src/l0/l1/l2/l3/l4/mod{d}) whose targets carry ~2x the :: segments, so the arm exercises the axis that CAN grow, its 1.8 budget asserts the flatness across file counts, and collide_ms_ceiling 19 bounds the absolute cost of the long-path probe. small_ms_ceiling and collide_ms_ceiling are ~4x measured as everywhere else: rust 10/19 (2.609/4.704), python 7/8 (1.76/1.929, retightened from 12/15 against 3.044/3.771 by #2913), javascript 85/89 (21.254/22.145), typescript 85/86 (21.250/21.464), vue 81/93 (20.164/23.227), c 7/11 (1.620/2.850), cpp 7/12 (1.581/3.009). Swift takes ~5x (2 against 0.421 and 4 against 0.821) \u2014 the multiplier dart and cobol already carry, because a fixed scheduler hiccup is a larger fraction of a sub-1 ms number. ONE CAVEAT ON THE THREE ts-FAMILY MS NUMBERS, stated because nothing else in this file would reveal it: resolveTsTarget carries a per-pass resolveCache keyed currentFile::importPath, which no other resolver here has, and ~10% of this corpus is repeat pairs. Their us/import is therefore a slight underestimate of a cold resolve. It is left in rather than defeated because it is what the real pipeline does, and it is identical across all three so the arms stay comparable. HEAP for the eight: rust, swift, typescript, vue, cpp and cobol are still NOT gated, all of them measured before being left out. rust builds no index on this hook (16 B at 8000 files, 0 B at 32000); swift holds one pointer per file-times-segment and mints no strings, reading 0.98 MB at 8000 files against 0.29 MB at 32000 \u2014 a 4x larger corpus reading 3x SMALLER, which is what a measurement below its own noise floor looks like, and the same reading cobol gives (0.54 MB then 0 B); typescript and vue duplicate javascript through the same builder over the same-shaped corpus, and cpp duplicates c (10021320 against 10016960, 0.04% apart). Those four duplications are the ONLY exclusions that still rest on 'it would be a duplicate', and they are duplicates of a builder AND of a read pattern, which is the pairing csharp_csproj failed once the read pattern started to matter \u2014 if any of the four ever diverges in what it ASKS the index, it earns an arm the same way csharp_csproj just did. All eight gated arms are read the same way now (retainedPassBytes, one real import), so unlike before they are directly comparable to one another. WALL CLOCK \u2014 ~33-35 s in report mode, down from ~46 s, and ~44-45 s for --check, which is essentially UNCHANGED from ~46 s. Only report mode got faster; do not read the pair as 46 -> 42. The breakdown is worth having before anyone trims it. Timing arms: go 2.02, csharp 1.09, csharp_csproj 3.22, dart 0.41, ruby 2.90, kotlin 0.85, php 3.46, java 1.57, cobol 0.09, swift 0.46, rust 0.85, python 1.22, javascript 3.23, typescript 2.72, vue 2.89, c 0.86, cpp 0.91 (28.7 s, from 39.8 s: repsFor() accounts for all of it, and every second of it comes from the six languages whose cheapest cell is 20-28 ms); heap arms 2.1 s for eight languages, from 3.0 s for seven (two reads per language instead of three, since flattening retired the warm-up pass); module load 3.9 s. --check pays one import that report mode does not: the inventory arm loads pipeline/registry.ts, which drags in every registered scope resolver and its providers. Measured in isolation with the bench's own static imports already resident, that import costs 6.3-6.5 s on one box and 9.3-10.0 s on another \u2014 i.e. it consumes almost the whole repsFor win, which is why --check did not get faster. It is loaded dynamically at the point of use rather than at the top of the file, so report mode does not pay it and both modes take their measurements in the same module state. IT WAS WEIGHED AND KEPT, on the number that decides it: the benchmarks job is not CI's critical path. On the last green run of main it took 9 m 23 s against 12 m 58 s for the sharded coverage job that gates the merge, so ~4 m 40 s of slack sits above this bench and those seconds buy zero merge latency. Moving the arm to a vitest file would move the registry load ONTO the critical path, and would weaken it as well: this reconciles LANG_REGISTRY's SupportedLanguages values, which are what the five dispatcher branches key off, whereas a test that cannot import measure.mjs can only reconcile this file's arm NAMES plus a hand-written rule for de-aliasing csharp_csproj. The contract test import-target-index-reuse.contract.test.ts already covers the ADAPTER-boundary contract for every registered resolver; this arm covers a different claim, that the BENCH covers the pipeline. The ts family is still the largest single block of the timing phase (8.8 s) \u2014 its cost is suffixResolve probing ~39 extensions per path part on a miss, which is the real resolver and cannot be tuned away from the bench side. IF IT HAS TO SHRINK, drop collide and collide_large for typescript and vue and nothing else: -3.9 s, and it is the only cut that removes near-duplicate work rather than coverage, because all three run the same resolveTsTarget over the same buildSuffixIndex and javascript keeps the collide arm that covers their shared collision axis. Do NOT reach for REPS_MAX: it is 15 because depth_ratio tripped its own budget about 1 run in 20 at 5 and once at 7, and lowering it would re-open that for the eleven languages whose cheapest cell is sub-5 ms \u2014 which is where every recorded trip happened. The six languages it was safe to lower have already been lowered, per language and from a measurement, by repsFor(). ---- THE FIFTH ARGUMENT (context) AND THE TWO ARMS IT MOVED ---- resolveOne now makes run.ts's five-argument call for the two hooks that declare a fifth parameter, so php and python time the legs behind it. Nothing else moved: the other fifteen arms are handed no context and build no ParsedFile[] at all, and over five runs their five ms numbers and four ratios sit exactly where they did. Both languages' ten fingerprints, resolved counts and distinct_outcomes are IDENTICAL \u2014 the leg AGREES with the cascade on this corpus, which is the whole reason the context arm had to be added rather than leaving the fingerprint to notice. PHP: small_ms 27.762 -> 35.125 (+26.5%) and collide_ms 29.407 -> 36.182 (+23.0%), which is filesByDirectory plus, on every import that resolves, a candidate gather over the resolved file's directory and a localDefs filter; the ms ceilings keep PHP's own 4.21x and 4.26x multipliers (117 -> 148, 125 -> 154). depth_ratio 1.144 -> 1.283 and the 1.9 budget is UNCHANGED, which makes it 1.48x measured rather than 1.66x: directoryAliases emits one entry per path segment, so filesByDirectory is O(files x depth) and the depth arm is the only one that can see it \u2014 that budget got TIGHTER relative to its measurement, not looser, and 1.48x sits inside the 1.37-1.75x band the other sixteen carry. Its heap reading rises 37576816 -> 49574008 (+31.9%) for the same structure, and the reading is the MEMO rather than the workspace it indexes: newPass allocates the ParsedFile objects before retainedPassBytes takes its baseline sample, so they sit outside the delta. PYTHON, WHOSE FIGURES ARE THE LEAST SETTLED THING IN THIS FILE AND ARE RECORDED IN TWO SNAPSHOTS BECAUSE OF IT. A named import is the only spelling that reads context.parsedFiles, and it costs up to three entries into the resolver per import (package probe, exports check, submodule probe) where the synthetic namespace spelling this arm used to pass costs one. Against the resolver as it stood when the call shape changed that read small_ms 1.76 -> 5.751 and collide_ms 1.929 -> 5.894, ~3.1x. Against the resolver a few commits later \u2014 which stopped re-running the whole tail after a null package probe, a double-probe this bench could not previously see because the namespace spelling never entered that branch \u2014 the same arms read 4.404 and 4.505. The ceilings are 18 and 19, chosen to clear BOTH: 4.09x and 4.22x of the current numbers, 3.13x and 3.22x of the higher ones, so neither state is red. Retighten toward 4x once that resolver settles. ITS DEPTH ARM WAS DILUTED AND THE BUDGET IS RETIGHTENED TO MATCH, which is the one thing here worth arguing about: the added work is depth-FLAT, so depth_ratio FALLS 1.872 -> 1.478 while the absolute cost more than doubles, and 2.6 against 1.478 would be 1.76x \u2014 far looser than the 1.39x #2913 chose deliberately to lock its own fix in. 2.1 restores that multiplier (1.42x). THE TWO MUTATION SCORES #2913 RECORDED (3.123 for reverting the per-directory memo, 2.734 for reverting the nested-name rejection) WERE TAKEN AGAINST THE OLD CALL SHAPE AND HAVE NOT BEEN RE-TAKEN. Modelled forward, with the depth-quadratic term reappearing in every resolver entry so its absolute contribution scales with the entry count, they land near 2.8 and 2.4 \u2014 both above 2.1, and the second BELOW 2.6, which is the arithmetic that decided the budget. Re-run the two mutations before trusting the lock-in claim above. python's heap reading is unchanged (10543152 recorded; 10529848-10544616 across eight runs) because its probe misses before the branch that reads parsedFiles \u2014 see _blind_spot for why no probe can reach that memo. Every figure in this section is the MAXIMUM over its snapshot's runs (five, then three), with peak-to-peak 1.031-1.058 on php and 1.019-1.081 on python, taken on a box that was NOT idle and with another change landing in python's resolver mid-measurement. Re-take them serially before merging.", + "_arms_note": "Five timing arms, one memory arm and one deterministic arm elsewhere, because none of them gates alone. scaling_ratio (t_large/t_small)/(1600/400) catches cost growing with FILE COUNT \u2014 the #2877-#2880, #2901, #2902 and #2908 regressions themselves; every one of those legs was Theta(files) per import, so a revert scores ~4 here by construction. depth_ratio (t_deep/t_small at a FIXED file count, ~6x the path components) catches cost growing with path DEPTH, which scaling_ratio divides out and structurally cannot see; buildSuffixIndex (C#, Ruby, PHP, Java) and Kotlin suffixByStem emit one entry per component, so they legitimately sit above 1.0 while Go, Dart and COBOL, whose indexes are depth-free, sit at ~1.0. csharp's depth_budget has now been retightened twice for the same reason, and the second time it did lock the win in. It was 5 against a then-measured 3.318; #2903 made buildSuffixIndex's dirMap lazy and it became 3.5 against 2.31, with the file stating plainly that 3.5 did NOT lock that win in because a revert to an eager dirMap scores 3.318 and passes. Extending the laziness to the two SUFFIX maps drops it again, to 1.438 (java likewise 2.214 -> 1.402), because the deep arm has ~6x the path components and an O(files x depth) build of a map the no-csproj leg never reads is exactly the cost that scales with depth. Both are now 2.2, which is this file's 1.5x convention against measurements whose own peak-to-peak over 4 runs is 1.04x and 1.07x \u2014 and 2.2 DOES lock it in: an eager rebuild scores 2.3+ and fails. The other fifteen depth budgets sit at 1.37-1.75x measured and are unchanged. collide_scaling_ratio is the same measurement on a SHARED-LEAF layout (svcN/internal, SrcN/Models, com/example/model in every service, a repeated mod0.dart/mod0.rb/Mod0.cpy basename) carrying an identical file, import and resolved count: the small/large/deep arms mint one directory name per index, so every index bucket in them holds exactly ONE entry (measured: max last-segment bucket 1 and max matching directories 1 for go and csharp at 400 and 1600 files; max basename bucket 1 for dart and ruby), and bucket cardinality is the only non-constant term the new indexes have. On the shared-leaf shape go, csharp, dart and java legitimately score 2.1-3.9 because the bucket grows with the file count BY CONSTRUCTION \u2014 this is a limit on the SCOPE of the \"independent of corpus size\" claim, not a regression (the indexed code is still faster there than the pre-change full scan); their collide budgets say so honestly instead of pretending 1.8. Ruby, Kotlin, PHP and COBOL answer from keyed maps and are collision-immune, so they keep the linear 1.8 budget and that immunity is the assertion. csharp_csproj is the one arm that runs the other way: its shared leaf collapses dirsByLastSegment to the single key Models, so the slash-free sweep (see CSPROJ_CONFIGS) is CHEAPER on the collide layout than on the unique one and its expensive scale arm is large, not collide_large. Its 1.8 collide budget is therefore the linear one, and the arm that carries its real cost is the unique one. The collide arm is also the only arm that reaches filesDirectlyInPkgDir's dirCount > 1 merge (go: 388 multi-directory calls at 400 files, up to 9 directories; 1517 at 1600 files, up to 34) and the only one that reaches COBOL's copybook-over-source tier tie-break, which needs one bookname to name two files. small_ms_ceiling and collide_ms_ceiling are ABSOLUTE (~4x the measured arm), because a constant-factor regression that grows both scale arms equally passes every ratio. The five arms added here use 4.2x, the middle of the 3.7-4.6x the original five already carry; the two COBOL arms use ~5x, the multiplier dart's sub-1 ms arm has always carried, because a fixed scheduler hiccup is a larger fraction of a smaller number \u2014 measured over 8 runs they sat at 0.25-0.37 ms and 0.18-0.30 ms, and the pre-#2908 two-scans-per-COPY implementation costs ~300 ms on the same arm, so 2.0 and 1.5 still separate fixed from broken by two orders of magnitude. NOISE, measured rather than assumed: depth_ratio divides two sub-3 ms numbers (Dart's are sub-1 ms) and is by far the noisiest arm here, so it set N for the whole file. fastest() is a min-of-N estimator, so N is the knob. Over 22 --check runs on an idle box, peak-to-peak: at N=5 go ran 0.757-1.748 (2.31x) and tripped its own 1.6 budget about 1 run in 20; at N=7 (the kotlin-import-target setting) Dart still ran 0.678-2.043 (3.01x) and tripped once; at N=15 (bench/cfg, bench/schema-pairs, bench/callable-value-flow) every language collapsed to a 1.13-1.26x swing with 22/22 passing. The budgets were NOT widened; the estimator was fixed instead, which is why the headroom above is real rather than granted. N IS NOW PER LANGUAGE, and that is a refinement of the same finding rather than a retreat from it. The overshoot of min-of-K against min-of-15 is a function of the CELL's absolute duration, not of the language: replayed against two independent runs' full sample sets, the worst overshoots at K=7 land on swift.small (0.43 ms, 31.8%) and dart.collide (1.5 ms, 37.6%), while every cell at or above 10 ms overshoots by at most 6.3%. So repsFor() keeps 15 while a language's cheapest arm is under 5 ms and otherwise spends ~150 ms per cell, floored at 7 \u2014 15 for go, csharp, dart, kotlin, java, cobol, swift, rust, python, c and cpp (every language the flakiness above was ever about, cheapest arm 0.19-3.2 ms) and 7-8 for csharp_csproj, ruby, php, javascript, typescript and vue (cheapest arm 20-28 ms). Per LANGUAGE, not per cell, so all five arms of a language share one estimator and the four ratios stay comparisons of like with like. The replay passed all 85 cells on all five gates at 0.4-0.7 of budget and saved 12.8 s and 12.4 s of a 46 s run; min-of-7 also reads slightly HIGHER than min-of-15, so the ceilings get marginally more sensitive rather than less. Confirmed on 4 fresh runs with the adaptive estimator live: every small arm inside 1.12x peak-to-peak and every collide arm inside 1.07x, with the six 7-8 rep languages at 1.008-1.071 \u2014 no worse than the 11 that kept 15. The chosen N is reported per language as `reps`. heap_ceiling_bytes bounds the retained per-pass import index, the only arm here that can see memory: buildSuffixIndex emits maps at O(files x depth), the profile package-dir-index.ts cites #2649 to avoid for itself, and csharp, ruby, php and java all retained NOTHING across imports at BASE (C#'s no-csproj leg and PHP's and Java's every leg re-scanned the raw Set; Ruby rebuilt and discarded a suffix index per require). It is measured at 8000 and 32000 files at HEAP_PAD depth rather than at the timing arms' sizes, because the finding is an ABSOLUTE footprint at repository scale. THE ARM NOW READS WHAT THE LANGUAGE READS, and that change is the whole reason this file was re-baselined. Four of these arms used to call getWorkspaceFileIndex(set) directly and then read index.all.length, which asks no suffix question at all \u2014 harmless only while buildSuffixIndex built both maps eagerly. The moment they went lazy the direct call built NO map, csharp, ruby, php and java each reported 0 B at 32000 files, and 0 B is under every ceiling: --check printed PASS over four gates that had silently become ceilings over nothing, which is precisely the failure this file's own header warns about for rust and cobol. Every arm now resolves a real MISSING import through the real resolver (HEAP_PROBE_TARGET, asserted to miss), so the maps it forces are the maps production forces, and a resolver that starts asking a new question moves the number without anyone editing the bench. That makes the READ PATTERN the dominant term, and the eight numbers say so: java 34958600 B and csharp 29862200 B ask index.get and never getInsensitive; php 37579888 B asks getInsensitive and never get, plus its own first-proper-suffix map; ruby 41025360 B and javascript 26745296 B read get(s) || getInsensitive(s) and pay for both, the second DERIVED from the first; and csharp_csproj 73705944 B additionally asks getFilesInDir. csharp_csproj IS NOW GATED, reversing the earlier decision that it would be 'a ceiling on a duplicate': at +20.8% of the C# index it was one, and at 2.47x of it \u2014 same corpus, same getWorkspaceFileIndex, three maps instead of one \u2014 it is the witness that the read pattern is the footprint. The old RESIDUAL note is superseded by that number: a dirMap-sized addition is no longer +18%, and a consumer that asks all three questions blows csharp's ceiling by 1.64x rather than sliding under it. A SECOND MEASUREMENT BIAS was removed at the same time and it moved every figure here, so do not read these against the old ones as if only the read pattern changed. buildFiles mints paths with template literals, which V8 keeps as ropes; the first traversal that slices one flattens it, allocating the flat string and dropping the rope's pieces, so a build measured over an unflattened corpus reports the index MINUS that net release \u2014 11% low, uniformly. bytes_small was read over a corpus a discarded warm-up pass had already flattened and bytes_large over a fresh one, so every ratio read ~0.85-0.89 for structures that are exactly linear in the file count. measureHeap now flattens each corpus before measuring it; all eight ratios read 0.998-1.017, and the warm-up pass is gone because with the corpus flat a language's first and second reads agree to within 0.3%. python's figure rises from 7624992 to 10362976 for this reason and not because anything regressed, and then to 10543152 (+1.7%) because #2913's nestedDirNames set is retained for the pass, and then FALLS to 6360936 (-39.7%) for a reason worth knowing: byBasename holds roughly one bucket per file, and building each with `[]` followed by `push` made V8 grow the backing store to its 16-slot minimum, so every single-file bucket retained 15 empty pointer slots. Constructing the one-element buckets directly (`set(base, [entry])`) is byte-identical in contents and 3.9 MiB smaller at 32000 paths — 37% of what this arm used to read was empty array slots — the ancestorsByDir memo itself is NOT in this reading, because python's probe target misses at the nested-name rejection and never reaches the walk, so this arm does not bound that memo; measured separately with a probe that does reach it, a 32000-file corpus with every file in its own 10-deep directory retains ~19 MB, which would clear this ceiling, so repointing python's heap probe at a walking spelling means re-recording the ceiling in the same change, and c is unchanged at 10018816 because its basename map does not slice paths. Its ceiling is 1.5x the measured arm, and the DIFFERENCE FROM THE 4x TIMING CONVENTION IS DELIBERATE \u2014 do not harmonise it back. 4x exists because runner contention dominates a wall-clock number; this one has essentially no measurement noise (across 4 runs the widest spread was 0.11% on python, 0.03% on csharp_csproj and 0.00% \u2014 identical to the byte \u2014 on ruby, php, java, javascript and c, and the same holds across separate processes), so 4x would throw away almost all of the gate's power and sail straight past the regression this arm exists to catch. 1.5x still tolerates ~50% of cross-platform and Node-version drift, far more than a Node major bump plausibly moves heapUsed accounting; it catches a duplicated index (+100%) or a second exactMap-sized suffix map (+~85%). heap_floor_fraction is the arm the 0 B incident proved was missing. A ceiling can only say 'not too big'; nothing said 'still measuring something', which is why four dead arms passed. The floor is 0.5 x each language's RECORDED READING (heap_reading_bytes), which is half the measured size and says so. It used to be 0.33 x the CEILING, described the same way \u2014 true only while every ceiling stayed at exactly 1.5x its reading, a convention this file states and nothing enforces, so re-tuning one ceiling upward would have loosened that language's floor by the same factor in the one direction a floor exists to watch. The two forms agree to within 0.8% for all eight today, so this is a correction of derivation, not of strength. It sits ~400x above the readings' own reproducibility and far below any collapse. A genuine 2x memory WIN trips it too, and that is intended: like a fingerprint move, it must be explained and re-baselined rather than absorbed. COBOL is left out for the opposite reason: its index is two Map, O(files) with no depth term, and at 32000 files its retained delta does not clear the noise of the measurement itself. heap_ratio_budget, the linear-growth check across the 4x file-count gap, is the orthogonal arm: it sees per-file and per-depth growth but not a constant factor. ---- THE EIGHT LANGUAGES ADDED LAST (swift, rust, python, javascript, typescript, vue, c, cpp) ---- They carry the SAME five arms and the same gates; what differs is which arm can actually fail for each, because each resolver has a different cost axis, and the budgets below say so instead of copying a number across. Every figure quoted is the MAXIMUM over 5 full runs on an idle box, and the peak-to-peak of every one of these arms stayed inside 1.10x over those runs \u2014 tighter than the 1.13-1.26x the original nine record, because none of these arms divides two sub-1 ms numbers the way dart depth_ratio does. depth_budget is ~1.5x measured throughout: swift 2.3 (1.487), rust 2.1 (1.377), javascript 2.1 (1.376), typescript 2.1 (1.381), vue 2.3 (1.563), c 3.0 (1.990), cpp 3.0 (1.999). PYTHON WAS 11 AGAINST 7.389 AND IS NOW 2.6 AGAINST 1.872, because #2913 fixed the resolver rather than the budget. Its INDEX was always depth-free; hasRepoCandidate and resolveAbsoluteFromFiles each rebuilt one ancestor prefix per directory component of the importer on EVERY import, and the index's own dirPrefixes build inserted one entry per component per file, so the resolver was quadratic in path depth where every other language here is linear or flat. The prefixes are a pure function of the importer's DIRECTORY, so they are now memoized per directory inside getPythonFileIndex (ancestorsByDir), the leading segment is rejected up front against a set of nested directory names, the module and package buckets are consulted before the walk rather than inside it, and the dirPrefixes build stops at the first ancestor already stored. All five fingerprints are byte-identical, so it is a hoist. The budget is 2.2, and BOTH numbers behind it were re-measured on a quiet box AFTER the context leg below started being measured, because that change moved the arm: the work it adds is depth-FLAT, so python's absolute cost more than doubled while depth_ratio FELL to 1.405-1.563 over 5 serial runs (peak-to-peak 1.11x). A budget carried over from before that change would have been slack against a smaller ratio. 2.2 is 1.41x the measured maximum, inside the 1.37-1.75x band the other fifteen sit in, and it LOCKS THE WIN IN: reverting the per-directory ancestor memo alone scores 2.524 and reverting the nested-name rejection alone scores 2.553, both measured under the current call shape, so each fails at 2.2 with 13% to spare. Do not read those two figures as the pre-#2913 cost — 7.239 was that, and the gap closed because the bare-import tier stopped walking at all (see below). The other two parts of the fix are not gated by this arm and are not meant to be: reverting the bucket prune or the dirPrefixes early break lands under any budget this arm's noise supports, so they are gated deterministically instead, by the prefix-parity and package-probe arms of test/unit/scope-resolution/python/python-importer-ancestors.test.ts and python-import-target-parity.test.ts, which go red on exactly those two mutations. A timing budget catches what it can measure; the counts catch the rest. THE BARE-IMPORT TIER (`import os`, single segment, no dot) was a separate O(depth) walk in import-resolvers/python.ts that this bench cannot see at all, because every python arm here spells its imports with a dot and returns at the `pathLike.includes('/')` guard before reaching it. It ran TWICE per `from x import y` — the package probe's recursion re-ran the whole tail on identical inputs — and is now one memoized chain plus an O(1) proof-of-absence against the index's basename buckets: 12/24/72 Set probes at depth 1/4/16 became a flat 2, and 11.615 us/import at 18 path components became 0.740. Gated by probe COUNT in test/unit/scope-resolution/python/python-import-probe-count.test.ts, not here. collide_scaling_budget splits three ways. Three languages scan a bucket that grows with the corpus and get their measured value x1.5: swift 4.9 (3.279 \u2014 its bucket is the module file list it RETURNS, and its collide arm is four modules instead of dirs of them so that bucket is fileCount/4, i.e. 100 files at 400 and 400 at 1600), c 3.8 (2.535) and cpp 4.0 (2.639, the same basename bucket its suffix fallback walks). Four answer from keyed maps and keep the linear 1.8 \u2014 python 1.097, javascript 1.083, typescript 1.053, vue 1.079 \u2014 and that immunity IS the assertion, exactly as for ruby, kotlin, php and cobol. RUST IS THE ONE ARM THAT WAS REDESIGNED RATHER THAN BUDGETED. It resolves by probing candidate paths with allFilePaths.has(...) and never searches, so its cost is O(path segments) and provably flat in the file count (1.095 scaling, 1.061 collide scaling): a shared-leaf collide arm for rust would have asserted nothing, which is worse than no arm. Its collide corpus is instead a deep module tree (src/l0/l1/l2/l3/l4/mod{d}) whose targets carry ~2x the :: segments, so the arm exercises the axis that CAN grow, its 1.8 budget asserts the flatness across file counts, and collide_ms_ceiling 19 bounds the absolute cost of the long-path probe. small_ms_ceiling and collide_ms_ceiling are ~4x measured as everywhere else: rust 10/19 (2.609/4.704), python 7/8 (1.76/1.929, retightened from 12/15 against 3.044/3.771 by #2913), javascript 85/89 (21.254/22.145), typescript 85/86 (21.250/21.464), vue 81/93 (20.164/23.227), c 7/11 (1.620/2.850), cpp 7/12 (1.581/3.009). Swift takes ~5x (2 against 0.421 and 4 against 0.821) \u2014 the multiplier dart and cobol already carry, because a fixed scheduler hiccup is a larger fraction of a sub-1 ms number. ONE CAVEAT ON THE THREE ts-FAMILY MS NUMBERS, stated because nothing else in this file would reveal it: resolveTsTarget carries a per-pass resolveCache keyed currentFile::importPath, which no other resolver here has, and ~10% of this corpus is repeat pairs. Their us/import is therefore a slight underestimate of a cold resolve. It is left in rather than defeated because it is what the real pipeline does, and it is identical across all three so the arms stay comparable. HEAP for the eight: rust, swift, typescript, vue, cpp and cobol are still NOT gated, all of them measured before being left out. rust builds no index on this hook (16 B at 8000 files, 0 B at 32000); swift holds one pointer per file-times-segment and mints no strings, reading 0.98 MB at 8000 files against 0.29 MB at 32000 \u2014 a 4x larger corpus reading 3x SMALLER, which is what a measurement below its own noise floor looks like, and the same reading cobol gives (0.54 MB then 0 B); typescript and vue duplicate javascript through the same builder over the same-shaped corpus, and cpp duplicates c (10021320 against 10016960, 0.04% apart). Those four duplications are the ONLY exclusions that still rest on 'it would be a duplicate', and they are duplicates of a builder AND of a read pattern, which is the pairing csharp_csproj failed once the read pattern started to matter \u2014 if any of the four ever diverges in what it ASKS the index, it earns an arm the same way csharp_csproj just did. All eight gated arms are read the same way now (retainedPassBytes, one real import), so unlike before they are directly comparable to one another. WALL CLOCK \u2014 ~33-35 s in report mode, down from ~46 s, and ~44-45 s for --check, which is essentially UNCHANGED from ~46 s. Only report mode got faster; do not read the pair as 46 -> 42. The breakdown is worth having before anyone trims it. Timing arms: go 2.02, csharp 1.09, csharp_csproj 3.22, dart 0.41, ruby 2.90, kotlin 0.85, php 3.46, java 1.57, cobol 0.09, swift 0.46, rust 0.85, python 1.22, javascript 3.23, typescript 2.72, vue 2.89, c 0.86, cpp 0.91 (28.7 s, from 39.8 s: repsFor() accounts for all of it, and every second of it comes from the six languages whose cheapest cell is 20-28 ms); heap arms 2.1 s for eight languages, from 3.0 s for seven (two reads per language instead of three, since flattening retired the warm-up pass); module load 3.9 s. --check pays one import that report mode does not: the inventory arm loads pipeline/registry.ts, which drags in every registered scope resolver and its providers. Measured in isolation with the bench's own static imports already resident, that import costs 6.3-6.5 s on one box and 9.3-10.0 s on another \u2014 i.e. it consumes almost the whole repsFor win, which is why --check did not get faster. It is loaded dynamically at the point of use rather than at the top of the file, so report mode does not pay it and both modes take their measurements in the same module state. IT WAS WEIGHED AND KEPT, on the number that decides it: the benchmarks job is not CI's critical path. On the last green run of main it took 9 m 23 s against 12 m 58 s for the sharded coverage job that gates the merge, so ~4 m 40 s of slack sits above this bench and those seconds buy zero merge latency. Moving the arm to a vitest file would move the registry load ONTO the critical path, and would weaken it as well: this reconciles LANG_REGISTRY's SupportedLanguages values, which are what the five dispatcher branches key off, whereas a test that cannot import measure.mjs can only reconcile this file's arm NAMES plus a hand-written rule for de-aliasing csharp_csproj. The contract test import-target-index-reuse.contract.test.ts already covers the ADAPTER-boundary contract for every registered resolver; this arm covers a different claim, that the BENCH covers the pipeline. The ts family is still the largest single block of the timing phase (8.8 s) \u2014 its cost is suffixResolve probing ~39 extensions per path part on a miss, which is the real resolver and cannot be tuned away from the bench side. IF IT HAS TO SHRINK, drop collide and collide_large for typescript and vue and nothing else: -3.9 s, and it is the only cut that removes near-duplicate work rather than coverage, because all three run the same resolveTsTarget over the same buildSuffixIndex and javascript keeps the collide arm that covers their shared collision axis. Do NOT reach for REPS_MAX: it is 15 because depth_ratio tripped its own budget about 1 run in 20 at 5 and once at 7, and lowering it would re-open that for the eleven languages whose cheapest cell is sub-5 ms \u2014 which is where every recorded trip happened. The six languages it was safe to lower have already been lowered, per language and from a measurement, by repsFor(). ---- THE FIFTH ARGUMENT (context) AND THE TWO ARMS IT MOVED ---- resolveOne now makes run.ts's five-argument call for the two hooks that declare a fifth parameter, so php and python time the legs behind it. Nothing else moved: the other fifteen arms are handed no context and build no ParsedFile[] at all, and over five runs their five ms numbers and four ratios sit exactly where they did. Both languages' ten fingerprints, resolved counts and distinct_outcomes are IDENTICAL \u2014 the leg AGREES with the cascade on this corpus, which is the whole reason the context arm had to be added rather than leaving the fingerprint to notice. PHP: small_ms 27.762 -> 35.125 (+26.5%) and collide_ms 29.407 -> 36.182 (+23.0%), which is filesByDirectory plus, on every import that resolves, a candidate gather over the resolved file's directory and a localDefs filter; the ms ceilings keep PHP's own 4.21x and 4.26x multipliers (117 -> 148, 125 -> 154). depth_ratio 1.144 -> 1.283 and the 1.9 budget is UNCHANGED, which makes it 1.48x measured rather than 1.66x: directoryAliases emits one entry per path segment, so filesByDirectory is O(files x depth) and the depth arm is the only one that can see it \u2014 that budget got TIGHTER relative to its measurement, not looser, and 1.48x sits inside the 1.37-1.75x band the other sixteen carry. Its heap reading rises 37576816 -> 49574008 (+31.9%) for the same structure, and the reading is the MEMO rather than the workspace it indexes: newPass allocates the ParsedFile objects before retainedPassBytes takes its baseline sample, so they sit outside the delta. PYTHON, WHOSE FIGURES ARE THE LEAST SETTLED THING IN THIS FILE AND ARE RECORDED IN TWO SNAPSHOTS BECAUSE OF IT. A named import is the only spelling that reads context.parsedFiles, and it costs up to three entries into the resolver per import (package probe, exports check, submodule probe) where the synthetic namespace spelling this arm used to pass costs one. Against the resolver as it stood when the call shape changed that read small_ms 1.76 -> 5.751 and collide_ms 1.929 -> 5.894, ~3.1x. Against the resolver a few commits later \u2014 which stopped re-running the whole tail after a null package probe, a double-probe this bench could not previously see because the namespace spelling never entered that branch \u2014 the same arms read 4.404 and 4.505. The ceilings are 18 and 19, chosen to clear BOTH: 4.09x and 4.22x of the current numbers, 3.13x and 3.22x of the higher ones, so neither state is red. Retighten toward 4x once that resolver settles. ITS DEPTH ARM WAS DILUTED AND THE BUDGET IS RETIGHTENED TO MATCH, which is the one thing here worth arguing about: the added work is depth-FLAT, so depth_ratio FALLS 1.872 -> 1.478 while the absolute cost more than doubles, and 2.6 against 1.478 would be 1.76x \u2014 far looser than the 1.39x #2913 chose deliberately to lock its own fix in. 2.1 restores that multiplier (1.42x). THE TWO MUTATION SCORES #2913 RECORDED (3.123 for reverting the per-directory memo, 2.734 for reverting the nested-name rejection) WERE TAKEN AGAINST THE OLD CALL SHAPE AND HAVE NOT BEEN RE-TAKEN. Modelled forward, with the depth-quadratic term reappearing in every resolver entry so its absolute contribution scales with the entry count, they land near 2.8 and 2.4 \u2014 both above 2.1, and the second BELOW 2.6, which is the arithmetic that decided the budget. Re-run the two mutations before trusting the lock-in claim above. python's heap reading is unchanged (10543152 recorded; 10529848-10544616 across eight runs) because its probe misses before the branch that reads parsedFiles \u2014 see _blind_spot for why no probe can reach that memo. Every figure in this section is the MAXIMUM over its snapshot's runs (five, then three), with peak-to-peak 1.031-1.058 on php and 1.019-1.081 on python, taken on a box that was NOT idle and with another change landing in python's resolver mid-measurement. Re-take them serially before merging.", "_triage": "Every ratio and ms ceiling here is a TIMING signal \u2014 re-run on an idle machine before investigating; runner contention dominates. depth_ratio is the noisiest of them by a wide margin (it divides two sub-3 ms numbers, and Dart's are sub-1 ms): if exactly one arm fails and it is that one, suspect the machine first. N is 15 for every language whose cheapest arm is under 5 ms, rather than this bench's original 5, specifically to hold that arm's peak-to-peak swing under 1.26x \u2014 see _arms_note for the measured distributions and for why the six languages that drop to 7-8 are the ones where cell size makes it safe \u2014 so a depth_ratio failure that REPRODUCES is a real signal, not noise. Each language's chosen N is printed as `reps`; read it before blaming the estimator. The fingerprint, shape and heap arms are the opposite: deterministic (over 4 runs the heap arm's widest spread was 0.11% on python and 0.00% on java, javascript and c), a re-run never changes them, and they must never be wished away. TWO heap failures mean the arm STOPPED MEASURING rather than that memory grew, and both are deterministic: a heap floor failure says the probe no longer forces the index it used to (this is how four arms read 0 B when buildSuffixIndex went lazy, and 0 B passes every ceiling), and a `heap probe ... resolved` throw says a probe target that must MISS now hits, so the reading is a materialized answer and the legs past it were never reached. The context arm is deterministic too, and a failure there means one specific thing rather than a range of them: run.ts's fifth argument is not reaching that resolver from this bench, or the leg behind it stopped running. Never a timing signal, never a re-run.", "_floor": "Measured against the pre-change implementations on THIS corpus at 150/600 files: go 3.36, csharp 4.10, dart 3.32, ruby 3.87. The issues report 4.00 / 3.43 / 4.05 on their own corpora; those are DIFFERENT numbers from different repositories and are not reproduced here \u2014 what they and these share is that both independently land in the quadratic band, well clear of the ~1.0 a linear result gives. Note also that this floor was taken at 150/600 while the gate runs at 400/1600, so it is a lower bound on what the pre-change code would score today. Kotlin's own bench measured its pre-index floor at 3.737. The four resolvers added later were NOT re-floored on this corpus, and the reason is that they do not need to be: every one of their pre-change legs walked the whole file set per import (PHP one findIndex per path part per extension, Java one scan per stripped prefix, COBOL two full scans per COPY, C# csproj one normalizedFileList pass per import per matching config), so their scaling_ratio is ~4 by construction rather than by measurement. Their per-import costs were measured on their own issue corpora instead: PHP 96.40 ms -> 0.036 ms, Java 8.05 ms -> 0.62 ms, COBOL 3879 us -> 10.5 us, C# csproj 1103 us -> 7.6 us. The 1.8 budget sits well above the linear result and well below every one of those. The eight languages added last were NOT floored either, and for a different reason again: they are not fixes, so there is no pre-change implementation to floor against. Their scaling budgets are the global linear 1.8 and the point of the arms is to hold the current numbers (measured 1.01-1.13) rather than to separate a fix from a break. The one exception is javascript, which IS a fix and does have a floor: 6448.9 us per import at 2000 files and 25972.6 us at 8000 \u2014 4.12x the per-import cost for 4x the files, i.e. O(imports x files) \u2014 against 28.5 / 27.4 us with the index PR #2911 gave it, and 25.0 / 27.0 us for TypeScript over the identical corpus.", "scaling_budget": 1.8, @@ -88,7 +88,7 @@ "ruby": 61600000, "php": 74400000, "java": 52500000, - "python": 15815772, + "python": 9541404, "javascript": 40200000, "c": 15000000 }, @@ -99,7 +99,7 @@ "ruby": 41020808, "php": 49574008, "java": 34958600, - "python": 10543848, + "python": 6360936, "javascript": 26745296, "c": 10018816 }, @@ -689,7 +689,7 @@ "collide_scaling_ratio": 1.085, "depth_ratio": 1.563, "scaling_ratio": 1.144, - "small_ms": 4.358 + "small_ms": 4.431 } }, "javascript": { diff --git a/gitnexus/bench/import-target/measure.mjs b/gitnexus/bench/import-target/measure.mjs index 47955408b..81d6437fe 100644 --- a/gitnexus/bench/import-target/measure.mjs +++ b/gitnexus/bench/import-target/measure.mjs @@ -834,17 +834,45 @@ function buildFiles(lang, fileCount, pad, shape) { } /** - * The `ParsedFile[]` the orchestrator threads beside the path set, for the two - * languages whose hook declares a `context` — see `CONTEXT_LANGS`. + * ONE `ParsedFile`, and the ONE place in this file that spells that shape. * * CARRIES THE FIELDS THE RESOLVERS READ AND NOTHING ELSE, deliberately. * `filesByDirectory` reads `filePath`; PHP's declaring-file filter reads * `localDefs[].type` and `localDefs[].qualifiedName`; Python's * `pythonFileExportsName` reads `localDefs[].qualifiedName`. `scopes`, * `parsedImports` and `referenceSites` are on the real shape and are inert on - * this path, and this array is rebuilt inside every timed pass (see `newPass`), - * so filling them would charge the RESOLUTION arms for extraction work that - * happens in another phase entirely. + * this path, and the timed corpora are rebuilt inside every pass (see + * `newPass`), so filling them would charge the RESOLUTION arms for extraction + * work that happens in another phase entirely. + * + * `nodeId` is inert as well — checked, not assumed: neither + * `php/import-target.ts` nor `python/import-target.ts` mentions it, and they are + * the two modules `resolveOne` enters. It is minted anyway because it is on the + * real shape, and its spelling is therefore free to be uniform. + * + * Both callers come through here — `buildParsedFiles` for the timed and heap + * corpora, `CONTEXT_PROBE` for the `context` arm's hand-built ones. It used to + * be spelled out twice, ~900 lines apart, differing only in that `nodeId`; this + * is an untyped `.mjs`, so nothing would have failed at build if `ParsedFile` + * grew a field and only one of the two copies learned about it. + */ +const probeFile = (filePath, defs) => ({ + filePath, + moduleScope: filePath, + scopes: [], + parsedImports: [], + localDefs: defs.map(([type, qualifiedName], n) => ({ + nodeId: `${filePath}#${n}`, + filePath, + type, + qualifiedName, + })), + referenceSites: [], +}); + +/** + * The `ParsedFile[]` the orchestrator threads beside the path set, for the two + * languages whose hook declares a `context` — see `CONTEXT_LANGS`. * * Two defs per file, and both are real shapes rather than padding. PHP keeps * classes and functions in SEPARATE symbol tables, so `App\Ns7\File7` naming @@ -870,17 +898,12 @@ function buildParsedFiles(lang, files) { const parent = slash < 0 ? '' : filePath.slice(0, slash); const owner = parent.slice(parent.lastIndexOf('/') + 1); const qualifiedName = lang === 'php' ? `App\\${owner}\\${stem}` : `${owner}.${stem}`; - parsedFiles.push({ - filePath, - moduleScope: filePath, - scopes: [], - parsedImports: [], - localDefs: [ - { nodeId: `${filePath}#c`, filePath, type: 'Class', qualifiedName }, - { nodeId: `${filePath}#f`, filePath, type: 'Function', qualifiedName }, - ], - referenceSites: [], - }); + parsedFiles.push( + probeFile(filePath, [ + ['Class', qualifiedName], + ['Function', qualifiedName], + ]), + ); } return parsedFiles; } @@ -1772,23 +1795,6 @@ function measureHeap(lang) { }; } -/** One hand-built `ParsedFile` for the `context` arm's corpora. Same fields - * `buildParsedFiles` fills, with the defs named explicitly because these two - * corpora exist to produce ONE specific disagreement each. */ -const probeFile = (filePath, defs) => ({ - filePath, - moduleScope: filePath, - scopes: [], - parsedImports: [], - localDefs: defs.map(([type, qualifiedName], n) => ({ - nodeId: `${filePath}#${n}`, - filePath, - type, - qualifiedName, - })), - referenceSites: [], -}); - /** * The `context` arm's corpora — one per `CONTEXT_LANGS` entry, each a handful * of files carrying ONE import whose answer DIFFERS between the production @@ -2014,6 +2020,58 @@ if (!CHECK) { const baseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8')); const failures = []; +/** + * PRESENCE, for one budget, in the one place that spells the reason. + * + * A missing budget is a DELETED GATE, not a passing arm: `got > undefined` is + * `false`, `ceiling * undefined` is `NaN` and `bytes < NaN` is `false`, so every + * comparison in this file answers "within budget" for every possible + * measurement the moment its key stops being a number. Each of the three call + * sites below is one deleted key away from a silent no-op, and the run still + * prints PASS. + * + * `Number.isFinite` rather than `typeof === 'number'`: over JSON input the two + * agree (JSON cannot express NaN or Infinity), and the stricter one is the one + * whose name says what the gate needs. + * + * The two per-site facts stay the caller's, because they are what a triager acts + * on: `reads` is the comparison that silently stopped gating, quoted, and + * `scope` is what deleting this one key actually costs — a single arm, or all + * eight at once. Only the shared framing and the shared trailing sentence live + * here. Returns the message rather than pushing it, so the timing loop can + * `continue` past a budget it must not then compare against. + */ +const requireNumericBudget = ({ key, value, reads, scope }) => + Number.isFinite(value) + ? null + : `no numeric ${key} in baselines.json — a missing budget is a DELETED GATE, not a passing ` + + `arm: the comparison it gates reads \`${reads}\`, which is false for every possible ` + + `measurement. ${scope} Deterministic: a re-run will not change it.`; + +/** + * The REVERSE direction of a reconciliation: every key declared in `label` that + * `codeList` does not name. + * + * The forward direction ("the code has an arm with no budget") is a presence + * check inside whichever loop iterates the code's list. This is the other way + * round — a budget, a baseline block or a registry row for an arm that is never + * measured — and no forward check can see it, because the thing it names is + * exactly the thing nothing iterates. + * + * `codeListName` and `why` stay the caller's: which list is authoritative and + * what the orphan costs are the two facts that differ between the three arms, + * and flattening them would leave a triager with a name and no reading of it. + */ +function expectNoOrphanKeys(label, declaredKeys, codeList, codeListName, why) { + for (const key of declaredKeys) { + if (codeList.includes(key)) continue; + failures.push( + `${label} has an entry for '${key}', which is not in ${codeListName} — ${why} ` + + `Deterministic: a re-run will not change it.`, + ); + } +} + /** The corpus-shape facts asserted for one timing scale. */ const SCALE_SHAPE = { fields: ['files', 'imports', 'resolved', 'distinct_outcomes', 'fingerprint'], @@ -2120,21 +2178,21 @@ for (const lang of LANGS) { }, ]; for (const check of timingChecks) { - // PRESENCE FIRST, because `got > undefined` is `false`: without this, going - // from a budget to no budget is going from a gate to no gate, and the run - // still prints PASS. All five maps are complete today, which is exactly - // when the check is worth adding — every one of the four per-language - // lookups above is one deleted key away from a silent no-op. The heap arm - // HAD THE SAME HOLE and this comment used to deny it: iterating the - // BASELINE's keys protects that loop against a deleted MEASUREMENT, which - // is a different thing from a deleted BUDGET. See `heapBudgetChecks`. - if (typeof check.budget !== 'number') { - failures.push( - `${lang}: no ${check.label} budget — baselines.json has no numeric ${check.key}. ` + - `A missing budget is a DELETED GATE, not a passing arm: the comparison below reads ` + - `\`${check.got} > undefined\`, which is false for every possible measurement. ` + - `Deterministic: a re-run will not change it.`, - ); + // PRESENCE FIRST — see `requireNumericBudget` for why. All five maps are + // complete today, which is exactly when the check is worth having: every one + // of the four per-language lookups above is one deleted key away from a + // silent no-op. The heap arm HAD THE SAME HOLE and the comment here used to + // deny it: iterating the BASELINE's keys protects that loop against a + // deleted MEASUREMENT, which is a different thing from a deleted BUDGET. + // See `heapBudgetChecks`. + const missing = requireNumericBudget({ + key: check.key, + value: check.budget, + reads: `${check.got} > undefined`, + scope: `That leaves ${lang}'s ${check.label} arm ungated.`, + }); + if (missing !== null) { + failures.push(`${lang}: ${missing}`); continue; } if (check.got > check.budget) { @@ -2211,23 +2269,19 @@ for (const lang of LANGS) { // five timing budgets get it — and the reason the comment up there used to give // for the heap arm not needing it was wrong. Iterating the baseline's keys // protects the loop below against a deleted MEASUREMENT (`heap == null`, right -// there); it does nothing about a deleted BUDGET. `ceiling * undefined` is -// `NaN` and `bytes_large < NaN` is false, `ratio > undefined` is false: these -// two keys are scalars rather than per-language maps, so deleting either is one -// keystroke that silently disables that arm for ALL EIGHT languages at once. -// That makes them the widest-blast-radius keys in this file, not the safest. +// there); it does nothing about a deleted BUDGET. These two keys are scalars +// rather than per-language maps, so deleting either is one keystroke that +// silently disables that arm for ALL EIGHT languages at once. That makes them +// the widest-blast-radius keys in this file, not the safest — which is what +// their `scope` sentence says and the per-language ones do not. const heapBudgetChecks = [ { key: 'heap_floor_fraction', value: baseline.heap_floor_fraction, reads: 'bytes_large < NaN' }, { key: 'heap_ratio_budget', value: baseline.heap_ratio_budget, reads: 'ratio > undefined' }, ]; +const heapArmScope = `This one key gates all ${HEAP_LANGS.length} heap arms at once.`; for (const check of heapBudgetChecks) { - if (Number.isFinite(check.value)) continue; - failures.push( - `no numeric ${check.key} in baselines.json — a missing budget is a DELETED GATE, not a ` + - `passing arm: the comparison below reads \`${check.reads}\`, which is false for every ` + - `possible measurement. This one key gates all ${HEAP_LANGS.length} heap arms at once. ` + - `Deterministic: a re-run will not change it.`, - ); + const missing = requireNumericBudget({ ...check, scope: heapArmScope }); + if (missing !== null) failures.push(missing); } // And EXACT KEY EQUALITY between `HEAP_LANGS` and the two per-language heap @@ -2242,13 +2296,13 @@ const heapBudgetMaps = [ ['heap_reading_bytes', baseline.heap_reading_bytes], ]; for (const [key, map] of heapBudgetMaps) { - for (const lang of Object.keys(map ?? {})) { - if (HEAP_LANGS.includes(lang)) continue; - failures.push( - `baselines.json ${key} has an entry for '${lang}', which is not in HEAP_LANGS — the bench ` + - `budgets a heap arm it does not measure. Deterministic: a re-run will not change it.`, - ); - } + expectNoOrphanKeys( + `baselines.json ${key}`, + Object.keys(map ?? {}), + HEAP_LANGS, + 'HEAP_LANGS', + 'the bench budgets a heap arm it does not measure.', + ); } // The same reverse direction for the context arm. The forward direction (a @@ -2257,15 +2311,13 @@ for (const [key, map] of heapBudgetMaps) { // other way round — a baseline block for a language the bench hands no context // is a gate over an arm that is never measured, and `armShapes` would never // look at it. -for (const lang of Object.keys(baseline.languages)) { - if (baseline.languages[lang].context === undefined) continue; - if (CONTEXT_LANGS.includes(lang)) continue; - failures.push( - `baselines.json languages.${lang} has a context block, but '${lang}' is not in ` + - `CONTEXT_LANGS — the bench pins an arm it does not run. Deterministic: a re-run will not ` + - `change it.`, - ); -} +expectNoOrphanKeys( + 'baselines.json languages.*.context', + Object.keys(baseline.languages).filter((lang) => baseline.languages[lang].context !== undefined), + CONTEXT_LANGS, + 'CONTEXT_LANGS', + 'the bench pins an arm it does not run.', +); // Driven by HEAP_LANGS, the CODE's list, exactly as the timing arms iterate // LANGS — so a deleted budget key is a presence failure rather than a language @@ -2275,17 +2327,22 @@ for (const lang of Object.keys(baseline.languages)) { for (const lang of HEAP_LANGS) { const ceiling = baseline.heap_ceiling_bytes?.[lang]; const reading = baseline.heap_reading_bytes?.[lang]; - for (const [key, value] of [ - ['heap_ceiling_bytes', ceiling], - ['heap_reading_bytes', reading], + // `reads` names the comparison each key gates further down: the ceiling is + // compared directly, the reading only after `reading * heap_floor_fraction` + // has turned a missing one into `NaN`. + for (const [key, value, reads] of [ + ['heap_ceiling_bytes', ceiling, 'bytes_large > undefined'], + ['heap_reading_bytes', reading, 'bytes_large < NaN'], ]) { - if (Number.isFinite(value)) continue; - failures.push( - `${lang}: no numeric ${key}.${lang} in baselines.json — a missing budget is a DELETED ` + - `GATE, not a passing arm, and this loop iterates HEAP_LANGS precisely so that deleting ` + - `the key fails here instead of dropping ${lang} out of the gate. Deterministic: a ` + - `re-run will not change it.`, - ); + const missing = requireNumericBudget({ + key: `${key}.${lang}`, + value, + reads, + scope: + `This loop iterates HEAP_LANGS precisely so that deleting the key fails here instead ` + + `of dropping ${lang} out of the gate.`, + }); + if (missing !== null) failures.push(`${lang}: ${missing}`); } const heap = report[lang]?.heap; if (heap == null) { @@ -2382,13 +2439,16 @@ for (const language of registeredLanguages) { `resolveOne) and a baselines.json entry. Deterministic: a re-run will not change it.`, ); } -for (const language of benchedLanguages) { - if (registeredLanguages.includes(language)) continue; - failures.push( - `LANG_REGISTRY names '${language}', which is not in SCOPE_RESOLVERS — this bench is gating a ` + - `resolver the pipeline no longer registers. Deterministic: a re-run will not change it.`, - ); -} +// The reverse half is the same loop as the two above it, so it goes through the +// same helper. Only the FORWARD half stays written out: its message is a +// five-step remediation for adding a language, which no shared framing carries. +expectNoOrphanKeys( + 'LANG_REGISTRY', + benchedLanguages, + registeredLanguages, + 'SCOPE_RESOLVERS', + 'this bench is gating a resolver the pipeline no longer registers.', +); // The SAME reconciliation for `CONTEXT_LANGS`, against the registry rather than // against a claim in a comment. `run.ts` passes the fifth argument to every diff --git a/gitnexus/src/core/ingestion/import-resolvers/per-file-set.ts b/gitnexus/src/core/ingestion/import-resolvers/per-file-set.ts index f4658c23f..4de0ee361 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/per-file-set.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/per-file-set.ts @@ -22,33 +22,36 @@ * does not grow across runs. There is no invalidation rule to get wrong because * there is nothing to invalidate — a new file set is a new key. * - * `K extends object` because the key is whatever object the pass keeps stable. - * Stability for the pass is the only property any of them needs — but WHICH - * object is the key decides whether the guards above can see the memo fail, and - * only one shape is instrumentable by them: + * The KEY TYPE is constrained rather than described, because which object is + * the key decides whether the guards above can see the memo fail, and a prose + * list of call sites is the thing this file elsewhere tells you not to write. + * `K` admits exactly the two shapes the orchestrator keeps stable for a pass: * - * - `ReadonlySet`, the pass's file set — every index derived from the - * file set, including the derived header-closure sets that - * `languages/{c,cpp}/scope-resolver.ts` memoize inside an outer per-file-set - * memo. Defeating one of these means copying the SET, which re-traverses it, - * which the `CountingSet` instrument (`test/helpers/counting-file-set.ts`) - * reads as a scan count rising with the import count. - * - `readonly ParsedFile[]`, the pass's parsed-file array — `languages/c/ - * static-linkage.ts`, `languages/cpp/file-local-linkage.ts` and - * `languages/php/import-target.ts`. Not derived from the file set at all, so - * the file-set guards do not reach them; these key on the array the - * orchestrator already threads through the pass, and their contract is that - * same pass-through discipline. + * - `ReadonlySet`, the pass's file set — every index derived from it, + * including the derived header-closure sets that `languages/{c,cpp}/ + * scope-resolver.ts` memoize inside an outer per-file-set memo. Defeating + * one of these means copying the SET, which re-traverses it, which the + * `CountingSet` instrument (`test/helpers/counting-file-set.ts`) reads as a + * scan count rising with the import count. + * - `readonly ParsedFile[]`, the pass's parsed-file array. Not derived from + * the file set at all, so the file-set guards do not reach them; these key + * on the array the orchestrator already threads through the pass, and their + * contract is that same pass-through discipline. The instrument that CAN see + * them counts element reads on that array — `countedParsedFiles`, beside + * `CountingSet`, driven by the contract test's `minimumParsedFileReads`. * - * No index derived from the file set is keyed on an ARRAY materialized from it. - * `import-resolvers/csharp.ts` was, until #2911, and the reason it is worth - * stating as a rule: copying an array mints a fresh `WeakMap` key while - * traversing the Set zero extra times, so every scan-counting guard stays green - * at its correct value while the index rebuilds once per import. That failure - * is invisible to the whole instrument family above and was caught only by a - * timing ratio in `bench/import-target/`. Derive the array inside the builder - * from `getWorkspaceFileIndex(allFilePaths)` instead, and the Set stays the - * single key shape and the single thing that has to be passed through. + * A THIRD shape — an array materialized from the file set — is what the type + * exists to reject. `import-resolvers/csharp.ts` used one until #2911, and it + * is worth a compile error rather than a rule: copying an array mints a fresh + * `WeakMap` key while traversing the Set zero extra times, so every + * scan-counting guard stays green at its correct value while the index rebuilds + * once per import. That failure is invisible to the whole instrument family + * above and was caught only by a timing ratio in `bench/import-target/`. Derive + * the array inside the builder from `getWorkspaceFileIndex(allFilePaths)` + * instead. `string[]` is not assignable to `K`, so the shape cannot come back + * silently — `configs/swift.ts` keeps the one hand-rolled `WeakMap` on + * `ctx.allFileList` in the tree, deliberately and with its reasons written + * down, and it is deliberately NOT on this primitive. * * `T extends object` is deliberate, chosen over probing `has` before `get`. * `WeakMap.get` returning `undefined` cannot distinguish "not built yet" from @@ -64,7 +67,9 @@ * Inert for the builders here — each is a pure, total pass over the file set — * and the safer of the two behaviours if that ever stops being true. */ -export function perFileSet( +import type { ParsedFile } from 'gitnexus-shared'; + +export function perFileSet | readonly ParsedFile[], T extends object>( build: (key: K) => T, ): (key: K) => T { const cache = new WeakMap(); diff --git a/gitnexus/src/core/ingestion/import-resolvers/python-file-index.ts b/gitnexus/src/core/ingestion/import-resolvers/python-file-index.ts index 583497fd1..d4418ce90 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/python-file-index.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/python-file-index.ts @@ -52,10 +52,24 @@ import { perFileSet } from './per-file-set.js'; * computed it — `norm.split('/').slice(0, -1).join('/')`, which for a path * without a separator is `''` (a root-level importer, whose chain is empty). */ -export function importerAncestors(index: PythonFileIndex, fromFile: string): readonly string[] { +/** + * The importer's own directory, normalized — the key BOTH per-directory memos + * below are stored under. + * + * One exported derivation rather than one per accessor: the two memos live in + * the same index and must agree on what "the importer's directory" is, and a + * caller that already holds the directory (the bare-import tier computes it for + * its own proximity check) should not pay for it twice. It was three copies of + * `replace / lastIndexOf / slice` across two modules before, byte-identical by + * inspection and by nothing else. + */ +export function importerDirOf(fromFile: string): string { const norm = fromFile.replace(/\\/g, '/'); const lastSlash = norm.lastIndexOf('/'); - const importerDir = lastSlash === -1 ? '' : norm.slice(0, lastSlash); + return lastSlash === -1 ? '' : norm.slice(0, lastSlash); +} + +export function importerAncestors(index: PythonFileIndex, importerDir: string): readonly string[] { const memoized = index.ancestorsByDir.get(importerDir); if (memoized !== undefined) return memoized; const built = buildImporterAncestors(importerDir); @@ -158,14 +172,24 @@ export const getPythonFileIndex = perFileSet( if (!norm.endsWith('.py')) continue; normSet.add(norm); + // ONE entry object per file, shared by both buckets below: a package file + // lands in `byBasename` and `byInitParent`, and two literals for the same + // `(raw, norm)` pair cost ~40 B each on every `__init__.py`. + const entry = { raw, norm }; + const lastSlash = norm.lastIndexOf('/'); const base = lastSlash >= 0 ? norm.slice(lastSlash + 1) : norm; - let bucket = byBasename.get(base); - if (bucket === undefined) { - bucket = []; - byBasename.set(base, bucket); - } - bucket.push({ raw, norm }); + // `set(base, [entry])` rather than `set(base, [])` then `push`: an empty + // array literal that is immediately pushed to makes V8 grow the backing + // store to its 16-slot minimum, so every bucket holding ONE file retains + // 15 empty pointer slots — 128 B — for the whole pass. `byBasename` has + // roughly one bucket per file, which made that the dominant term in this + // index: measured 5.50 MiB against 1.60 MiB for the one-element form at + // 32 000 `.py` paths, byte-identical contents. Same shape as + // `languages/php/import-target.ts`'s directory buckets. + const bucket = byBasename.get(base); + if (bucket === undefined) byBasename.set(base, [entry]); + else bucket.push(entry); // Package files also get a parent-keyed bucket so a `pkg.sub` lookup hits // only `…/sub/__init__.py` candidates, not every `__init__.py` (P2b). @@ -175,12 +199,9 @@ export const getPythonFileIndex = perFileSet( const parentName = parentSlash >= 0 ? dir.slice(parentSlash + 1) : dir; if (parentName) { const initKey = `${parentName}/__init__.py`; - let ib = byInitParent.get(initKey); - if (ib === undefined) { - ib = []; - byInitParent.set(initKey, ib); - } - ib.push({ raw, norm }); + const ib = byInitParent.get(initKey); + if (ib === undefined) byInitParent.set(initKey, [entry]); + else ib.push(entry); } } diff --git a/gitnexus/src/core/ingestion/import-resolvers/python.ts b/gitnexus/src/core/ingestion/import-resolvers/python.ts index b82323e38..9914a7613 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/python.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/python.ts @@ -9,6 +9,7 @@ import { getPythonFileIndex, importerBarePrefixes, + importerDirOf, pythonSegmentAbsent, } from './python-file-index.js'; import { tryResolveWithExtensions } from './utils.js'; @@ -69,13 +70,11 @@ export function resolvePythonImportInternal( const index = getPythonFileIndex(allFiles); if (pythonSegmentAbsent(index, pathLike)) return null; - // Normalize for Windows backslashes. `lastIndexOf` rather than - // `split('/').slice(0, -1).join('/')`: identical for every input (a path with - // no separator has no directory, which is `''` both ways) without the - // per-import array of one element per path component. - const norm = currentFile.replace(/\\/g, '/'); - const lastSlash = norm.lastIndexOf('/'); - const importerDir = lastSlash === -1 ? '' : norm.slice(0, lastSlash); + // One derivation, shared with the index's other per-directory memo — see + // `importerDirOf`. It replaced `split('/').slice(0, -1).join('/')`: identical + // for every input (a path with no separator has no directory, which is `''` + // both ways) without the per-import array of one element per path component. + const importerDir = importerDirOf(currentFile); // Proximity check — only applies when the importer lives in a subdirectory. // Root-level importers (importerDir === '') skip straight to the ancestor diff --git a/gitnexus/src/core/ingestion/import-resolvers/utils.ts b/gitnexus/src/core/ingestion/import-resolvers/utils.ts index 0696c4fa2..a6ee1b874 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/utils.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/utils.ts @@ -97,23 +97,18 @@ export interface SuffixIndex { /** * Get all files in a directory suffix. * - * The directory map behind this is built on the FIRST call and memoized — - * see `buildSuffixIndex`. Callers that never ask a directory question never - * pay for it. + * `readonly` is the CONTRACT, and it is the contract for every implementation + * of this interface, not a description of any one of them: an implementation + * is free to return its own bucket by reference, so callers must treat the + * result as shared and never `sort`/`splice` it in place. The compiler now + * refuses that at the call site. Whether a given implementation shares or + * copies is its own business and documented where it is built — + * `buildSuffixIndex` shares, the root-anchored parity index in + * `languages/php/import-target.ts` returns a filtered copy. * - * Returns the index's OWN bucket, by reference, and `readonly` is the - * contract rather than a comment about one: the map is built once and then - * held for the whole pass, so an in-place `sort`/`splice` would corrupt every - * later import in that pass, and the compiler now refuses it at the call - * site. Sharing beats copying here because no caller keeps the array — of the - * four (see `dirMap` below) two only measure it (`configs/python.ts` reads - * `.length`, `php.ts` reads `[0]`) and two build a fresh array from it - * (`jvm.ts` flatMaps, `csharp.ts` pushes into its own `results`) — so a - * defensive copy would allocate a whole bucket per import on the path this - * index exists to keep flat. `package-dir-index.ts` reached the same - * conclusion for the same reason and states it the same way, with read-only - * containers plus one copy where a bucket genuinely LEAVES (`sortedRootFiles`); - * `configs/swift.ts` is that leaving case and copies. + * Implementations that memoize should note the directory map behind this may + * be built on the FIRST call rather than up front, so a caller that never + * asks a directory question never pays for it — see `buildSuffixIndex`. */ getFilesInDir(dirSuffix: string, extension: string): readonly string[]; } @@ -330,6 +325,18 @@ export function buildSuffixIndex( return { get: (suffix: string) => getExactMap().get(suffix), getInsensitive: (suffix: string) => getLowerMap().get(suffix.toLowerCase()), + // THIS implementation shares: it hands back `dirMap`'s own bucket rather + // than a copy. The map is built on first query and then held for the whole + // pass, so the window in which a mutating caller could corrupt later + // imports is the whole pass — which is why the interface makes the result + // `readonly` and the compiler refuses the mutation at the call site. + // + // Sharing beats copying because no caller keeps the array: two only measure + // it and two build a fresh array from it, so a defensive copy would + // allocate a whole bucket per import on the path this index exists to keep + // flat. `package-dir-index.ts` reached the same conclusion the same way — + // read-only containers, plus one copy where a bucket genuinely LEAVES + // (`sortedRootFiles`), which is the case `configs/swift.ts` is in. getFilesInDir: (dirSuffix: string, extension: string) => { return getDirMap().get(`${dirSuffix}:${extension}`) || []; }, diff --git a/gitnexus/src/core/ingestion/languages/python/import-target.ts b/gitnexus/src/core/ingestion/languages/python/import-target.ts index c61319945..e31f038b0 100644 --- a/gitnexus/src/core/ingestion/languages/python/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/python/import-target.ts @@ -12,7 +12,11 @@ import type { ParsedFile, ParsedImport, WorkspaceIndex } from 'gitnexus-shared'; import { perFileSet } from '../../import-resolvers/per-file-set.js'; -import { getPythonFileIndex, importerAncestors } from '../../import-resolvers/python-file-index.js'; +import { + getPythonFileIndex, + importerAncestors, + importerDirOf, +} from '../../import-resolvers/python-file-index.js'; import { resolvePythonImportInternal } from '../../import-resolvers/python.js'; export interface PythonResolveContext { @@ -284,7 +288,7 @@ function resolveAbsoluteFromFiles( // The chain comes from `importerAncestors`, which builds it ONCE per importer // directory per pass. Rebuilding it here — one `slice(0, i).join('/')` per // path component, on every import — was half of the depth quadratic in #2913. - for (const ancestor of importerAncestors(index, fromFile)) { + for (const ancestor of importerAncestors(index, importerDirOf(fromFile))) { if (mayBeModule) { const candidateFile = `${ancestor}/${directFile}`; if (allFilePaths.has(candidateFile)) return candidateFile; @@ -394,7 +398,7 @@ function hasRepoCandidate( // only be a directory prefix if `seg` names a directory under the non-empty // parent `A`, so a miss here means the old loop would have missed too. if (!index.nestedDirNames.has(leadingSegment)) return false; - for (const ancestor of importerAncestors(index, fromFile)) { + for (const ancestor of importerAncestors(index, importerDirOf(fromFile))) { if (index.dirPrefixes.has(`${ancestor}/${prefix}`)) return true; } return false; diff --git a/gitnexus/test/helpers/counting-file-set.ts b/gitnexus/test/helpers/counting-file-set.ts index b0d702705..45bc6cb38 100644 --- a/gitnexus/test/helpers/counting-file-set.ts +++ b/gitnexus/test/helpers/counting-file-set.ts @@ -57,10 +57,14 @@ * * `expectDistinctFileSetsGetOwnIndex` below is the one arm of those guards that * is identical in every language once the four values that differ are named, so - * it lives here beside the instrument it reads rather than in each guard. + * it lives here beside the instrument it reads rather than in each guard. The + * `ChainMemoArm` section at the bottom applies the same rule to the guards that + * watch a MEMO instead of a scan count — the two Python importer-chain guards, + * which this instrument provably cannot see (their headers say why) and which + * were arm-for-arm the same suite written twice. */ import { expect } from 'vitest'; -import type { ParsedFile } from 'gitnexus-shared'; +import type { ParsedFile, ParsedImport } from 'gitnexus-shared'; import type { ScopeResolver } from '../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; export class CountingSet extends Set { @@ -245,3 +249,240 @@ export function expectDistinctFileSetsGetOwnIndex(arm: DistinctFileSetArm): void expect(a.scans).toBe(arm.expectedScans); expect(b.scans).toBe(arm.expectedScans); } + +// ─── Python import shapes ──────────────────────────────────────────────────── + +/** + * `from import Widget`. The shape that makes + * `resolvePythonImportTarget` run the package-attribute probe + * (`pythonFileExportsName`, the `context.parsedFiles` reader) ahead of the + * submodule fallback — so it is the shape that re-enters the resolver and pays + * the importer's chain TWICE, and the shape `pythonImportedSubmoduleTarget` + * fires for. + * + * The default the adapter synthesizes when `context` is absent is a `namespace` + * import, and that shape never reaches the probe. A guard that means to measure + * either leg therefore has to pass this one explicitly. + */ +export const pythonNamedImport = (targetRaw: string): ParsedImport => ({ + kind: 'named', + localName: 'Widget', + importedName: 'Widget', + targetRaw, +}); + +/** `import ` — the single-walk shape, and the adapter's default. */ +export const pythonNamespaceImport = (targetRaw: string): ParsedImport => ({ + kind: 'namespace', + localName: '_', + importedName: '_', + targetRaw, +}); + +/** + * ONE array per file that uses it, never a fresh `[]` per call: + * `parsedFileByPath` memoizes on its identity, and a new array per import would + * mint a `WeakMap` key per import for a channel these guards are not measuring + * (`countedParsedFiles` above is the instrument for that one). Empty, so + * `pythonFileExportsName` answers false and the package-vs-submodule precedence + * never fires — the walk, not the precedence, is what the numbers measure. + */ +export const NO_PARSED_FILES: readonly ParsedFile[] = []; + +// ─── the Python importer-chain memo guards ─────────────────────────────────── + +/** + * What one resolution answered, as the chain-memo arms read it: a path, a path + * list (the `ScopeResolver` signature allows one), or `null`. The two values + * the non-vacuity pairing rule counts are `arm.hitResult` and `null`. + */ +export type ChainMemoResult = string | readonly string[] | null; + +/** + * Everything that differs between the two Python importer-chain memo guards: + * `test/unit/import-resolvers/python-importer-prefixes.test.ts` + * (`bareImportPrefixesByDir`) and + * `test/unit/scope-resolution/python/python-importer-ancestors.test.ts` + * (`ancestorsByDir`). + * + * The two memos hold DIFFERENT SEQUENCES under the same key — self included or + * not, workspace root included or not, empty components kept or dropped; see + * `importerBarePrefixes`'s header for why neither guard can be deleted in + * favour of the other. But each guard is the same four arms over the same + * importer corpus once these four values are named, so the arms live here and + * each guard supplies its own four. + */ +export interface ChainMemoArm { + /** The memo under test, read off the pass's per-file-set index. */ + readonly memoOf: (files: ReadonlySet) => ReadonlyMap; + /** + * Drives a production surface `perImporter` times from `fromFile`, with + * spellings that reach the memo, and answers what each call resolved to. + * Exactly one call per invocation must answer `arm.hitResult`, and at least + * one must answer `null`. Must pass `files` THROUGH: both memos are keyed on + * its identity, so a copy here would measure nothing. + */ + readonly drive: ( + files: Set, + fromFile: string, + perImporter: number, + ) => readonly ChainMemoResult[]; + /** + * The verbatim pre-change chain builder, which is the specification: the memo + * agreeing with it is what makes the change a hoist rather than a behaviour + * change. + */ + readonly legacyChain: (fromFile: string) => readonly string[]; + /** What the one must-resolve spelling in `drive` answers, once per importer. */ + readonly hitResult: string; +} + +/** Every file that issues an import in the chain-memo arms. */ +export const CHAIN_MEMO_IMPORTERS: readonly string[] = [ + 'svc/a/one.py', + 'svc/a/two.py', + 'svc/b/one.py', + 'deep/x/y/z/one.py', + 'root.py', +]; + +/** Four directories for those five importers — `svc/a` holds two of them. */ +export const CHAIN_MEMO_IMPORTER_DIRS: readonly string[] = ['svc/a', 'svc/b', 'deep/x/y/z', '']; + +/** The directory two importers share, which is where identity is measured. */ +const SHARED_DIR = 'svc/a'; +const SHARED_DIR_IMPORTERS: readonly string[] = ['svc/a/one.py', 'svc/a/two.py']; + +/** Imports one directory issues before its chain's identity is re-read. */ +const CHAIN_IDENTITY_REPEATS = 40; + +/** A sorted copy, so a key set is compared without depending on fill order. */ +export const sortedStrings = (values: Iterable): string[] => [...values].sort(); + +/** + * The importer directory both memos are keyed on, derived exactly as the + * pre-change inline code derived it — `''` for a path with no separator. + */ +const importerDirOf = (fromFile: string): string => + fromFile.replace(/\\/g, '/').split('/').slice(0, -1).join('/'); + +/** + * The path-shape space an importer chain has to be correct over, as ONE table + * both guards run: they enumerate the same space and nothing kept the two + * copies in lockstep. + * + * Each `why` names the SHAPE, not what either chain does with it, because the + * two chains do different things with several of these rows — the bare-prefix + * chain KEEPS the empty component an absolute path or a doubled separator + * produces and the ancestor chain drops it, and that difference decides real + * resolutions. Which is why every arm below compares against the guard's own + * `legacyChain` rather than against a shared expectation. + */ +export const IMPORTER_PATH_SHAPES: readonly { readonly fromFile: string; readonly why: string }[] = + [ + { fromFile: 'svc/a/one.py', why: 'a two-component directory' }, + { fromFile: 'deep/x/y/z/one.py', why: 'a four-component directory' }, + { fromFile: 'root.py', why: 'a workspace-root importer' }, + { fromFile: '/abs/svc/a/one.py', why: 'an absolute path (leading empty component)' }, + { fromFile: 'svc//a/one.py', why: 'a doubled separator (empty component)' }, + { fromFile: 'svc\\a\\one.py', why: 'Windows separators' }, + { fromFile: 'trailing/', why: 'a path ending in a separator' }, + ]; + +/** + * The gate: N imports from five importers over four directories leave FOUR + * entries, for every N. That is "the chain work is O(1) amortized after the + * first import from a given directory", stated as a number. A chain rebuilt per + * import cannot be memoized at all (size 0); a chain keyed on the importing + * FILE reads five. + * + * Paired with the non-vacuity assertions every guard in this family states: a + * perfect memo count is equally true of an adapter that resolves nothing. + */ +export function expectOneChainPerImporterDir( + arm: ChainMemoArm, + files: Set, + perImporter: number, +): void { + const resolved: ChainMemoResult[] = []; + for (const fromFile of CHAIN_MEMO_IMPORTERS) { + resolved.push(...arm.drive(files, fromFile, perImporter)); + } + + expect(arm.memoOf(files).size).toBe(CHAIN_MEMO_IMPORTER_DIRS.length); + expect(sortedStrings(arm.memoOf(files).keys())).toEqual(sortedStrings(CHAIN_MEMO_IMPORTER_DIRS)); + + expect(resolved.filter((value) => value === arm.hitResult)).toHaveLength( + CHAIN_MEMO_IMPORTERS.length, + ); + expect(resolved.filter((value) => value === null).length).toBeGreaterThan(0); +} + +/** + * The stored-object arm: a memo that stores a FRESH chain on every import posts + * a perfect size while doing all of the work again, so the size gate above is + * paired with reference identity across many later imports from the same + * directory — issued from BOTH files in it, so a chain keyed on the importing + * file would be replaced rather than reused. + * + * Contents are asserted FIRST: `toBe` against an absent entry would pass on + * `undefined === undefined` if the memo were deleted outright. + */ +export function expectSameChainObjectReused(arm: ChainMemoArm, files: Set): void { + const [firstImporter] = SHARED_DIR_IMPORTERS; + arm.drive(files, firstImporter, 1); + const first = arm.memoOf(files).get(SHARED_DIR); + expect(first).toEqual(arm.legacyChain(firstImporter)); + + for (const fromFile of SHARED_DIR_IMPORTERS) { + arm.drive(files, fromFile, CHAIN_IDENTITY_REPEATS); + } + + expect(arm.memoOf(files).get(SHARED_DIR)).toBe(first); +} + +/** + * The legacy-equality arm for one path shape: what the memo stored under + * `fromFile`'s directory is what the pre-change inline code built for it. + * + * Returns the memoized chain, so a guard whose memo feeds a SECOND consumer can + * go on to assert that consumer's derived form of it. + */ +export function expectMemoizedChainMatchesLegacy( + arm: ChainMemoArm, + files: Set, + fromFile: string, +): readonly string[] { + arm.drive(files, fromFile, 1); + + const chain = arm.memoOf(files).get(importerDirOf(fromFile)); + expect(chain).toEqual(arm.legacyChain(fromFile)); + return chain ?? []; +} + +/** + * The distinct-file-set arm: two independently built file sets each get their + * own memo — equal in content, never the same object, neither leaking into the + * other. The two are driven interleaved, so a memo keyed on anything but the + * Set's identity shows up here as a SHARED entry rather than as a stale one. + */ +export function expectDistinctFileSetsGetOwnChainMemo( + arm: ChainMemoArm, + a: Set, + b: Set, + perImporter: number, +): void { + for (const fromFile of CHAIN_MEMO_IMPORTERS) { + arm.drive(a, fromFile, perImporter); + arm.drive(b, fromFile, perImporter); + } + + const memoA = arm.memoOf(a); + const memoB = arm.memoOf(b); + + expect(memoA).not.toBe(memoB); + expect(memoA.get(SHARED_DIR)).not.toBe(memoB.get(SHARED_DIR)); + expect(memoA.get(SHARED_DIR)).toEqual(memoB.get(SHARED_DIR)); + expect(memoA.size).toBe(CHAIN_MEMO_IMPORTER_DIRS.length); + expect(memoB.size).toBe(CHAIN_MEMO_IMPORTER_DIRS.length); +} diff --git a/gitnexus/test/integration/csharp-import-index-reuse.test.ts b/gitnexus/test/integration/csharp-import-index-reuse.test.ts index 10f6cd8b9..8f4978260 100644 --- a/gitnexus/test/integration/csharp-import-index-reuse.test.ts +++ b/gitnexus/test/integration/csharp-import-index-reuse.test.ts @@ -54,22 +54,41 @@ const { resolveImportTarget } = csharpScopeResolver; const FROM_FILE = 'App/Program.cs'; +/** Where a workspace's padded filler files go, and what is appended after them. */ +interface WorkspaceLayout { + /** Directory the filler files live in. */ + readonly dir: string; + /** Basename stem the filler files are numbered from. */ + readonly stem: string; + /** The files the resolutions actually target, appended in order. */ + readonly extras: readonly string[]; +} + +/** + * `fileCount` filler files under `layout.dir`, then `layout.extras`. The filler + * is what makes a traversal expensive enough for a per-`using` rebuild to be a + * different number rather than a different constant; the counting instrument + * reads the traversals either way. + */ +function buildWorkspace(fileCount: number, layout: WorkspaceLayout): CountingSet { + const files: string[] = []; + for (let i = 0; i < fileCount; i++) { + files.push(`${layout.dir}/${layout.stem}${String(i).padStart(5, '0')}.cs`); + } + return new CountingSet([...files, ...layout.extras]); +} + /** * A synthetic C# solution with no `.csproj` discovered, which is the leg #2878 * moved onto the indexes. `App/Models/User.cs` answers the whole-path lookup, * `App/Services/` answers the namespace-directory lookup, and `Domain/Order.cs` * is reachable only after progressive prefix stripping. */ -function buildWorkspace(fileCount: number): CountingSet { - const files: string[] = []; - for (let i = 0; i < fileCount; i++) { - files.push(`App/Services/Service${String(i).padStart(5, '0')}.cs`); - } - files.push('App/Models/User.cs'); - files.push('Domain/Order.cs'); - files.push('App/Program.cs'); - return new CountingSet(files); -} +const NO_CSPROJ_LAYOUT: WorkspaceLayout = { + dir: 'App/Services', + stem: 'Service', + extras: ['App/Models/User.cs', 'Domain/Order.cs', 'App/Program.cs'], +}; /** The one `.csproj` config that puts the adapter on the csproj leg. */ const CSPROJ_CONFIG: CsharpResolutionConfig = { @@ -88,18 +107,15 @@ const CSPROJ_CONFIG: CsharpResolutionConfig = { * A layout where the first two legs answer would leave the index unbuilt and * the build count blind to the very copy it is here to catch. */ -function buildCsprojWorkspace(fileCount: number): CountingSet { - const files: string[] = []; - for (let i = 0; i < fileCount; i++) { - files.push(`src/MyApp/Models/Entity${String(i).padStart(5, '0')}.cs`); - } - files.push('App/Program.cs'); - return new CountingSet(files); -} +const CSPROJ_LAYOUT: WorkspaceLayout = { + dir: 'src/MyApp/Models', + stem: 'Entity', + extras: ['App/Program.cs'], +}; describe('C# import resolution — index reuse across usings (#2878)', () => { it('builds each index once for many usings over a stable file set', () => { - const files = buildWorkspace(300); + const files = buildWorkspace(300, NO_CSPROJ_LAYOUT); const resolved: (string | readonly string[] | null)[] = []; for (let i = 0; i < 200; i++) { @@ -126,7 +142,7 @@ describe('C# import resolution — index reuse across usings (#2878)', () => { it('a distinct file set gets its own indexes (no stale cross-run reuse)', () => { expectDistinctFileSetsGetOwnIndex({ resolveImportTarget, - buildWorkspace: () => buildWorkspace(20), + buildWorkspace: () => buildWorkspace(20, NO_CSPROJ_LAYOUT), targetRaw: 'App.Models.User', fromFile: FROM_FILE, resolutionConfig: undefined, @@ -138,7 +154,7 @@ describe('C# import resolution — index reuse across usings (#2878)', () => { }); it('still resolves real usings correctly (the perf test is not vacuous)', () => { - const files = buildWorkspace(5); + const files = buildWorkspace(5, NO_CSPROJ_LAYOUT); // Whole-path match on the namespace path. expect(resolveImportTarget('App.Models.User', FROM_FILE, files, undefined)).toBe( @@ -161,7 +177,7 @@ describe('C# import resolution — index reuse across usings (#2878)', () => { describe('C# import resolution — index reuse on the csproj leg (#2911)', () => { it('builds each index once for many usings over a stable file set', () => { - const files = buildCsprojWorkspace(300); + const files = buildWorkspace(300, CSPROJ_LAYOUT); const resolved: (string | readonly string[] | null)[] = []; for (let i = 0; i < 200; i++) { @@ -188,7 +204,7 @@ describe('C# import resolution — index reuse on the csproj leg (#2911)', () => it('a distinct file set gets its own indexes (no stale cross-run reuse)', () => { expectDistinctFileSetsGetOwnIndex({ resolveImportTarget, - buildWorkspace: () => buildCsprojWorkspace(20), + buildWorkspace: () => buildWorkspace(20, CSPROJ_LAYOUT), targetRaw: 'App.Models', fromFile: FROM_FILE, resolutionConfig: CSPROJ_CONFIG, diff --git a/gitnexus/test/integration/python-import-index-reuse.test.ts b/gitnexus/test/integration/python-import-index-reuse.test.ts index 87229ed70..46b07afd9 100644 --- a/gitnexus/test/integration/python-import-index-reuse.test.ts +++ b/gitnexus/test/integration/python-import-index-reuse.test.ts @@ -1,7 +1,8 @@ /** * Production-path regression guard for PR #1918 review finding P1. * - * The Python file index (`getPythonFileIndex` in `import-target.ts`) is + * The Python file index (`getPythonFileIndex` in + * `import-resolvers/python-file-index.ts`) is * memoized on the `allFilePaths` Set identity via a WeakMap. The registry- * primary path reaches it through `pythonScopeResolver.resolveImportTarget` * (the orchestrator adapter) — NOT by calling `resolvePythonImportTarget` diff --git a/gitnexus/test/unit/import-resolvers/python-importer-prefixes.test.ts b/gitnexus/test/unit/import-resolvers/python-importer-prefixes.test.ts index d892adc65..c6e2fcc22 100644 --- a/gitnexus/test/unit/import-resolvers/python-importer-prefixes.test.ts +++ b/gitnexus/test/unit/import-resolvers/python-importer-prefixes.test.ts @@ -40,13 +40,22 @@ * pipeline's, via `ImportTargetWorkspace`'s shared `ResolveCtx`). They thread * different objects around the same Set, and the memo is keyed on the Set. * - * `legacyPrefixes` / `legacyImporterDir` are verbatim copies of the pre-change - * inline code, in the house style of `python-importer-ancestors.test.ts`: they - * are the specification, and the memo agreeing with them is what makes this a - * hoist rather than a behaviour change. + * `legacyPrefixes` is a verbatim copy of the pre-change inline code, in the + * house style of `python-importer-ancestors.test.ts`: it is the specification, + * and the memo agreeing with it is what makes this a hoist rather than a + * behaviour change. + * + * The four arms themselves live in `test/helpers/counting-file-set.ts`, beside + * the other import-target scaffolding: this guard and the #2913 one are the + * same suite over the same importer corpus once four values are named (the + * memo, the drive, the legacy builder, the hit), and they were previously + * written out twice. The two chains still DIFFER — this one keeps the empty + * components an absolute path or a doubled separator produces, and #2913's + * drops them — which is why `legacyChain` is per-guard and the shared path-shape + * table names shapes rather than expectations. */ import { describe, expect, it } from 'vitest'; -import type { ParsedFile, ParsedImport } from 'gitnexus-shared'; +import type { ParsedImport } from 'gitnexus-shared'; import type { ImportResolutionContext } from '../../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; import { pythonScopeResolver } from '../../../src/core/ingestion/languages/python/scope-resolver.js'; import { resolvePythonImportInternal } from '../../../src/core/ingestion/import-resolvers/python.js'; @@ -61,10 +70,22 @@ import type { ImportResult, ResolveCtx, } from '../../../src/core/ingestion/import-resolvers/types.js'; +import { + IMPORTER_PATH_SHAPES, + NO_PARSED_FILES, + expectDistinctFileSetsGetOwnChainMemo, + expectMemoizedChainMatchesLegacy, + expectOneChainPerImporterDir, + expectSameChainObjectReused, + pythonNamedImport, + pythonNamespaceImport, + type ChainMemoArm, + type ChainMemoResult, +} from '../../helpers/counting-file-set.js'; const { resolveImportTarget } = pythonScopeResolver; -// ─── verbatim pre-change implementations ───────────────────────────────────── +// ─── verbatim pre-change implementation ────────────────────────────────────── /** The prefix sequence the inline walk materialized on every import. */ function legacyPrefixes(currentFile: string): string[] { @@ -78,11 +99,6 @@ function legacyPrefixes(currentFile: string): string[] { return prefixes; } -/** The importer directory the inline code derived, and now the memo KEY. */ -function legacyImporterDir(currentFile: string): string { - return currentFile.replace(/\\/g, '/').split('/').slice(0, -1).join('/'); -} - // ─── surfaces ──────────────────────────────────────────────────────────────── /** @@ -109,35 +125,20 @@ function makeResolveCtx(allFilePaths: Set): ResolveCtx { }; } -const namespaceImport = (targetRaw: string): ParsedImport => ({ - kind: 'namespace', - localName: '_', - importedName: '_', - targetRaw, -}); - -/** The kind that re-enters `resolvePythonImportTarget` and walks twice. */ -const namedImport = (targetRaw: string): ParsedImport => ({ - kind: 'named', - localName: 'Widget', - importedName: 'Widget', - targetRaw, -}); - -/** - * ONE array, not a fresh `[]` per call: `parsedFileByPath` memoizes on its - * identity, and a new array per import would mint a `WeakMap` key per import - * for a channel this file is not measuring. Empty, so `pythonFileExportsName` - * answers false and the package-vs-submodule precedence never fires — the - * walk, not the precedence, is under test here. - */ -const NO_PARSED_FILES: readonly ParsedFile[] = []; - const ctxFor = (parsedImport: ParsedImport): ImportResolutionContext => ({ parsedFiles: NO_PARSED_FILES, parsedImport, }); +/** + * An `ImportResult` as the arms read it: the resolved path, or `null` for a + * miss. A stop-the-chain result carrying no files joins to `''`, which is + * neither the hit nor a miss — the same distinction the pre-collapse arm drew + * with `value?.kind === 'files' && value.files.join() === HIT_RESULT`. + */ +const resolvedPath = (result: ImportResult): ChainMemoResult => + result === null ? null : result.files.join(); + // ─── the workspace the surfaces are driven against ─────────────────────────── /** @@ -164,91 +165,79 @@ const WORKSPACE: readonly string[] = [ 'root.py', ]; -/** Every file that issues an import below. */ -const IMPORTERS: readonly string[] = [ - 'svc/a/one.py', - 'svc/a/two.py', - 'svc/b/one.py', - 'deep/x/y/z/one.py', - 'root.py', -]; - -/** Four directories for five importers — `svc/a` holds two of them. */ -const IMPORTER_DIRS: readonly string[] = ['svc/a', 'svc/b', 'deep/x/y/z', '']; - const HIT_TARGET = 'shared'; const HIT_RESULT = 'shared.py'; -/** Survives the absence proof, then walks the whole chain and misses. */ +/** Survives the absence proof, then walks the whole chain and misses — the + * dotted tier's suffix fallback picks it up as `elsewhere/deep/probe.py`. */ const WALK_TARGET = 'probe'; -/** …and is then picked up by the dotted tier's suffix fallback. */ -const WALK_RESULT = 'elsewhere/deep/probe.py'; /** Provably absent: retired before the walk, so it never touches the memo. */ const ABSENT_TARGET = 'ghostmod'; -const sorted = (values: Iterable): string[] => [...values].sort(); +/** + * The spelling sequence BOTH surfaces are driven with, `perImporter` rounds of + * it plus the one spelling that must resolve: one target that walks the whole + * chain and misses, one that the absence proof retires before the walk, and one + * that hits at the workspace root. No spelling is varied per round — the Python + * chain keeps no per-target cache, so a repeated target really is re-resolved. + * + * `resolve` is the only thing that differs between the orchestrator adapter and + * the import-resolver pipeline; everything about the sequence is shared, which + * is why the two used to be the same loop written twice. + */ +function drive( + fromFile: string, + perImporter: number, + resolve: (targetRaw: string, fromFile: string) => T, +): T[] { + const out: T[] = []; + for (let i = 0; i < perImporter; i++) { + for (const target of [WALK_TARGET, ABSENT_TARGET]) out.push(resolve(target, fromFile)); + } + out.push(resolve(HIT_TARGET, fromFile)); + return out; +} + +/** The ORCHESTRATOR ADAPTER — the surface that pays the walk twice. */ +const adapterArm = (mkImport: (targetRaw: string) => ParsedImport): ChainMemoArm => ({ + memoOf: prefixMemo, + drive: (files, fromFile, perImporter) => + drive(fromFile, perImporter, (target, from) => + resolveImportTarget(target, from, files, undefined, ctxFor(mkImport(target))), + ), + legacyChain: legacyPrefixes, + hitResult: HIT_RESULT, +}); /** - * Drives the ORCHESTRATOR ADAPTER, `perImporter` times per importer, with three - * spellings: one that walks the whole chain and misses, one that is retired by - * the absence proof before the walk, and one that hits at the workspace root. - * No spelling is varied per iteration — the Python chain keeps no per-target - * cache, so a repeated target really is re-resolved. + * The import-resolver pipeline surface. One `ResolveCtx` per drive rather than + * one for the run, on purpose: everything the strategy reads off it is derived + * from the same Set (and `pythonImportStrategy` only ever WRITES + * `resolveCache`), so a fresh ctx around the same Set is exactly the "different + * objects, same Set" case the memo has to survive. */ -function driveAdapter( - files: ReadonlySet, - perImporter: number, - mkImport: (targetRaw: string) => ParsedImport, -): (string | readonly string[] | null)[] { - const out: (string | readonly string[] | null)[] = []; - for (const fromFile of IMPORTERS) { - for (let i = 0; i < perImporter; i++) { - for (const target of [WALK_TARGET, ABSENT_TARGET]) { - out.push(resolveImportTarget(target, fromFile, files, undefined, ctxFor(mkImport(target)))); - } - } - out.push( - resolveImportTarget(HIT_TARGET, fromFile, files, undefined, ctxFor(mkImport(HIT_TARGET))), +const strategyArm: ChainMemoArm = { + memoOf: prefixMemo, + drive: (files, fromFile, perImporter) => { + const ctx = makeResolveCtx(files); + return drive(fromFile, perImporter, (target, from) => + resolvedPath(pythonImportStrategy(target, from, ctx)), ); - } - return out; -} - -/** The same drive against the import-resolver pipeline surface. */ -function driveStrategy(ctx: ResolveCtx, perImporter: number): ImportResult[] { - const out: ImportResult[] = []; - for (const fromFile of IMPORTERS) { - for (let i = 0; i < perImporter; i++) { - out.push(pythonImportStrategy(WALK_TARGET, fromFile, ctx)); - out.push(pythonImportStrategy(ABSENT_TARGET, fromFile, ctx)); - } - out.push(pythonImportStrategy(HIT_TARGET, fromFile, ctx)); - } - return out; -} + }, + legacyChain: legacyPrefixes, + hitResult: HIT_RESULT, +}; describe('Python bare-import prefix memo', () => { it.each([ - { perImporter: 1, kind: 'namespace', mkImport: namespaceImport }, - { perImporter: 40, kind: 'namespace', mkImport: namespaceImport }, - { perImporter: 1, kind: 'named', mkImport: namedImport }, - { perImporter: 40, kind: 'named', mkImport: namedImport }, + { perImporter: 1, kind: 'namespace', mkImport: pythonNamespaceImport }, + { perImporter: 40, kind: 'namespace', mkImport: pythonNamespaceImport }, + { perImporter: 1, kind: 'named', mkImport: pythonNamedImport }, + { perImporter: 40, kind: 'named', mkImport: pythonNamedImport }, ])( 'holds one chain per importer DIRECTORY, not per import — $perImporter x $kind', ({ perImporter, mkImport }) => { - const files = new Set(WORKSPACE); - const resolved = driveAdapter(files, perImporter, mkImport); - - // The gate. Five importers over four directories, any number of imports: - // four entries. A chain rebuilt per import cannot be memoized at all - // (size 0); a chain keyed on the importing FILE reads five. - expect(prefixMemo(files).size).toBe(IMPORTER_DIRS.length); - expect(sorted(prefixMemo(files).keys())).toEqual(sorted(IMPORTER_DIRS)); - - // Non-vacuity: a perfect memo count is equally true of an adapter that - // resolves nothing. - expect(resolved.filter((value) => value === HIT_RESULT)).toHaveLength(IMPORTERS.length); - expect(resolved.filter((value) => value === null).length).toBeGreaterThan(0); + expectOneChainPerImporterDir(adapterArm(mkImport), new Set(WORKSPACE), perImporter); }, ); @@ -258,76 +247,24 @@ describe('Python bare-import prefix memo', () => { ])( 'holds one chain per importer DIRECTORY on the import-resolver surface too — $label', ({ perImporter }) => { - const files = new Set(WORKSPACE); - const ctx = makeResolveCtx(files); - const resolved = driveStrategy(ctx, perImporter); - - expect(prefixMemo(files).size).toBe(IMPORTER_DIRS.length); - expect(sorted(prefixMemo(files).keys())).toEqual(sorted(IMPORTER_DIRS)); - - expect( - resolved.filter((value) => value?.kind === 'files' && value.files.join() === HIT_RESULT), - ).toHaveLength(IMPORTERS.length); - expect(resolved.filter((value) => value === null).length).toBeGreaterThan(0); + expectOneChainPerImporterDir(strategyArm, new Set(WORKSPACE), perImporter); }, ); it('reuses the SAME chain object, rather than rebuilding and re-storing it', () => { - const files = new Set(WORKSPACE); - resolveImportTarget( - WALK_TARGET, - 'svc/a/one.py', - files, - undefined, - ctxFor(namedImport(WALK_TARGET)), - ); - const first = prefixMemo(files).get('svc/a'); - - // Contents first: `toBe` against an absent entry would pass on - // `undefined === undefined` if the memo were deleted outright. - expect(first).toEqual(['svc/', '']); - - for (let i = 1; i <= 40; i++) { - resolveImportTarget( - WALK_TARGET, - 'svc/a/one.py', - files, - undefined, - ctxFor(namedImport(WALK_TARGET)), - ); - resolveImportTarget( - HIT_TARGET, - 'svc/a/two.py', - files, - undefined, - ctxFor(namespaceImport(HIT_TARGET)), - ); - } - - expect(prefixMemo(files).get('svc/a')).toBe(first); + expectSameChainObjectReused(adapterArm(pythonNamedImport), new Set(WORKSPACE)); }); - it.each([ - { fromFile: 'svc/a/one.py', why: 'a two-component directory' }, - { fromFile: 'deep/x/y/z/one.py', why: 'a four-component directory' }, - { fromFile: 'root.py', why: 'a workspace-root importer (root-only chain)' }, - { fromFile: '/abs/svc/a/one.py', why: 'an absolute path (empty components KEPT)' }, - { fromFile: 'svc//a/one.py', why: 'a doubled separator (empty component KEPT)' }, - { fromFile: 'svc\\a\\one.py', why: 'Windows separators' }, - { fromFile: 'trailing/', why: 'a path ending in a separator' }, - ])('memoizes the chain the pre-change code built — $why', ({ fromFile }) => { - const files = new Set(WORKSPACE); - resolveImportTarget( - WALK_TARGET, - fromFile, - files, - undefined, - ctxFor(namespaceImport(WALK_TARGET)), - ); - - const chain = prefixMemo(files).get(legacyImporterDir(fromFile)); - expect(chain).toEqual(legacyPrefixes(fromFile)); - }); + it.each(IMPORTER_PATH_SHAPES)( + 'memoizes the chain the pre-change code built — $why', + ({ fromFile }) => { + expectMemoizedChainMatchesLegacy( + adapterArm(pythonNamespaceImport), + new Set(WORKSPACE), + fromFile, + ); + }, + ); it.each([ { perImporter: 2, label: 'two imports per importer' }, @@ -335,20 +272,12 @@ describe('Python bare-import prefix memo', () => { ])( 'gives a distinct file set its own memo (no leak across passes) — $label', ({ perImporter }) => { - const a = new Set(WORKSPACE); - const b = new Set(WORKSPACE); - - driveAdapter(a, perImporter, namedImport); - driveAdapter(b, perImporter, namedImport); - - const memoA = prefixMemo(a); - const memoB = prefixMemo(b); - - expect(memoA).not.toBe(memoB); - expect(memoA.get('svc/a')).not.toBe(memoB.get('svc/a')); - expect(memoA.get('svc/a')).toEqual(memoB.get('svc/a')); - expect(memoA.size).toBe(IMPORTER_DIRS.length); - expect(memoB.size).toBe(IMPORTER_DIRS.length); + expectDistinctFileSetsGetOwnChainMemo( + adapterArm(pythonNamedImport), + new Set(WORKSPACE), + new Set(WORKSPACE), + perImporter, + ); }, ); @@ -370,7 +299,7 @@ describe('Python bare-import prefix memo', () => { } paths.push('shared.py'); const files = new Set(paths); - const resolved: (string | readonly string[] | null)[] = []; + const resolved: ChainMemoResult[] = []; for (let d = 0; d < dirs; d++) { for (let i = 0; i < importsPerDir; i++) { @@ -380,7 +309,7 @@ describe('Python bare-import prefix memo', () => { `pkg${d}/nest/file0.py`, files, undefined, - ctxFor(namedImport(HIT_TARGET)), + ctxFor(pythonNamedImport(HIT_TARGET)), ), ); } @@ -462,7 +391,7 @@ describe('Python bare-import ancestor walk — resolution', () => { // strategy must agree: all three reach the same walk. expect(resolvePythonImportInternal(fromFile, 'user', set)).toBe(expected); expect( - resolveImportTarget('user', fromFile, set, undefined, ctxFor(namespaceImport('user'))), + resolveImportTarget('user', fromFile, set, undefined, ctxFor(pythonNamespaceImport('user'))), ).toBe(expected); expect(pythonImportStrategy('user', fromFile, makeResolveCtx(set))).toEqual({ kind: 'files', diff --git a/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts b/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts index 9b824b3a9..04acd1bb4 100644 --- a/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts +++ b/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts @@ -43,19 +43,19 @@ * adapter declares three or four parameters and cannot observe a fifth. Which * ones those are is not a number to maintain here: the per-language * `minimumParsedFileReads` floor in the table IS the record, and it is what a - * new reader has to change. What the `parsedFileReads` arm gates differs by - * language, and only the first of these is a memo: + * new reader has to change. Both languages that read this key memoize on it, + * and the two memos fail differently: * * - PHP: the `filesByDirectory` memo, `perFileSet`-keyed on the `parsedFiles` * array. Defeat it and every import rebuilds a `Map`; the arm reads 603 against 9 (proven by mutation). - * - Python: there is NO memo on this key — `pythonFileExportsName` runs a - * `parsedFiles.find` for each import whose package probe resolves, and the - * misses here do not get that far, so one resolving import costs one scan. - * The arm pins exactly that: the read stays off the per-import path. - * Widening the fixture's `missTarget` to a spelling whose package DOES - * resolve puts it on that path and the arm reads 603 against 9, so it is a - * live gate rather than a `0 === 0`. + * - Python: `parsedFileByPath`, keyed the same way, behind + * `pythonFileExportsName`. It is built by the FIRST import whose package + * probe resolves — one pass over the array — and every later one is a + * `Map.get`, so the floor of 1 is that single build and the equality half + * is what proves it does not repeat. Before that memo the same call was a + * `parsedFiles.find` per resolving import, which is the shape #2901 + * removed on the file-set key; this arm is why it cannot come back. * - Every other language: `0 === 0`, recorded as a floor of 0. A new reader * arrives with that floor already in place and is caught by the equality * half, which needs no per-language knowledge at all. @@ -119,7 +119,11 @@ import type { ParsedImport } from 'gitnexus-shared'; import { SCOPE_RESOLVERS } from '../../../src/core/ingestion/scope-resolution/pipeline/registry.js'; import type { ScopeResolver } from '../../../src/core/ingestion/scope-resolution/contract/scope-resolver.js'; import type { ComposerConfig } from '../../../src/core/ingestion/language-config.js'; -import { CountingSet, countedParsedFiles } from '../../helpers/counting-file-set.js'; +import { + CountingSet, + countedParsedFiles, + pythonNamedImport, +} from '../../helpers/counting-file-set.js'; /** * The minimum a language needs for one `resolveImportTarget` call to reach @@ -233,13 +237,6 @@ const PHP_FUNCTION_IMPORT = (targetRaw: string): ParsedImport => ({ * submodule fallback. The default the adapter synthesizes when `context` is * absent is a `namespace` import, and that shape never reaches the probe. */ -const PYTHON_NAMED_IMPORT = (targetRaw: string): ParsedImport => ({ - kind: 'named', - localName: 'Widget', - importedName: 'Widget', - targetRaw, -}); - const FIXTURES: ReadonlyMap = new Map< SupportedLanguages, ImportTargetFixture @@ -255,10 +252,11 @@ const FIXTURES: ReadonlyMap = new Map< resolutionConfig: undefined, missTarget: (i) => `realpkg.ghost${i}`, hitTarget: 'realpkg.widget', - parsedImport: PYTHON_NAMED_IMPORT, + parsedImport: pythonNamedImport, minimumScans: 1, - // `pythonFileExportsName`'s `parsedFiles.find` on the one import whose - // package probe resolves — the misses never get that far. + // The one build of `parsedFileByPath`, triggered by the single import + // whose package probe resolves — the misses never get that far, and + // every later resolver is a `Map.get` rather than another pass. minimumParsedFileReads: 1, }, ], diff --git a/gitnexus/test/unit/scope-resolution/python/python-import-probe-count.test.ts b/gitnexus/test/unit/scope-resolution/python/python-import-probe-count.test.ts index ddf715d2d..94a22f98c 100644 --- a/gitnexus/test/unit/scope-resolution/python/python-import-probe-count.test.ts +++ b/gitnexus/test/unit/scope-resolution/python/python-import-probe-count.test.ts @@ -44,9 +44,14 @@ * simply stopped working would post a perfect flat line. */ import { describe, expect, it } from 'vitest'; -import type { ParsedFile, ParsedImport } from 'gitnexus-shared'; +import type { ParsedImport } from 'gitnexus-shared'; import { pythonScopeResolver } from '../../../../src/core/ingestion/languages/python/scope-resolver.js'; import { resolvePythonImportInternal } from '../../../../src/core/ingestion/import-resolvers/python.js'; +import { + NO_PARSED_FILES, + pythonNamedImport, + pythonNamespaceImport, +} from '../../../helpers/counting-file-set.js'; const { resolveImportTarget } = pythonScopeResolver; @@ -61,23 +66,9 @@ class ProbeCountingSet extends Set { } } -/** One array for the whole file: `parsedFileByPath` memoizes on its identity, - * and it is empty, so the package-export precedence never fires and the walk - * is what the numbers measure. */ -const NO_PARSED_FILES: readonly ParsedFile[] = []; - -const namespaceImport = (targetRaw: string): ParsedImport => ({ - kind: 'namespace', - localName: '_', - importedName: '_', - targetRaw, -}); -const namedImport = (targetRaw: string): ParsedImport => ({ - kind: 'named', - localName: 'Widget', - importedName: 'Widget', - targetRaw, -}); +/** `import x as y`. The third kind, and the only one of the three that is not + * shared with the memo guards — it reaches the same package-attribute probe + * `pythonNamedImport` does, and both arms below assert they cost the same. */ const aliasImport = (targetRaw: string): ParsedImport => ({ kind: 'alias', localName: 'w', @@ -162,8 +153,8 @@ const DOTTED_SUBMODULE_PROBES: readonly number[] = [5, 6, 8, 12, 20]; describe('Python import probe count', () => { it.each([ - { kind: 'import x (namespace)', mkImport: namespaceImport }, - { kind: 'from x import y (named)', mkImport: namedImport }, + { kind: 'import x (namespace)', mkImport: pythonNamespaceImport }, + { kind: 'from x import y (named)', mkImport: pythonNamedImport }, { kind: 'import x as y (alias)', mkImport: aliasImport }, ])('costs the same for every import KIND — single-segment, resolving — $kind', ({ mkImport }) => { const counted = DEPTHS.map((depth) => probeCount(mkImport, depth, PRESENT_TARGET)); @@ -175,8 +166,16 @@ describe('Python import probe count', () => { }); it.each([ - { kind: 'import x (namespace)', mkImport: namespaceImport, expected: DOTTED_NAMESPACE_PROBES }, - { kind: 'from x import y (named)', mkImport: namedImport, expected: DOTTED_SUBMODULE_PROBES }, + { + kind: 'import x (namespace)', + mkImport: pythonNamespaceImport, + expected: DOTTED_NAMESPACE_PROBES, + }, + { + kind: 'from x import y (named)', + mkImport: pythonNamedImport, + expected: DOTTED_SUBMODULE_PROBES, + }, { kind: 'import x as y (alias)', mkImport: aliasImport, expected: DOTTED_SUBMODULE_PROBES }, ])( 'runs the package tail ONCE for a dotted target that misses — $kind', @@ -191,8 +190,8 @@ describe('Python import probe count', () => { ); it.each([ - { kind: 'import x (namespace)', mkImport: namespaceImport }, - { kind: 'from x import y (named)', mkImport: namedImport }, + { kind: 'import x (namespace)', mkImport: pythonNamespaceImport }, + { kind: 'from x import y (named)', mkImport: pythonNamedImport }, { kind: 'import x as y (alias)', mkImport: aliasImport }, ])('retires a provably absent segment in a CONSTANT probe count — $kind', ({ mkImport }) => { const counted = DEPTHS.map((depth) => probeCount(mkImport, depth, ABSENT_TARGET)); @@ -207,7 +206,9 @@ describe('Python import probe count', () => { // Control for the arm above: the same instrument, the same corpus, the same // depths, one different spelling — and the count triples across the range. // So a flat line means the walk was skipped, not that nothing is counted. - const present = DEPTHS.map((depth) => probeCount(namedImport, depth, PRESENT_TARGET).probes); + const present = DEPTHS.map( + (depth) => probeCount(pythonNamedImport, depth, PRESENT_TARGET).probes, + ); expect(present).toEqual(PRESENT_PROBES); expect(new Set(present).size).toBe(DEPTHS.length); }); diff --git a/gitnexus/test/unit/scope-resolution/python/python-importer-ancestors.test.ts b/gitnexus/test/unit/scope-resolution/python/python-importer-ancestors.test.ts index d771c85e8..f29ac144f 100644 --- a/gitnexus/test/unit/scope-resolution/python/python-importer-ancestors.test.ts +++ b/gitnexus/test/unit/scope-resolution/python/python-importer-ancestors.test.ts @@ -40,12 +40,31 @@ * in the house style of `import-target-index-parity.test.ts`: they are the * specification, and the memo agreeing with them is what makes this a hoist * rather than a behaviour change. + * + * The four memo arms live in `counting-file-set.ts` beside the other + * import-target scaffolding, because this guard and the bare-prefix one + * (`test/unit/import-resolvers/python-importer-prefixes.test.ts`) are the same + * suite over the same importer corpus once four values are named (the memo, the + * drive, the legacy builder, the hit). The two CHAINS still differ — this one + * drops the empty components an absolute path or a doubled separator produces + * and the other keeps them — so `legacyChain` stays per-guard and the shared + * path-shape table names shapes rather than expectations. */ import { describe, expect, it } from 'vitest'; import type { ParsedImport } from 'gitnexus-shared'; import { pythonScopeResolver } from '../../../../src/core/ingestion/languages/python/scope-resolver.js'; import { getPythonFileIndex } from '../../../../src/core/ingestion/import-resolvers/python-file-index.js'; -import { countedParsedFiles } from '../../../helpers/counting-file-set.js'; +import { + IMPORTER_PATH_SHAPES, + countedParsedFiles, + expectDistinctFileSetsGetOwnChainMemo, + expectMemoizedChainMatchesLegacy, + expectOneChainPerImporterDir, + expectSameChainObjectReused, + sortedStrings, + type ChainMemoArm, + type ChainMemoResult, +} from '../../../helpers/counting-file-set.js'; const { resolveImportTarget } = pythonScopeResolver; @@ -104,8 +123,6 @@ function specNestedDirNames(dirPrefixes: ReadonlySet): Set { return names; } -const sorted = (values: Iterable): string[] => [...values].sort(); - // ─── the workspace the adapter is driven against ───────────────────────────── /** @@ -124,18 +141,6 @@ const WORKSPACE: readonly string[] = [ 'root.py', ]; -/** Every file that issues an import below, and the directory it sits in. */ -const IMPORTERS: readonly string[] = [ - 'svc/a/one.py', - 'svc/a/two.py', - 'svc/b/one.py', - 'deep/x/y/z/one.py', - 'root.py', -]; - -/** Four directories for five importers — `svc/a` holds two of them. */ -const IMPORTER_DIRS: readonly string[] = ['svc/a', 'svc/b', 'deep/x/y/z', '']; - /** * One import that must resolve, so a memo count is never the count of an * adapter that has stopped resolving anything (the pairing rule every guard in @@ -146,7 +151,7 @@ const HIT_TARGET = 'outer.nested.mod'; const HIT_RESULT = 'outer/nested/mod.py'; /** - * Drives the ORCHESTRATOR ADAPTER, `perImporter` times per importer, with two + * Drives the ORCHESTRATOR ADAPTER `perImporter` times from `fromFile`, with two * spellings that between them enter the memo from both call sites: * - `nested.ghost{i}` — reaches `hasRepoCandidate`'s ancestor walk and misses. * Spelled differently every iteration, so nothing upstream can answer it @@ -157,84 +162,54 @@ const HIT_RESULT = 'outer/nested/mod.py'; * `one.py` has to be a real basename somewhere or the walk is skipped * before it starts, and the Python chain keeps no per-target cache, so a * repeated spelling really is re-resolved. + * …then the one spelling that must resolve. */ -function driveAdapter( - files: ReadonlySet, +function driveImporter( + files: Set, + fromFile: string, perImporter: number, -): (string | readonly string[] | null)[] { - const out: (string | readonly string[] | null)[] = []; - for (const fromFile of IMPORTERS) { - for (let i = 0; i < perImporter; i++) { - out.push(resolveImportTarget(`nested.ghost${i}`, fromFile, files, undefined, undefined)); - out.push(resolveImportTarget('outer.one', fromFile, files, undefined, undefined)); - } - out.push(resolveImportTarget(HIT_TARGET, fromFile, files, undefined, undefined)); +): ChainMemoResult[] { + const out: ChainMemoResult[] = []; + for (let i = 0; i < perImporter; i++) { + out.push(resolveImportTarget(`nested.ghost${i}`, fromFile, files, undefined, undefined)); + out.push(resolveImportTarget('outer.one', fromFile, files, undefined, undefined)); } + out.push(resolveImportTarget(HIT_TARGET, fromFile, files, undefined, undefined)); return out; } +const ancestorArm: ChainMemoArm = { + memoOf: (files) => getPythonFileIndex(files).ancestorsByDir, + drive: driveImporter, + legacyChain: legacyAncestorChain, + hitResult: HIT_RESULT, +}; + describe('Python importer-ancestor memo (#2913)', () => { it.each([ { perImporter: 1, label: 'one import per importer' }, { perImporter: 40, label: 'forty imports per importer' }, ])('holds one chain per importer DIRECTORY, not per import — $label', ({ perImporter }) => { - const files = new Set(WORKSPACE); - const resolved = driveAdapter(files, perImporter); - - // The gate. Five importers over four directories, any number of imports: - // four entries. A chain rebuilt per import cannot be memoized at all - // (size 0); a chain keyed on the importing FILE reads five. - expect(getPythonFileIndex(files).ancestorsByDir.size).toBe(IMPORTER_DIRS.length); - expect(sorted(getPythonFileIndex(files).ancestorsByDir.keys())).toEqual(sorted(IMPORTER_DIRS)); - - // Non-vacuity: a perfect memo count is equally true of an adapter that - // resolves nothing. - expect(resolved.filter((value) => value === HIT_RESULT)).toHaveLength(IMPORTERS.length); - expect(resolved.filter((value) => value === null).length).toBeGreaterThan(0); + expectOneChainPerImporterDir(ancestorArm, new Set(WORKSPACE), perImporter); }); it('reuses the SAME chain object, rather than rebuilding and re-storing it', () => { - const files = new Set(WORKSPACE); - resolveImportTarget('nested.ghost0', 'svc/a/one.py', files, undefined, undefined); - const first = getPythonFileIndex(files).ancestorsByDir.get('svc/a'); - - // Contents first: `toBe` against an absent entry would pass on - // `undefined === undefined` if the memo were deleted outright. - expect(first).toEqual(['svc/a', 'svc']); - - for (let i = 1; i <= 40; i++) { - resolveImportTarget(`nested.ghost${i}`, 'svc/a/one.py', files, undefined, undefined); - resolveImportTarget('outer.one', 'svc/a/two.py', files, undefined, undefined); - } - - expect(getPythonFileIndex(files).ancestorsByDir.get('svc/a')).toBe(first); + expectSameChainObjectReused(ancestorArm, new Set(WORKSPACE)); }); - it.each([ - { fromFile: 'svc/a/one.py', why: 'a two-component directory' }, - { fromFile: 'deep/x/y/z/one.py', why: 'a four-component directory' }, - { fromFile: 'root.py', why: 'a workspace-root importer (no ancestors)' }, - { fromFile: '/abs/svc/a/one.py', why: 'an absolute path (leading empty part dropped)' }, - { fromFile: 'svc//a/one.py', why: 'a doubled separator (empty part dropped)' }, - { fromFile: 'svc\\a\\one.py', why: 'Windows separators' }, - { fromFile: 'trailing/', why: 'a path ending in a separator' }, - ])('memoizes the chain the pre-#2913 code built — $why', ({ fromFile }) => { - const files = new Set(WORKSPACE); - resolveImportTarget('nested.ghost', fromFile, files, undefined, undefined); + it.each(IMPORTER_PATH_SHAPES)( + 'memoizes the chain the pre-#2913 code built — $why', + ({ fromFile }) => { + const chain = expectMemoizedChainMatchesLegacy(ancestorArm, new Set(WORKSPACE), fromFile); - const norm = fromFile.replace(/\\/g, '/'); - const lastSlash = norm.lastIndexOf('/'); - const importerDir = lastSlash === -1 ? '' : norm.slice(0, lastSlash); - const chain = getPythonFileIndex(files).ancestorsByDir.get(importerDir); - - // Both consumers' chains, from the one memo: `resolveAbsoluteFromFiles` - // walked these directories, `hasRepoCandidate` walked the same directories - // with `//` appended. - expect(chain).toEqual(legacyAncestorChain(fromFile)); - expect((chain ?? []).map((ancestor) => `${ancestor}/nested/`)).toEqual( - legacyAncestorPrefixes(fromFile, 'nested'), - ); - }); + // Both consumers' chains, from the one memo: `resolveAbsoluteFromFiles` + // walked these directories, `hasRepoCandidate` walked the same directories + // with `//` appended. + expect(chain.map((ancestor) => `${ancestor}/nested/`)).toEqual( + legacyAncestorPrefixes(fromFile, 'nested'), + ); + }, + ); it.each([ { why: 'relative paths sharing directories', files: WORKSPACE }, @@ -251,8 +226,8 @@ describe('Python importer-ancestor memo (#2913)', () => { // The build now walks separators from the deepest outward and stops at // the first prefix already present. Skipping the rest is only sound // because a prefix is always stored with all of its own ancestors. - expect(sorted(index.dirPrefixes)).toEqual(sorted(legacy)); - expect(sorted(index.nestedDirNames)).toEqual(sorted(specNestedDirNames(legacy))); + expect(sortedStrings(index.dirPrefixes)).toEqual(sortedStrings(legacy)); + expect(sortedStrings(index.nestedDirNames)).toEqual(sortedStrings(specNestedDirNames(legacy))); }); /** @@ -275,7 +250,7 @@ describe('Python importer-ancestor memo (#2913)', () => { const paths = ['pkg/__init__.py', ...modules, 'app/main.py']; const workspace = countedParsedFiles(paths); const files = new Set(paths); - const resolved: (string | readonly string[] | null)[] = []; + const resolved: ChainMemoResult[] = []; for (let i = 0; i < imports; i++) { const targetRaw = `pkg.m${i % modules.length}`; @@ -307,17 +282,10 @@ describe('Python importer-ancestor memo (#2913)', () => { const a = new Set(WORKSPACE); const b = new Set(WORKSPACE); - driveAdapter(a, 2); - driveAdapter(b, 2); + expectDistinctFileSetsGetOwnChainMemo(ancestorArm, a, b, 2); - const indexA = getPythonFileIndex(a); - const indexB = getPythonFileIndex(b); - - expect(indexA).not.toBe(indexB); - expect(indexA.ancestorsByDir).not.toBe(indexB.ancestorsByDir); - expect(indexA.ancestorsByDir.get('svc/a')).not.toBe(indexB.ancestorsByDir.get('svc/a')); - expect(indexA.ancestorsByDir.get('svc/a')).toEqual(indexB.ancestorsByDir.get('svc/a')); - expect(indexA.ancestorsByDir.size).toBe(IMPORTER_DIRS.length); - expect(indexB.ancestorsByDir.size).toBe(IMPORTER_DIRS.length); + // The whole per-file-set index, not only the memo inside it: the WeakMap is + // keyed on the Set, so two Sets can never share one index. + expect(getPythonFileIndex(a)).not.toBe(getPythonFileIndex(b)); }); });