From b6ee577e03d72533aaa55a399a8bd4772f04e51d Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Sun, 9 Aug 2026 10:38:22 +0000 Subject: [PATCH] perf(import-resolvers): build buildSuffixIndex's dirMap lazily (#2903) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `buildSuffixIndex` eagerly built three maps. `dirMap` is the array-valued one — one entry per directory suffix per file, so O(files x depth) in entries and array churn — and only four call sites ever read it, all via `getFilesInDir`: `import-resolvers/{php,csharp,jvm}.ts` and `import-resolvers/configs/python.ts`. Ruby (through workspace-file-index), the TypeScript scope resolver, Vue's import-target and the include-extractor never ask a directory question, and built it anyway. Since #2880 these indexes are retained for a whole resolution pass rather than rebuilt per import, so that waste is now resident memory. Deferring it to the first `getFilesInDir` call is behaviour-identical — same key, same descending-suffix order, same per-bucket push order, same `substring(lastIndexOf('.'))` extension clamp. The builder assigns the MAP on completion, so a repeated miss cannot rebuild it. Measured on `buildSuffixIndex` alone, 32k paths, index built and `getFilesInDir` never called: C# layout, 13 segments 79,018,680 -> 66,580,488 B -15.74% Ruby layout, 11 segments 60,752,792 -> 48,656,856 B -19.91% and on the whole retained WorkspaceFileIndex the bench measures: csharp 32k 73.62 -> 61.76 MiB ruby 32k 55.26 -> 43.69 MiB When `getFilesInDir` IS called the footprint is unchanged, so the deferral is never a loss. No new retention: all five construction sites already hold both input arrays alive beside the index. The laziness is pinned structurally rather than by timing. The test's corpus is a `string[]` whose elements are accessor properties, so an indexed read is observable and the read count IS the pass count: 14 after construction, still 14 after any number of get/getInsensitive, 28 after the first `getFilesInDir`, 28 after five more. Memoizing the decision instead of the map would read 42. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Co58j4au9JLf8dmJpwdF9B --- .../core/ingestion/import-resolvers/utils.ts | 64 +++++++++++++++--- .../suffix-index-lazy-dir-map.test.ts | Bin 0 -> 17754 bytes 2 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 gitnexus/test/unit/import-resolvers/suffix-index-lazy-dir-map.test.ts diff --git a/gitnexus/src/core/ingestion/import-resolvers/utils.ts b/gitnexus/src/core/ingestion/import-resolvers/utils.ts index 6a033c1ee..ebd4650fe 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/utils.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/utils.ts @@ -83,7 +83,13 @@ export interface SuffixIndex { get(suffix: string): string | undefined; /** Case-insensitive suffix lookup */ getInsensitive(suffix: string): string | undefined; - /** Get all files in a directory suffix */ + /** + * Get all files in a directory suffix. + * + * The directory map behind this is built on the FIRST call and memoized — + * see `buildSuffixIndex`. Callers that never ask a directory question never + * pay for it. + */ getFilesInDir(dirSuffix: string, extension: string): string[]; } @@ -92,8 +98,6 @@ export function buildSuffixIndex(normalizedFileList: string[], allFileList: stri const exactMap = new Map(); // Map: lowercase suffix -> original file path const lowerMap = new Map(); - // Map: directory suffix -> list of file paths in that directory - const dirMap = new Map(); for (let i = 0; i < normalizedFileList.length; i++) { const normalized = normalizedFileList[i]; @@ -112,11 +116,49 @@ export function buildSuffixIndex(normalizedFileList: string[], allFileList: stri lowerMap.set(lower, original); } } + } + + /** + * Map: `${directory suffix}:${extension}` -> file paths in that directory. + * + * DEFERRED, not dropped (#2903). This is the array-valued map of the three + * and by far the most expensive: one entry — and one array push — per file + * per directory component, so O(files × depth) in entries AND in array + * churn. Measured on the 32k-path arms of `bench/import-target/`, it is + * ~15% of the retained C# index and ~19% of the retained Ruby one. + * + * Only `getFilesInDir` reads it, and only four call sites reach that: + * `import-resolvers/{php,csharp,jvm}.ts` and `import-resolvers/configs/ + * python.ts`. Every other consumer of this index — `workspace-file-index.ts` + * serving Ruby, `languages/typescript/scope-resolver.ts`, + * `languages/vue/import-target.ts`, `group/extractors/include-extractor.ts` + * — asks only suffix questions and was paying the whole footprint for a map + * it never touched. Since these indexes are now retained for a whole + * resolution pass rather than rebuilt per import (#2877-#2880), that is + * retained memory against the #2649 kernel-scale OOM constraint. + * + * `null` until the first `getFilesInDir`; the MAP is memoized, not the + * decision to build it, so a repeated miss cannot rebuild it. Building it + * later is behaviour-identical because it is a pure function of + * `normalizedFileList` / `allFileList`, and it retains nothing new: every + * production caller already holds both arrays alive alongside the index + * (`WorkspaceFileIndex.normalized`/`.all`, the TS and Vue `PassCache`s, + * `IncludeExtractor.extract`'s locals). + */ + let dirMap: Map | null = null; + + const getDirMap = (): Map => { + if (dirMap !== null) return dirMap; + const built = new Map(); + for (let i = 0; i < normalizedFileList.length; i++) { + const normalized = normalizedFileList[i]; + const original = allFileList[i]; + const lastSlash = normalized.lastIndexOf('/'); + // A file at the repo root is in no directory suffix. + if (lastSlash < 0) continue; - // Index directory membership - const lastSlash = normalized.lastIndexOf('/'); - if (lastSlash >= 0) { // Build all directory suffixes + const parts = normalized.split('/'); const dirParts = parts.slice(0, -1); const fileName = parts[parts.length - 1]; const ext = fileName.substring(fileName.lastIndexOf('.')); @@ -124,21 +166,23 @@ export function buildSuffixIndex(normalizedFileList: string[], allFileList: stri for (let j = dirParts.length - 1; j >= 0; j--) { const dirSuffix = dirParts.slice(j).join('/'); const key = `${dirSuffix}:${ext}`; - let list = dirMap.get(key); + let list = built.get(key); if (!list) { list = []; - dirMap.set(key, list); + built.set(key, list); } list.push(original); } } - } + dirMap = built; + return built; + }; return { get: (suffix: string) => exactMap.get(suffix), getInsensitive: (suffix: string) => lowerMap.get(suffix.toLowerCase()), getFilesInDir: (dirSuffix: string, extension: string) => { - return dirMap.get(`${dirSuffix}:${extension}`) || []; + return getDirMap().get(`${dirSuffix}:${extension}`) || []; }, }; } diff --git a/gitnexus/test/unit/import-resolvers/suffix-index-lazy-dir-map.test.ts b/gitnexus/test/unit/import-resolvers/suffix-index-lazy-dir-map.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..16ead360bef0252a042847eef3c6b85e1ae69b76 GIT binary patch literal 17754 zcmdU1>v9{%mEPZaijJKWK$!q6*|`|9l%g$a!&*vYk#_CY(yCwpG>8!gGw}2bDMrOq z9wJryK>H?nlKsAOy5|P)64}{GSdK)^bf4}%m+#zqGJ5>@k$G$;(=txxr{!W1zduOl z_Wh*i%skHRtjO|fvx?Rxc7}rrlci=6=dLiNv!?&))2}}NVrK{<+vym)iVJ6oWo|9$ zxS{#(Dl<{)u550ctZ05Ltt;Xzb1^3vc|+ZYs>7e3 z{Q6fWTTm^yv)mR@oPy4`pF(nAhb;N$Ctrm@^$7BYIzo+`LvrcNnopnp?zg`k;LqQE zPHG?u%@;Zpdv5F5i!3*hxmspWC=qK(wKnU>Ig?|L(12nYrFbWY7y03tvQzZ6+Um-# zXnN5(b`E2sf`9t-H(!2bE^MCKWZ~pRM#RI2yfUVjb7EARWV}snIc``JQAHTs9(#)n*q-cueSJ|bV4<7?S zEVL+xMbcW$)F#;#V;4fs7&*^6w}Tnjac)hLW$UmL*kY8&1ife(@afu!8OWnqmc!m*kB~;5%}a=t@IbA* zu8}Sf^SCt2GjL;Wu41T{4uu-q)Iq4bW=UY9maJlo!mmkup;XrimWl!~oNzw4!ig|b z&IA=G%o<{qolVQxh4qxM`8YR6ColI;OjZI{&XaH22xZns4XQ6RZ8p% zm;|OnFCYqv>?+UjXOY2Lp@UQth`^X{WNDM-b1I>O6r%ZHYQg>Vd~7B_BQaL!Bw!8L z3+~P1c?2{&`~2-h$vmZM9EjBg@u(oxypP^|yH5rA4q{27KfwyIR~2lT;?!!6ghdy2 zrFN^QN!KOtfxr&OvFW8?NSJ7BG=qHf+{V5K@+8+4HCixHUbs}Lh+uvayNbO+3hIVk znS)c2jLeWHWAh}?ixvflsm+x$G8rzSh=FHMjJ}J~Ym=6%DTd)FUX`HW7vD! zinS4q4tw(Ii&I3`elU6Zg?I>R$(Ot4*YMgz2r+4Z4Gv!BQKIbHHB`dLz{nN5=cC^3ZCDiysE>ZOvU10@Rhger&+!N^!~}tUm@H%gffj`kc6nNJ9j(%Z2JThw9glEf zxg>JMDgIctNAvLkcIej=Uf{IhB!L{5cI?-dJq%qE&ustmT{C!sg-Ij~ zCH}sNRu&7(&@-K&Y%&A$#2-71hu_7}Fl*?_sY*>Ea}pZ1Ha+a^gp)Qz^bRD#9=$}k z)HCzW@HqH;FsRQImc|T4lICnZ*}I+LyDUx{OsY@F_~W^m{PIIJ|7QHl4>-roB(TC5 zDh|Dnl7_|&;iCJPTCq#{{tFIjZVgj)+F8XYE{(NF#3>P4ET+}V-FO-3>>)Fteo(1u zm=5BfH&k%-{+{x)M|jqPLLXTDqrvU}hkga2C(#x3vRJ}lBh}L&MZ%<^*_U8LQcAc( zz#Sa8_&mfS4o-{^2W9pFEJ5!oE0ei6$r>Ipcvi>P12a*7v>HJ}BF1pLGWKR7E?Hoc z)-sKpP5Da+^F*+Vhz}x{G@UspBTMgR1)1TMv6Yap8<|Aw_2_j5f9^(y_B@(hkG@A1Fhq!`JB&1YV0N7#UP%y|>>neF zm2`*O*D%*_&hH+6cNf`GjOb0)mpEd*MgU5#Pjo%zcskkiNob|T2-t`wiqLv+J#gXL zoPPi6)xm#hPy+XiiUu`XPw_gsz=Oou4)FLj8pa?GW^pHJv!aT|49LhM1i&)=bA%`iW*!V#Ku`osF-{0D^7>G^QxPZGhfXBQ5#Mnwv+Q^`#;CDIfw=F!#Q(hdA>>SzHm z75w|yABYK-2$6eQNEO7qIXZd0cX;sM`!8hy(KV+3B9(*0Hrrw@j%B4FQ<_{$j^Ge_<9w zU|xdKV1M+(4M-ssB3uVh@7no(sk0y5ItGf z2-UZ!5`yyT#2=tFdPYs3t!uo5%1O*4k++VL`X5q~+9E=%6y9q@%yF$)m@$(M0p=dx z6T4QS{@)izHV^W!5R#GhFT%-(zJlud`ebLvOU614;QV7i41P3htD?3*MB}zf$Td{{ zMCtB_a@a=+x7x-}6mLiiz+IKFq#g2W%Y8!G&1WX0A}SZ`0n-A72PPKevA(cx(Dh7|A57pEnS;5U_aJtAQ`E6yhL!)4XM0J~$3gV{*qnGRyidUx2xxhAZ2Fa` zgA)^xYa1av689YZ-|izg!xIZ7oZnfb(k?3W{noM1&4(MJPQzDrHIJn{J>CuiWVU79 zS*vRCZ@y~r7C5PAs!t>*5#9BS^b|Jh1hn6Q6w{d!h__erciK{Jkcv-RhtgLgBe*6 zR6wH|iNt&>gTow&FAy+qTR{i3+8~6sR=Fk(ujUBSXa*P=O4IsmPco-9te?%w%z2*(S}LRx>m*ljA&le?5Vh zu}oaye58x7Ewpo+mS_l=iP)TGeiL84(<+AULkX|Y;mYR78kfyi|rhgA1^<5@b*Y819DBu*j#9Vv!NzS41S=U{WkF zE1W2e7fRcoP;5&U8^VTJd+PUvIS9Z-oa%~>rteU>86xKftH4c)>AKGqFmBiTD^@JZ zmCf0@9UF9UEMba2*bpuhpry0%25ulIj;_SvdYI3iqjc&CR56Y0Ej+zZ`Zwy_nlSM} zDwDfPDJ-C5fmf_C!qAQ&b&Z7B;M{E8`F*JPL;DYC#@4lO8bZu7D^{R9KYFwO$K$=z zQ}DgYr#z>-Jsuw~{ga^5Pu#|%N1=n(?{$HSCFfU&9-LgM5mVW0$cRD3xY9C+=L}3x z7lDsfM4@7^yWp{347eM?BFK@pEE`NTaOvQ5iz#R>w}=Ky(faFi6fN4fl=|Hac-K^E zuV`B!fHEKpnn3jDt=`c?=c|9jWcfwI*6v&^D8}H!op4pb&tV*)RKPU^1nM}Mlgr`S zGN=c%PN6)zmtb{R=_RNuuUZc~=J&rx{v0;TXj99SuoMvtZ=2*mi4=B^a+h7LaT5dB zKg*H?*SC-h$T9rY3N3T3;OOM+Ag6&pFKxuEhU-<3%V<_+rG$h4A?e_HZUmSc7TGtp z|KvAc`V;#>9@hj_TwfTlGeqzJ2aTErl~(H-{LPaxFVL9kko~GnHK_LvR^9)xj1o_o zxJ9t?`Ycy8#JT!HA?t^Xu(ecJU(jI1Q2a^rwhoJHGsLw&r$!5tJ!@!n`~lBTx3mUa zJY|$#)so?~f%&k!Ih-t72U;1xYZtAiz<%E9V93s&o-gB#HLyklpUuFIj)p-D)pguX za3{S+qv38Eje*}R1#{v~# zbrOO^YDws6RGij)XDpe_EqvA?vL#kIYc4D0JLl?U-s+Jlq+2U}}*36jJ*?=8NO_%{UI5dhqYCF%M zmM8+z#3El83ONjGHT(fJ1RK&H#%)L~Z}FBDIoPfifwJDd_RYU`U8oZH_%^QbAPtUT zV|U{$2FTf^OH^}uC6%B%F-?ca19k6CN`B#S)w!#Qv>j55 zh`y@s9rWr+y7K6w=GIQ)thc4dXPOV;OX@`f_9`ax!2XFg__vHyl&;6s9mbk!4UA_5 z%5JN$9@s|Fdj<#e0zVtgN%)h{sI2MMQ1^hrO^j;|ujvsma1k#=mQ7q}JYKcTzVRX` zmB2(zsV*siFmq2WKdQ5(R|us{)U)KyGgBA_U>*atj>-Gr7%HIzAfe7?L*96W{yTij zM`fNmS8}jk)eiT0Nd2x+WAFFAKixkW9{=H(xAchK^{`h=a3nwPYh9c{w-memP|Z4# zqieit;Lt>fUrC^n%dZ%GK~SFK3O=T6RR|4FKT03A=1{8MwJ-BEgieYl zwUb7)(ht6=hT+PC!sEY^&$C!G#Fq=Wx6k4~QhCkukcN9B2tK`f-ddqn*}60HFZeBo>XfX&8L;PZV3WB=kLDXLQ`!p@&3UrPHy*;-Ygmi+~y7fYxQir zRmb`-zn#NRjgev zQn{B#3z7^$PJR)=FTJR)a(|4UZHeKv9ZrjK@G=dZe-3hW%W0#&*#i3rcLX})o7GRz z0fqHjQLA2gsJ#Nvl;7G5@spQtSbN2KA8@{KyM5gF9XE@>qdxn@NrK!3g!~hOioR^7 zLcsopLu|Zin5BBb>fQZNbJv6xUD;0uW#tPSIc6{`d2%XQ)1iL9GybUa2Yal_jUmUV zm+TS_jSA1s@K1B|@V$jOw~(me0nDJ1Yr8Vk=~>sp7l^#e%;t;2uS80w)i2P3PRikO zntw?R*Gs1AH-8=#Zw73ePM+|rrYSu885Rb*{{f}x-Ef~r{U)wbqg*W?CXNli$=dvM z40*pRV7mT`9x)6MvBKi_P}l$_@zPE{;uH~RDVUvCS<>Cd*9$NyG-y~L##K_e%*otgDO-UA2XO*bshIw_F9DJ7O6Ww2KJQF-x^=hMj#KA`9OmJ@b&;g49+Z=O*f zZl=rItV{UO)z2*)E`!FLr)z+rF=1Hd^P8izl5gGRI1q{Cpzm+|*=9WZitH>S*2;m4_8HMRxG^%1GFC~BS aRx$Cx6-lMB$$_xAfSErveG}1(|NB3<%$bP* literal 0 HcmV?d00001