mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-08-28 05:25:25 +00:00
perf(import-resolvers): build buildSuffixIndex's dirMap lazily (#2903)
`buildSuffixIndex` eagerly built three maps. `dirMap` is the array-valued one —
one entry per directory suffix per file, so O(files x depth) in entries and
array churn — and only four call sites ever read it, all via `getFilesInDir`:
`import-resolvers/{php,csharp,jvm}.ts` and `import-resolvers/configs/python.ts`.
Ruby (through workspace-file-index), the TypeScript scope resolver, Vue's
import-target and the include-extractor never ask a directory question, and
built it anyway. Since #2880 these indexes are retained for a whole resolution
pass rather than rebuilt per import, so that waste is now resident memory.
Deferring it to the first `getFilesInDir` call is behaviour-identical — same
key, same descending-suffix order, same per-bucket push order, same
`substring(lastIndexOf('.'))` extension clamp. The builder assigns the MAP on
completion, so a repeated miss cannot rebuild it.
Measured on `buildSuffixIndex` alone, 32k paths, index built and
`getFilesInDir` never called:
C# layout, 13 segments 79,018,680 -> 66,580,488 B -15.74%
Ruby layout, 11 segments 60,752,792 -> 48,656,856 B -19.91%
and on the whole retained WorkspaceFileIndex the bench measures:
csharp 32k 73.62 -> 61.76 MiB ruby 32k 55.26 -> 43.69 MiB
When `getFilesInDir` IS called the footprint is unchanged, so the deferral is
never a loss. No new retention: all five construction sites already hold both
input arrays alive beside the index.
The laziness is pinned structurally rather than by timing. The test's corpus is
a `string[]` whose elements are accessor properties, so an indexed read is
observable and the read count IS the pass count: 14 after construction, still
14 after any number of get/getInsensitive, 28 after the first `getFilesInDir`,
28 after five more. Memoizing the decision instead of the map would read 42.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Co58j4au9JLf8dmJpwdF9B
This commit is contained in:
parent
81100e2c74
commit
b6ee577e03
2 changed files with 54 additions and 10 deletions
|
|
@ -83,7 +83,13 @@ export interface SuffixIndex {
|
|||
get(suffix: string): string | undefined;
|
||||
/** Case-insensitive suffix lookup */
|
||||
getInsensitive(suffix: string): string | undefined;
|
||||
/** Get all files in a directory suffix */
|
||||
/**
|
||||
* Get all files in a directory suffix.
|
||||
*
|
||||
* The directory map behind this is built on the FIRST call and memoized —
|
||||
* see `buildSuffixIndex`. Callers that never ask a directory question never
|
||||
* pay for it.
|
||||
*/
|
||||
getFilesInDir(dirSuffix: string, extension: string): string[];
|
||||
}
|
||||
|
||||
|
|
@ -92,8 +98,6 @@ export function buildSuffixIndex(normalizedFileList: string[], allFileList: stri
|
|||
const exactMap = new Map<string, string>();
|
||||
// Map: lowercase suffix -> original file path
|
||||
const lowerMap = new Map<string, string>();
|
||||
// Map: directory suffix -> list of file paths in that directory
|
||||
const dirMap = new Map<string, string[]>();
|
||||
|
||||
for (let i = 0; i < normalizedFileList.length; i++) {
|
||||
const normalized = normalizedFileList[i];
|
||||
|
|
@ -112,11 +116,49 @@ export function buildSuffixIndex(normalizedFileList: string[], allFileList: stri
|
|||
lowerMap.set(lower, original);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Map: `${directory suffix}:${extension}` -> file paths in that directory.
|
||||
*
|
||||
* DEFERRED, not dropped (#2903). This is the array-valued map of the three
|
||||
* and by far the most expensive: one entry — and one array push — per file
|
||||
* per directory component, so O(files × depth) in entries AND in array
|
||||
* churn. Measured on the 32k-path arms of `bench/import-target/`, it is
|
||||
* ~15% of the retained C# index and ~19% of the retained Ruby one.
|
||||
*
|
||||
* Only `getFilesInDir` reads it, and only four call sites reach that:
|
||||
* `import-resolvers/{php,csharp,jvm}.ts` and `import-resolvers/configs/
|
||||
* python.ts`. Every other consumer of this index — `workspace-file-index.ts`
|
||||
* serving Ruby, `languages/typescript/scope-resolver.ts`,
|
||||
* `languages/vue/import-target.ts`, `group/extractors/include-extractor.ts`
|
||||
* — asks only suffix questions and was paying the whole footprint for a map
|
||||
* it never touched. Since these indexes are now retained for a whole
|
||||
* resolution pass rather than rebuilt per import (#2877-#2880), that is
|
||||
* retained memory against the #2649 kernel-scale OOM constraint.
|
||||
*
|
||||
* `null` until the first `getFilesInDir`; the MAP is memoized, not the
|
||||
* decision to build it, so a repeated miss cannot rebuild it. Building it
|
||||
* later is behaviour-identical because it is a pure function of
|
||||
* `normalizedFileList` / `allFileList`, and it retains nothing new: every
|
||||
* production caller already holds both arrays alive alongside the index
|
||||
* (`WorkspaceFileIndex.normalized`/`.all`, the TS and Vue `PassCache`s,
|
||||
* `IncludeExtractor.extract`'s locals).
|
||||
*/
|
||||
let dirMap: Map<string, string[]> | null = null;
|
||||
|
||||
const getDirMap = (): Map<string, string[]> => {
|
||||
if (dirMap !== null) return dirMap;
|
||||
const built = new Map<string, string[]>();
|
||||
for (let i = 0; i < normalizedFileList.length; i++) {
|
||||
const normalized = normalizedFileList[i];
|
||||
const original = allFileList[i];
|
||||
const lastSlash = normalized.lastIndexOf('/');
|
||||
// A file at the repo root is in no directory suffix.
|
||||
if (lastSlash < 0) continue;
|
||||
|
||||
// Index directory membership
|
||||
const lastSlash = normalized.lastIndexOf('/');
|
||||
if (lastSlash >= 0) {
|
||||
// Build all directory suffixes
|
||||
const parts = normalized.split('/');
|
||||
const dirParts = parts.slice(0, -1);
|
||||
const fileName = parts[parts.length - 1];
|
||||
const ext = fileName.substring(fileName.lastIndexOf('.'));
|
||||
|
|
@ -124,21 +166,23 @@ export function buildSuffixIndex(normalizedFileList: string[], allFileList: stri
|
|||
for (let j = dirParts.length - 1; j >= 0; j--) {
|
||||
const dirSuffix = dirParts.slice(j).join('/');
|
||||
const key = `${dirSuffix}:${ext}`;
|
||||
let list = dirMap.get(key);
|
||||
let list = built.get(key);
|
||||
if (!list) {
|
||||
list = [];
|
||||
dirMap.set(key, list);
|
||||
built.set(key, list);
|
||||
}
|
||||
list.push(original);
|
||||
}
|
||||
}
|
||||
}
|
||||
dirMap = built;
|
||||
return built;
|
||||
};
|
||||
|
||||
return {
|
||||
get: (suffix: string) => exactMap.get(suffix),
|
||||
getInsensitive: (suffix: string) => lowerMap.get(suffix.toLowerCase()),
|
||||
getFilesInDir: (dirSuffix: string, extension: string) => {
|
||||
return dirMap.get(`${dirSuffix}:${extension}`) || [];
|
||||
return getDirMap().get(`${dirSuffix}:${extension}`) || [];
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Binary file not shown.
Loading…
Add table
Reference in a new issue