From f45d86bb199d877b20a33a4dd21caf84ad444ff6 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Fri, 5 Jun 2026 11:33:38 +0000 Subject: [PATCH] refactor(ingestion): worker-pool-only parsing; remove sequential parser (#1983) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the #1983 huge-repo parse-OOM effort by making the worker pool GitNexus's sole parse path. Parallel serialization (the perf core): workers serialize their ParsedFiles to a disk store in parallel and stream them back to scope-resolution, so the main thread no longer re-parses every file (the tree-sitter native-memory leak that caused the OOM). Adds chunk merge-pipelining + work-proportional chunk sizing so the pool stays saturated. Remove the sequential parser: `--workers 0`, `GITNEXUS_WORKER_POOL_SIZE=0`, and `skipWorkers` now hard-error (no silent degrade — #1741); the small-repo threshold no longer selects an in-process path; pool creation stays lazy / cache-miss-gated so warm all-hit runs never spawn workers. Worker-path parity fixes — removing sequential surfaced two pre-existing gaps that tiny-fixture tests had masked by running below the worker threshold, both fixed by carrying per-file metadata as DATA across the worker boundary (never re-parsing on the main thread, preserving the OOM fix): - C++: templateConstraints wired into worker node identity (SFINAE overload disambiguation) + ADL / inline-namespace capture side-channel serialized onto the ParsedFile. - Kotlin: companion-scope side-channel serialized the same way (companion / static dispatch). Validation: tsc + build clean; full suite green (10,190 pass — the only deterministic failures were the now-fixed C++/Kotlin worker-path gaps; the 2 remaining full-run failures are pre-existing load flakiness, green in isolation); cpp-pipeline benchmark stays linear on a 1-worker pool. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/scope-resolution/parsed-file.ts | 24 + gitnexus/README.md | 2 +- gitnexus/src/cli/analyze.ts | 12 +- gitnexus/src/cli/i18n/en.ts | 2 +- gitnexus/src/cli/i18n/zh-CN.ts | 2 +- gitnexus/src/cli/index.ts | 2 +- .../src/core/ingestion/language-provider.ts | 21 + .../src/core/ingestion/languages/c-cpp.ts | 6 + .../src/core/ingestion/languages/cpp/adl.ts | 52 + .../languages/cpp/capture-side-channel.ts | 98 ++ .../languages/cpp/file-local-linkage.ts | 41 + .../src/core/ingestion/languages/cpp/index.ts | 4 + .../languages/cpp/inline-namespaces.ts | 24 + .../ingestion/languages/cpp/scope-resolver.ts | 19 + .../languages/cpp/two-phase-lookup.ts | 50 + .../ingestion/languages/csharp/captures.ts | 9 +- .../languages/csharp/namespace-siblings.ts | 5 +- .../src/core/ingestion/languages/kotlin.ts | 8 + .../languages/kotlin/capture-side-channel.ts | 75 ++ .../languages/kotlin/companion-scopes.ts | 11 + .../core/ingestion/languages/kotlin/index.ts | 5 + .../languages/kotlin/scope-resolver.ts | 18 + .../languages/typescript/captures.ts | 9 +- .../src/core/ingestion/parsing-processor.ts | 1058 ++--------------- .../ingestion/pipeline-phases/parse-impl.ts | 755 ++++++------ .../core/ingestion/pipeline-phases/parse.ts | 39 +- gitnexus/src/core/ingestion/pipeline.ts | 31 +- .../contract/scope-resolver.ts | 39 + .../scope-resolution/pipeline/phase.ts | 101 +- .../scope-resolution/pipeline/run.ts | 69 +- .../src/core/ingestion/utils/heap-probe.ts | 26 +- .../core/ingestion/workers/parse-worker.ts | 124 +- .../src/core/ingestion/workers/worker-pool.ts | 54 +- gitnexus/src/core/run-analyze.ts | 4 +- gitnexus/src/storage/parse-cache.ts | 4 +- gitnexus/src/storage/parsedfile-store.ts | 216 ++++ gitnexus/test/helpers/worker-parse.ts | 79 ++ .../c-cpp-typedef-legacy-parse.test.ts | 15 +- .../integration/cpp-adl-benchmark.test.ts | 13 +- .../cpp-pipeline-benchmark.test.ts | 10 +- .../fastapi-prefix-pipeline.test.ts | 4 +- .../integration/ignore-and-skip-e2e.test.ts | 16 +- .../object-literal-method-exports.test.ts | 47 + .../object-literal-owner-resolution.test.ts | 179 ++- .../parse-impl-chunk-concurrency.test.ts | 6 +- .../parse-impl-env-reads.test.ts | 6 +- .../parse-impl-large-fixture.test.ts | 20 +- .../parse-impl-progress-monotonic.test.ts | 17 +- .../parse-impl-quarantine-cache-skip.test.ts | 3 - .../php-pipeline-benchmark.test.ts | 2 +- .../qualified-class-lookups.test.ts | 73 +- .../test/integration/resolvers/cpp.test.ts | 8 +- .../test/integration/resolvers/csharp.test.ts | 4 +- .../test/integration/resolvers/go.test.ts | 2 +- .../test/integration/resolvers/java.test.ts | 10 +- .../laravel-route-resolution.test.ts | 8 +- .../test/integration/resolvers/python.test.ts | 4 +- .../test/integration/resolvers/ruby.test.ts | 11 +- .../test/integration/resolvers/rust.test.ts | 4 +- .../integration/resolvers/typescript.test.ts | 4 +- .../ruby-pipeline-benchmark.test.ts | 2 +- .../worker-sequential-parity.test.ts | 79 -- .../unit/analyze-worker-pool-size.test.ts | 17 +- .../unit/language-availability-skip.test.ts | 89 ++ .../unit/parse-impl-worker-lazy-cache.test.ts | 148 +-- .../parse-impl-worker-startup-gating.test.ts | 24 +- gitnexus/test/unit/parsedfile-store.test.ts | 236 ++++ .../test/unit/parsing-worker-fallback.test.ts | 66 +- .../sequential-language-availability.test.ts | 86 -- 69 files changed, 2273 insertions(+), 2038 deletions(-) create mode 100644 gitnexus/src/core/ingestion/languages/cpp/capture-side-channel.ts create mode 100644 gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts create mode 100644 gitnexus/src/storage/parsedfile-store.ts create mode 100644 gitnexus/test/helpers/worker-parse.ts create mode 100644 gitnexus/test/integration/object-literal-method-exports.test.ts rename gitnexus/test/{unit => integration}/parse-impl-chunk-concurrency.test.ts (97%) rename gitnexus/test/{unit => integration}/parse-impl-env-reads.test.ts (95%) rename gitnexus/test/{unit => integration}/parse-impl-progress-monotonic.test.ts (88%) delete mode 100644 gitnexus/test/integration/worker-sequential-parity.test.ts create mode 100644 gitnexus/test/unit/language-availability-skip.test.ts create mode 100644 gitnexus/test/unit/parsedfile-store.test.ts delete mode 100644 gitnexus/test/unit/sequential-language-availability.test.ts diff --git a/gitnexus-shared/src/scope-resolution/parsed-file.ts b/gitnexus-shared/src/scope-resolution/parsed-file.ts index 50eb5a795..98a327ad4 100644 --- a/gitnexus-shared/src/scope-resolution/parsed-file.ts +++ b/gitnexus-shared/src/scope-resolution/parsed-file.ts @@ -74,4 +74,28 @@ export interface ParsedFile { */ readonly localDefs: readonly SymbolDefinition[]; readonly referenceSites: readonly ReferenceSite[]; + /** + * Opaque, language-private serialization of capture-time side-channel + * state that a provider's `emitScopeCaptures` populates into module-level + * maps as a SIDE EFFECT (not onto the scopes/defs of this `ParsedFile`). + * + * Such state is computed inside the parse worker (where `emitScopeCaptures` + * runs) and would otherwise be lost across the worker→main MessageChannel + * and the disk store, because scope-resolution reuses the serialized + * `ParsedFile` and SKIPS re-extraction on the main thread (#1983 — the + * whole point is to avoid a main-thread tree-sitter re-parse). Carrying the + * data here lets the main thread repopulate those maps WITHOUT re-parsing. + * + * Shared / ingestion code treats this as opaque (`unknown`) per AGENTS.md + * (no language names in shared code). The producing language fills it via + * the `LanguageProvider.collectCaptureSideChannel` hook (worker side) and + * consumes it via the `ScopeResolver.applyCaptureSideChannel` hook + * (main-thread resolution side). It MUST be plain JSON-serializable data + * (objects / arrays / primitives) so it round-trips through the disk-backed + * `parsedfile-store` (JSON.stringify + interning reviver). + * + * Optional: providers whose `emitScopeCaptures` is pure (no module-level + * side effects — the contract default) leave this undefined. + */ + readonly captureSideChannel?: unknown; } diff --git a/gitnexus/README.md b/gitnexus/README.md index 7544508ba..54c0f7c81 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -400,7 +400,7 @@ Values above **32768 KB (32 MB)** are clamped to the tree-sitter parser ceiling; ### Analyze reports a worker timeout -Worker parse timeouts are recoverable. GitNexus retries stalled worker jobs with backoff, splits large jobs to isolate slow files, and falls back to the sequential parser when needed. If a large repository needs more time per worker job, use either: +Worker parse timeouts are recoverable. GitNexus retries stalled worker jobs with backoff, splits large jobs to isolate slow files, and quarantines a file that repeatedly crashes its worker (respawning the slot so the pool keeps going). If a large repository needs more time per worker job, use either: ```bash # CLI flag, in seconds diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 3481fd2ed..bea500320 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -474,8 +474,7 @@ async function ensureHeap(): Promise { cliError( ` Analysis aborted in a native worker or native binding path.\n` + ` Try one of these recovery paths:\n` + - ` gitnexus analyze --workers 0\n` + - ` npm uninstall -g gitnexus && npm install -g gitnexus@latest\n` + + ` npm uninstall -g gitnexus && npm install -g gitnexus@latest (rebuilds native bindings)\n` + ` Use Node 22 LTS if you are on a newer non-LTS runtime.\n`, { recoveryHint: 'native-worker-abort' }, ); @@ -599,7 +598,7 @@ export interface AnalyzeOptions { workerTimeout?: string; /** Control LadybugDB WAL auto-checkpoint threshold during analyze. */ walCheckpointThreshold?: string; - /** Parse worker pool size; 0 disables workers (sequential fallback). */ + /** Parse worker pool size (>=1); 0 is rejected (no sequential mode). */ workers?: string; embeddingThreads?: string; embeddingBatchSize?: string; @@ -794,10 +793,11 @@ const analyzeCommandImpl = async ( let workerPoolSize: number | undefined; if (options.workers !== undefined) { const parsedWorkers = Number(options.workers); - if (!Number.isInteger(parsedWorkers) || parsedWorkers < 0) { + if (!Number.isInteger(parsedWorkers) || parsedWorkers < 1) { cliError( - ' --workers must be a non-negative integer. ' + - 'Pass 0 to disable the worker pool (sequential fallback).\n', + ' --workers must be a positive integer (>= 1). ' + + 'GitNexus parses through a worker pool only — there is no sequential ' + + 'mode, so 0 is not allowed. Omit --workers for an auto-sized pool.\n', ); process.exitCode = 1; return; diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 6b57c2aa6..040008570 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -175,7 +175,7 @@ export const en = { 'help.option.analyze.walCheckpointThreshold': 'LadybugDB WAL auto-checkpoint threshold in bytes during analyze (integer >= -1; default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).', 'help.option.analyze.workers': - 'Parse worker pool size. Default: cores-1 capped at 16. Pass 0 to disable workers (sequential).', + 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.', 'help.option.analyze.embeddingThreads': 'Limit local ONNX embedding CPU threads', 'help.option.analyze.embeddingBatchSize': 'Number of nodes per embedding batch', 'help.option.analyze.embeddingSubBatchSize': 'Number of chunks per embedding model call', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 0eec71c44..6d1efb77a 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -164,7 +164,7 @@ export const zhCN = { 'help.option.analyze.walCheckpointThreshold': 'analyze 期间 LadybugDB WAL 自动 checkpoint 阈值（字节，整数 >= -1；默认：67108864 = 64 MiB；-1 保持 Ladybug 默认约 16 MiB）。', 'help.option.analyze.workers': - '解析 worker 池大小。默认：cores-1，最多 16。传 0 禁用 worker（顺序执行）。', + '解析 worker 池大小（>=1）。默认：cores-1，最多 16，按仓库规模自适应。', 'help.option.analyze.embeddingThreads': '限制本地 ONNX 嵌入 CPU 线程数', 'help.option.analyze.embeddingBatchSize': '每个嵌入批次的节点数', 'help.option.analyze.embeddingSubBatchSize': '每次嵌入模型调用的分块数', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 8ae99455d..a2a88bd83 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -87,7 +87,7 @@ program ) .option( '--workers ', - 'Parse worker pool size. Default: cores-1 capped at 16. Pass 0 to disable workers (sequential).', + 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.', ) .option('--embedding-threads ', 'Limit local ONNX embedding CPU threads') .option('--embedding-batch-size ', 'Number of nodes per embedding batch') diff --git a/gitnexus/src/core/ingestion/language-provider.ts b/gitnexus/src/core/ingestion/language-provider.ts index c979102e5..b375050fa 100644 --- a/gitnexus/src/core/ingestion/language-provider.ts +++ b/gitnexus/src/core/ingestion/language-provider.ts @@ -311,6 +311,27 @@ interface LanguageProviderConfig { }, ) => readonly CaptureMatch[]; + /** + * Snapshot the capture-time side-channel state that this provider's + * `emitScopeCaptures` just populated for `filePath` into module-level maps, + * returning a plain JSON-serializable value (or `undefined` when there is + * nothing to carry). + * + * Called in the parse worker IMMEDIATELY after `emitScopeCaptures` runs for + * a file (see `parse-worker.ts`), and the result is stored on the produced + * `ParsedFile.captureSideChannel`. Scope-resolution on the main thread reuses + * that serialized `ParsedFile` and skips re-extraction (#1983), so this hook + * is how the worker-computed marks survive the worker→main boundary and the + * disk store WITHOUT a main-thread re-parse. The main thread restores them + * via the matching `ScopeResolver.applyCaptureSideChannel` hook. + * + * MUST return plain data (objects / arrays / primitives) so it round-trips + * through `JSON.stringify` + the parsedfile-store interning reviver. + * + * Default: undefined (provider has no capture-time module-level side effects). + */ + readonly collectCaptureSideChannel?: (filePath: string) => unknown; + /** * Interpret a raw `@import.statement` capture group into a `ParsedImport`. * The central finalize algorithm resolves `ParsedImport.targetRaw` to a diff --git a/gitnexus/src/core/ingestion/languages/c-cpp.ts b/gitnexus/src/core/ingestion/languages/c-cpp.ts index 4ce17a9c5..ac8d4d30d 100644 --- a/gitnexus/src/core/ingestion/languages/c-cpp.ts +++ b/gitnexus/src/core/ingestion/languages/c-cpp.ts @@ -62,6 +62,7 @@ import { cppBindingScopeFor, cppImportOwningScope, cppReceiverBinding, + collectCppCaptureSideChannel, } from './cpp/index.js'; import { extractCppTemplateConstraints } from './cpp/constraint-extractor.js'; @@ -465,6 +466,11 @@ export const cppProvider = defineLanguage({ // ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ────────── emitScopeCaptures: emitCppScopeCaptures, + // Worker-side: snapshot the module-level capture marks `emitCppScopeCaptures` + // just populated for this file into plain data on `ParsedFile.captureSideChannel`, + // so the main thread can restore them via `applyCaptureSideChannel` WITHOUT a + // re-parse (#1983). See `cpp/capture-side-channel.ts`. + collectCaptureSideChannel: collectCppCaptureSideChannel, interpretImport: interpretCppImport, interpretTypeBinding: interpretCppTypeBinding, bindingScopeFor: cppBindingScopeFor, diff --git a/gitnexus/src/core/ingestion/languages/cpp/adl.ts b/gitnexus/src/core/ingestion/languages/cpp/adl.ts index 565c23125..67ced2016 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/adl.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/adl.ts @@ -379,6 +379,58 @@ export function markCppAdlSiteNoAdl(filePath: string, line: number, col: number) noAdlSites.add(siteKey(filePath, line, col)); } +/** + * Plain-data, JSON-serializable snapshot of the per-file ADL capture state + * (`argInfoBySite` entries for this file + `noAdlSites` keys for this file). + * Carried on `ParsedFile.captureSideChannel` across the worker→main boundary + * (#1983); the call-site key's `line:col` are stored per-entry so the full + * `filePath:line:col` key can be reconstructed without parsing. + */ +export interface CppAdlSideChannel { + /** Per-call-site arg info: `[line, col, args]` for sites in this file. */ + readonly argInfoBySite: readonly [number, number, readonly CppAdlArgInfo[]][]; + /** ADL-suppressed sites in this file: `[line, col]`. */ + readonly noAdlSites: readonly [number, number][]; +} + +const SITE_KEY_RE = /^(.*):(\d+):(\d+)$/; + +/** Split a `filePath:line:col` site key, tolerating colons in the path. */ +function parseSiteKey(key: string): { filePath: string; line: number; col: number } | undefined { + const m = SITE_KEY_RE.exec(key); + if (m === null) return undefined; + return { filePath: m[1], line: Number(m[2]), col: Number(m[3]) }; +} + +/** Snapshot this file's ADL capture state for the worker→main side-channel. */ +export function collectCppAdlSideChannel(filePath: string): CppAdlSideChannel { + const args: [number, number, readonly CppAdlArgInfo[]][] = []; + for (const [key, value] of argInfoBySite) { + const parsed = parseSiteKey(key); + if (parsed !== undefined && parsed.filePath === filePath) { + args.push([parsed.line, parsed.col, value]); + } + } + const noAdl: [number, number][] = []; + for (const key of noAdlSites) { + const parsed = parseSiteKey(key); + if (parsed !== undefined && parsed.filePath === filePath) { + noAdl.push([parsed.line, parsed.col]); + } + } + return { argInfoBySite: args, noAdlSites: noAdl }; +} + +/** Restore this file's ADL capture state from the side-channel (no parse). */ +export function applyCppAdlSideChannel(filePath: string, data: CppAdlSideChannel): void { + for (const [line, col, value] of data.argInfoBySite) { + argInfoBySite.set(siteKey(filePath, line, col), value); + } + for (const [line, col] of data.noAdlSites) { + noAdlSites.add(siteKey(filePath, line, col)); + } +} + /** Clear ADL state. Called from `cppScopeResolver.loadResolutionConfig` * (alongside `clearFileLocalNames`) so all C++ resolver per-pipeline state is * reset together at the start of each resolution pass. */ diff --git a/gitnexus/src/core/ingestion/languages/cpp/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/cpp/capture-side-channel.ts new file mode 100644 index 000000000..d8b1d32da --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/cpp/capture-side-channel.ts @@ -0,0 +1,98 @@ +/** + * C++ capture-time side-channel serialization (#1983). + * + * `emitCppScopeCaptures` populates several MODULE-LEVEL maps as a side effect + * that are NOT part of the returned `ParsedFile`'s scopes/defs: + * + * - `argInfoBySite` / `noAdlSites` (adl.ts) + * - `inlineNamespaceRangesByFile` (inline-namespaces.ts) + * - `fileLocalNames` / `anonymousNamespaceRangesByFile` (file-local-linkage.ts) + * - `dependentBasesByFile` / `dependentPackBaseClassesByFile` (two-phase-lookup.ts) + * + * On the worker path those maps are filled in the WORKER process and lost + * across the worker→main MessageChannel (and the disk-backed parsedfile-store), + * because scope-resolution reuses the serialized `ParsedFile` and SKIPS the + * main-thread re-extraction — the entire point of #1983 is to avoid a + * main-thread tree-sitter re-parse on huge `.h`/`.cpp` repos (the OOM). + * + * This module snapshots the per-file slice of those maps into a plain, + * JSON-serializable object (carried on `ParsedFile.captureSideChannel`) and + * restores it on the main thread WITHOUT any parse. It is the data-only + * replacement for the removed re-parse `replayCaptureSideChannel` hook. + * + * The derived state each `populateOwners` / `populateWorkspaceOwners` pass + * builds (resolved scope-id Sets, `dependentBaseNodeIds`, etc.) is recomputed + * on the main thread from these restored capture-time maps, so only the + * capture-time maps need to cross the boundary. + */ + +import type { ParsedFile } from 'gitnexus-shared'; +import { collectCppAdlSideChannel, applyCppAdlSideChannel, type CppAdlSideChannel } from './adl.js'; +import { + collectCppInlineNamespaceSideChannel, + applyCppInlineNamespaceSideChannel, +} from './inline-namespaces.js'; +import { + collectCppFileLocalSideChannel, + applyCppFileLocalSideChannel, + type CppFileLocalSideChannel, +} from './file-local-linkage.js'; +import { + collectCppTwoPhaseSideChannel, + applyCppTwoPhaseSideChannel, + type CppTwoPhaseSideChannel, +} from './two-phase-lookup.js'; + +/** + * Plain JSON-serializable composite of every C++ capture-time side-channel + * slice for one file. Carried opaquely on `ParsedFile.captureSideChannel`. + */ +export interface CppCaptureSideChannel { + readonly adl: CppAdlSideChannel; + /** Inline-namespace source-range keys recorded for this file. */ + readonly inlineNamespaceRanges: readonly string[]; + readonly fileLocal: CppFileLocalSideChannel; + readonly twoPhase: CppTwoPhaseSideChannel; +} + +/** + * `LanguageProvider.collectCaptureSideChannel` implementation for C++. + * Returns `undefined` when this file recorded no side-channel state at all, so + * the produced `ParsedFile` carries the field only when there's data to ship. + */ +export function collectCppCaptureSideChannel(filePath: string): CppCaptureSideChannel | undefined { + const adl = collectCppAdlSideChannel(filePath); + const inlineNamespaceRanges = collectCppInlineNamespaceSideChannel(filePath); + const fileLocal = collectCppFileLocalSideChannel(filePath); + const twoPhase = collectCppTwoPhaseSideChannel(filePath); + + const isEmpty = + adl.argInfoBySite.length === 0 && + adl.noAdlSites.length === 0 && + inlineNamespaceRanges.length === 0 && + fileLocal.fileLocalNames.length === 0 && + fileLocal.anonymousNamespaceRanges.length === 0 && + twoPhase.dependentBases.length === 0 && + twoPhase.dependentPackBaseClasses.length === 0; + if (isEmpty) return undefined; + + return { adl, inlineNamespaceRanges, fileLocal, twoPhase }; +} + +/** + * `ScopeResolver.applyCaptureSideChannel` implementation for C++. Reads the + * worker-serialized snapshot from `parsed.captureSideChannel` and writes it + * back into the module-level maps. Tolerant of `undefined` (file carried no + * data) and of an unexpected shape (defensive — never throws on a malformed + * snapshot). Does NO tree-sitter parse. + */ +export function applyCppCaptureSideChannel(parsed: ParsedFile): void { + const data = parsed.captureSideChannel as CppCaptureSideChannel | undefined; + if (data === undefined || data === null || typeof data !== 'object') return; + if (data.adl !== undefined) applyCppAdlSideChannel(parsed.filePath, data.adl); + if (data.inlineNamespaceRanges !== undefined) { + applyCppInlineNamespaceSideChannel(parsed.filePath, data.inlineNamespaceRanges); + } + if (data.fileLocal !== undefined) applyCppFileLocalSideChannel(parsed.filePath, data.fileLocal); + if (data.twoPhase !== undefined) applyCppTwoPhaseSideChannel(parsed.filePath, data.twoPhase); +} diff --git a/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts b/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts index dd5fc8c0a..09461d134 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts @@ -95,6 +95,47 @@ export function isCppAnonymousNamespaceScope(scopeId: ScopeId): boolean { return anonymousNamespaceScopeIds.has(scopeId); } +/** + * Plain-data, JSON-serializable snapshot of the per-file capture-time + * file-local-linkage state. Carried on `ParsedFile.captureSideChannel` across + * the worker→main boundary (#1983). The derived sets (`nonGloballyVisibleNodeIds`, + * `anonymousNamespaceScopeIds`) are recomputed by `populateCppNonGloballyVisible` + * / `populateCppAnonymousNamespaceScopes` during `populateOwners`, so only the + * two capture-time maps cross the boundary. + */ +export interface CppFileLocalSideChannel { + /** File-local symbol names (static / anonymous-namespace) in this file. */ + readonly fileLocalNames: readonly string[]; + /** Anonymous-namespace source-range keys recorded for this file. */ + readonly anonymousNamespaceRanges: readonly string[]; +} + +/** Snapshot this file's file-local-linkage capture state for the side-channel. */ +export function collectCppFileLocalSideChannel(filePath: string): CppFileLocalSideChannel { + const names = fileLocalNames.get(filePath); + const anon = anonymousNamespaceRangesByFile.get(filePath); + return { + fileLocalNames: names === undefined ? [] : [...names], + anonymousNamespaceRanges: anon === undefined ? [] : [...anon], + }; +} + +/** Restore this file's file-local-linkage capture state from the side-channel. */ +export function applyCppFileLocalSideChannel( + filePath: string, + data: CppFileLocalSideChannel, +): void { + for (const name of data.fileLocalNames) markFileLocal(filePath, name); + if (data.anonymousNamespaceRanges.length > 0) { + let set = anonymousNamespaceRangesByFile.get(filePath); + if (set === undefined) { + set = new Set(); + anonymousNamespaceRangesByFile.set(filePath, set); + } + for (const r of data.anonymousNamespaceRanges) set.add(r); + } +} + /** Clear tracked file-local names (call at start of each resolution pass). */ export function clearFileLocalNames(): void { fileLocalNames.clear(); diff --git a/gitnexus/src/core/ingestion/languages/cpp/index.ts b/gitnexus/src/core/ingestion/languages/cpp/index.ts index c4d208d76..4bc52dd32 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/index.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/index.ts @@ -14,3 +14,7 @@ export { clearFileLocalNames, expandCppWildcardNames, } from './file-local-linkage.js'; +export { + collectCppCaptureSideChannel, + applyCppCaptureSideChannel, +} from './capture-side-channel.js'; diff --git a/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts b/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts index eec44c68f..c7280bf46 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts @@ -61,6 +61,30 @@ export function markCppInlineNamespaceRange(filePath: string, range: RangeKey): set.add(rangeKey(range)); } +/** Snapshot this file's captured inline-namespace ranges for the worker→main + * side-channel (#1983). `populateCppInlineNamespaceScopes` (in `populateOwners`) + * later resolves these range keys to ScopeIds on the main thread, so only the + * capture-time ranges need to cross the boundary. Returns the rangeKey strings + * as a plain array (empty when this file recorded none). */ +export function collectCppInlineNamespaceSideChannel(filePath: string): readonly string[] { + const set = inlineNamespaceRangesByFile.get(filePath); + return set === undefined ? [] : [...set]; +} + +/** Restore this file's captured inline-namespace ranges from the side-channel. */ +export function applyCppInlineNamespaceSideChannel( + filePath: string, + ranges: readonly string[], +): void { + if (ranges.length === 0) return; + let set = inlineNamespaceRangesByFile.get(filePath); + if (set === undefined) { + set = new Set(); + inlineNamespaceRangesByFile.set(filePath, set); + } + for (const r of ranges) set.add(r); +} + /** Clear all inline-namespace state. Called from `clearFileLocalNames`. */ export function clearCppInlineNamespaces(): void { inlineNamespaceRangesByFile.clear(); diff --git a/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts index 459b313c6..725106c3d 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts @@ -30,6 +30,7 @@ import { isCppDependentBaseMember, } from './two-phase-lookup.js'; import { populateCppAssociatedNamespaces, clearCppAdlState, pickCppAdlCandidates } from './adl.js'; +import { applyCppCaptureSideChannel } from './capture-side-channel.js'; import { clearCppInlineNamespaces, populateCppInlineNamespaceScopes, @@ -103,6 +104,24 @@ export const cppScopeResolver: ScopeResolver = { buildMro: (graph, parsedFiles, nodeLookup) => buildMro(graph, parsedFiles, nodeLookup, defaultLinearize), + // Worker-boundary restore (see `ScopeResolver.applyCaptureSideChannel`). + // `emitCppScopeCaptures` records per-file ADL call-site arg shapes + // (`markCppAdlSiteArgs`/`markCppAdlSiteNoAdl`), inline-/anonymous-namespace + // ranges (`markCppInlineNamespaceRange`/`markCppAnonymousNamespaceRange`), + // dependent-base names (`markCppDependentBase`/`markCppDependentPackBase`), + // and file-local linkage (`markFileLocal`) into module-level maps as a SIDE + // EFFECT — none of it is serialized onto the returned ParsedFile's scopes/defs. + // On the worker path those marks are populated in the worker process and lost + // across the MessageChannel / disk store; the main thread reuses the + // serialized ParsedFile and skips `extractParsedFile`, so `populateOwners` + + // the ADL / two-phase-lookup passes would see empty maps and emit zero edges. + // The worker stashed a plain-data snapshot on `parsed.captureSideChannel` via + // `cppProvider.collectCaptureSideChannel`; this restores it into the module + // maps WITHOUT any tree-sitter re-parse (the #1983 fix — the old re-parse + // replay re-OOM'd huge `.h`/`.cpp` repos). The freshly-extracted leg never + // calls this — its marks were just populated in this process. + applyCaptureSideChannel: applyCppCaptureSideChannel, + populateOwners: (parsed: ParsedFile) => { populateClassOwnedMembers(parsed); // #1982: tag namespace-nested defs with their enclosing-namespace prefix so diff --git a/gitnexus/src/core/ingestion/languages/cpp/two-phase-lookup.ts b/gitnexus/src/core/ingestion/languages/cpp/two-phase-lookup.ts index 7ec55bc9a..739cceeb5 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/two-phase-lookup.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/two-phase-lookup.ts @@ -104,6 +104,56 @@ export function markCppDependentPackBase(filePath: string, className: string): v perFile.add(className); } +/** + * Plain-data, JSON-serializable snapshot of the per-file capture-time + * two-phase-lookup state. Carried on `ParsedFile.captureSideChannel` across the + * worker→main boundary (#1983). The resolved `dependentBaseNodeIds` index is + * rebuilt by `populateCppDependentBases` (workspace pass) after all files have + * their `populateOwners` applied, so only the two capture-time maps cross. + * + * Nested `Map`/`Set` are flattened to arrays here so the snapshot stays plain + * JSON (avoids relying on the parsedfile-store's Map/Set replacer for nested + * structures): `dependentBases` is `[className, [baseName, qualifiers[]][]][]`. + */ +export interface CppTwoPhaseSideChannel { + readonly dependentBases: readonly [string, readonly [string, readonly string[]][]][]; + readonly dependentPackBaseClasses: readonly string[]; +} + +/** Snapshot this file's two-phase-lookup capture state for the side-channel. */ +export function collectCppTwoPhaseSideChannel(filePath: string): CppTwoPhaseSideChannel { + const perFile = dependentBasesByFile.get(filePath); + const dependentBases: [string, [string, string[]][]][] = []; + if (perFile !== undefined) { + for (const [className, bases] of perFile) { + const baseEntries: [string, string[]][] = []; + for (const [baseName, quals] of bases) { + baseEntries.push([baseName, [...quals]]); + } + dependentBases.push([className, baseEntries]); + } + } + const pack = dependentPackBaseClassesByFile.get(filePath); + return { + dependentBases, + dependentPackBaseClasses: pack === undefined ? [] : [...pack], + }; +} + +/** Restore this file's two-phase-lookup capture state from the side-channel. */ +export function applyCppTwoPhaseSideChannel(filePath: string, data: CppTwoPhaseSideChannel): void { + for (const [className, baseEntries] of data.dependentBases) { + for (const [baseName, quals] of baseEntries) { + for (const qualifier of quals) { + markCppDependentBase(filePath, className, baseName, qualifier); + } + } + } + for (const className of data.dependentPackBaseClasses) { + markCppDependentPackBase(filePath, className); + } +} + /** Clear two-phase-lookup state. Called from `clearFileLocalNames`. */ export function clearCppDependentBases(): void { dependentBasesByFile.clear(); diff --git a/gitnexus/src/core/ingestion/languages/csharp/captures.ts b/gitnexus/src/core/ingestion/languages/csharp/captures.ts index b3b720587..3f28e6be9 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/captures.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/captures.ts @@ -86,10 +86,11 @@ export function emitCsharpScopeCaptures( _filePath: string, cachedTree?: unknown, ): readonly CaptureMatch[] { - // Skip the parse when the caller (parse phase's scopeTreeCache) - // already produced a Tree for this source. Cache miss = re-parse, - // same as before. The cachedTree parameter is typed as `unknown` at - // the LanguageProvider contract layer; cast here at the use site. + // Reuse a pre-parsed Tree when the caller passes one via `cachedTree`; a + // miss re-parses. (The cache is currently always empty — its only producer, + // the sequential parser, was removed — so this re-parses in practice.) The + // cachedTree parameter is typed `unknown` at the LanguageProvider contract + // layer; cast here at the use site. let tree = cachedTree as ReturnType['parse']> | undefined; if (tree === undefined) { tree = parseSourceSafe(getCsharpParser(), sourceText, undefined, { diff --git a/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts b/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts index 634afdeec..5dab83afe 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts @@ -356,8 +356,9 @@ function extractFileStructure(content: string, cachedTree: unknown): CsharpFileS /** Content + (optional) pre-parsed tree-sitter trees keyed by filePath. * The orchestrator builds `fileContents` from the pipeline's file list; - * `treeCache` is the same `scopeTreeCache` already populated by the - * parse phase, so cache hits avoid a second `parser.parse()`. */ + * `treeCache` is currently always empty (its only producer, the sequential + * parser, was removed), so the providers re-parse. Kept as an extension + * point that would let cache hits avoid a second `parser.parse()`. */ export interface CsharpSiblingInputs { readonly fileContents: ReadonlyMap; readonly treeCache?: { get(filePath: string): unknown }; diff --git a/gitnexus/src/core/ingestion/languages/kotlin.ts b/gitnexus/src/core/ingestion/languages/kotlin.ts index 31116663e..6ee7b1dda 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin.ts @@ -28,6 +28,7 @@ import { kotlinMethodConfig } from '../method-extractors/configs/jvm.js'; import { createVariableExtractor } from '../variable-extractors/generic.js'; import { kotlinVariableConfig } from '../variable-extractors/configs/jvm.js'; import { + collectKotlinCaptureSideChannel, emitKotlinScopeCaptures, interpretKotlinImport, interpretKotlinTypeBinding, @@ -175,6 +176,13 @@ export const kotlinProvider = defineLanguage({ // ── RFC #909 Ring 3: scope-based resolution hooks ── emitScopeCaptures: emitKotlinScopeCaptures, + // Worker-side: snapshot the module-level companion-scope marks + // `emitKotlinScopeCaptures` just populated for this file (`markCompanionScope` + // → `companionScopesByFile`) into plain data on `ParsedFile.captureSideChannel`, + // so the main thread can restore them via `applyCaptureSideChannel` WITHOUT a + // re-parse (#1983). Without this, companion/static dispatch emits no CALLS + // edges on the worker path. See `kotlin/capture-side-channel.ts`. + collectCaptureSideChannel: collectKotlinCaptureSideChannel, interpretImport: interpretKotlinImport, interpretTypeBinding: interpretKotlinTypeBinding, bindingScopeFor: kotlinBindingScopeFor, diff --git a/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts new file mode 100644 index 000000000..375e0f679 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts @@ -0,0 +1,75 @@ +/** + * Kotlin capture-time side-channel serialization (#1983). + * + * `emitKotlinScopeCaptures` populates one MODULE-LEVEL, per-file map as a side + * effect that is NOT part of the returned `ParsedFile`'s scopes/defs: + * + * - `companionScopesByFile` (companion-scopes.ts) — the `ScopeId`s that came + * from a `companion_object` AST node, recorded via `markCompanionScope` + * from the `@scope.companion` marker capture. + * + * On the worker path that map is filled in the WORKER process and lost across + * the worker→main MessageChannel (and the disk-backed parsedfile-store), + * because scope-resolution reuses the serialized `ParsedFile` and SKIPS the + * main-thread re-extraction (the #1983 fix that avoids a main-thread + * tree-sitter re-parse / OOM on huge repos). The main thread then reads the map + * empty in `isKotlinStaticOnly` / `populateCompanionMembersOnEnclosingClass` + * (owners.ts) — so companion methods aren't identified as static and + * companion/static dispatch emits no CALLS edges. + * + * This module snapshots the per-file slice of that map into a plain, + * JSON-serializable object (carried on `ParsedFile.captureSideChannel`) and + * restores it on the main thread WITHOUT any parse. It mirrors the C++ pattern + * in `cpp/capture-side-channel.ts`. + * + * The single generic `ParsedFile.captureSideChannel` field is shared with C++, + * which is safe because each file is one language (a `.kt` file uses the kotlin + * provider, a `.cpp` file the cpp provider). The payload is self-describing + * (`{ kind: 'kotlin', companionScopes }`) so `applyKotlinCaptureSideChannel` + * only restores kotlin state and ignores a foreign-shaped snapshot. + */ + +import type { ParsedFile, ScopeId } from 'gitnexus-shared'; +import { getCompanionScopesForFile, markCompanionScope } from './companion-scopes.js'; + +/** + * Plain JSON-serializable snapshot of the per-file Kotlin capture-time + * side-channel. Carried opaquely on `ParsedFile.captureSideChannel`. The + * `kind` tag makes the payload self-describing so `apply` can distinguish a + * kotlin snapshot from another language's (C++ shares the same field). + */ +export interface KotlinCaptureSideChannel { + readonly kind: 'kotlin'; + /** Companion-object scope ids recorded for this file. */ + readonly companionScopes: readonly ScopeId[]; +} + +/** + * `LanguageProvider.collectCaptureSideChannel` implementation for Kotlin. + * Returns `undefined` when this file recorded no companion scopes at all, so + * the produced `ParsedFile` carries the field only when there's data to ship. + */ +export function collectKotlinCaptureSideChannel( + filePath: string, +): KotlinCaptureSideChannel | undefined { + const companionScopes = getCompanionScopesForFile(filePath); + if (companionScopes.length === 0) return undefined; + return { kind: 'kotlin', companionScopes }; +} + +/** + * `ScopeResolver.applyCaptureSideChannel` implementation for Kotlin. Reads the + * worker-serialized snapshot from `parsed.captureSideChannel` and re-populates + * the module-level companion-scope map via `markCompanionScope`. Tolerant of + * `undefined` (file carried no data) and of an unexpected / foreign shape + * (defensive — the `kind` tag guards against restoring a non-kotlin payload). + * Does NO tree-sitter parse. + */ +export function applyKotlinCaptureSideChannel(parsed: ParsedFile): void { + const data = parsed.captureSideChannel as KotlinCaptureSideChannel | undefined; + if (data === undefined || data === null || typeof data !== 'object') return; + if (data.kind !== 'kotlin' || !Array.isArray(data.companionScopes)) return; + for (const scopeId of data.companionScopes) { + markCompanionScope(parsed.filePath, scopeId); + } +} diff --git a/gitnexus/src/core/ingestion/languages/kotlin/companion-scopes.ts b/gitnexus/src/core/ingestion/languages/kotlin/companion-scopes.ts index ac4034620..f1c45c65c 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/companion-scopes.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/companion-scopes.ts @@ -55,6 +55,17 @@ export function isCompanionScope(filePath: string, scopeId: ScopeId): boolean { return companionScopesByFile.get(filePath)?.has(scopeId) ?? false; } +/** + * Snapshot the companion-object scope ids recorded for `filePath` as a plain + * array (for the worker→main capture side-channel, #1983). Returns an empty + * array when the file recorded no companion scopes. See + * `capture-side-channel.ts`. + */ +export function getCompanionScopesForFile(filePath: string): ScopeId[] { + const scopes = companionScopesByFile.get(filePath); + return scopes === undefined ? [] : [...scopes]; +} + /** Clear all tracked companion scopes (for testing). */ export function clearCompanionScopes(): void { companionScopesByFile.clear(); diff --git a/gitnexus/src/core/ingestion/languages/kotlin/index.ts b/gitnexus/src/core/ingestion/languages/kotlin/index.ts index 206128e52..206254540 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/index.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/index.ts @@ -1,4 +1,9 @@ export { emitKotlinScopeCaptures } from './captures.js'; +export { + collectKotlinCaptureSideChannel, + applyKotlinCaptureSideChannel, + type KotlinCaptureSideChannel, +} from './capture-side-channel.js'; export { getKotlinCaptureCacheStats, resetKotlinCaptureCacheStats } from './cache-stats.js'; export { interpretKotlinImport, interpretKotlinTypeBinding } from './interpret.js'; export { kotlinArityCompatibility } from './arity.js'; diff --git a/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts index bd63c1e86..6bd48f0d1 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts @@ -14,6 +14,7 @@ import { type KotlinResolveContext, } from './index.js'; import { clearCompanionScopes } from './companion-scopes.js'; +import { applyKotlinCaptureSideChannel } from './capture-side-channel.js'; import { isKotlinStaticOnly } from './owners.js'; /** @@ -84,6 +85,23 @@ export const kotlinScopeResolver: ScopeResolver = { buildMro: (graph, parsedFiles, nodeLookup) => buildKotlinMro(graph, parsedFiles, nodeLookup), + // Worker-boundary restore (see `ScopeResolver.applyCaptureSideChannel`). + // `emitKotlinScopeCaptures` records per-file companion-object scope ids + // (`markCompanionScope` → `companionScopesByFile`) as a SIDE EFFECT — that + // state is NOT serialized onto the returned ParsedFile's scopes/defs. On the + // worker path those marks are populated in the worker process and lost across + // the MessageChannel / disk store; the main thread reuses the serialized + // ParsedFile and skips `extractParsedFile`, so `isKotlinStaticOnly` and + // `populateCompanionMembersOnEnclosingClass` (owners.ts) would see an empty + // map and companion/static dispatch would emit zero CALLS edges. The worker + // stashed a plain-data snapshot on `parsed.captureSideChannel` via + // `kotlinProvider.collectCaptureSideChannel`; this restores it into the + // module map WITHOUT any tree-sitter re-parse (the #1983 fix). The + // freshly-extracted leg never calls this — its marks were just populated in + // this process. Runs BEFORE `populateOwners` so the restored companion map is + // visible to it. + applyCaptureSideChannel: applyKotlinCaptureSideChannel, + populateOwners: (parsed: ParsedFile) => populateKotlinOwners(parsed), isSuperReceiver: (text) => text.trim() === 'super', diff --git a/gitnexus/src/core/ingestion/languages/typescript/captures.ts b/gitnexus/src/core/ingestion/languages/typescript/captures.ts index b3e338242..9d466c3f9 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/captures.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/captures.ts @@ -163,10 +163,11 @@ export function emitTsScopeCaptures( filePath: string, cachedTree?: unknown, ): readonly CaptureMatch[] { - // Skip the parse when the caller (parse phase's scopeTreeCache) already - // produced a Tree for this source. Cache miss = re-parse, same as before. - // The cachedTree parameter is typed as `unknown` at the LanguageProvider - // contract layer; cast here at the use site. + // Reuse a pre-parsed Tree when the caller passes one via `cachedTree`; a + // miss re-parses. (The cache is currently always empty — its only producer, + // the sequential parser, was removed — so this re-parses in practice.) The + // cachedTree parameter is typed `unknown` at the LanguageProvider contract + // layer; cast here at the use site. // // Grammar selection: `.tsx` files are parsed with the TSX grammar, // `.ts` files with the TypeScript grammar. The two grammars have diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index 1858256ce..a08684bdc 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -1,48 +1,11 @@ -import type { GraphNode, GraphRelationship, NodeLabel, ParameterTypeClass } from 'gitnexus-shared'; +import type { NodeLabel } from 'gitnexus-shared'; import { KnowledgeGraph } from '../graph/types.js'; -import Parser from 'tree-sitter'; -import { loadParser, loadLanguage, isLanguageAvailable } from '../tree-sitter/parser-loader.js'; -import { getProvider } from './languages/index.js'; -import { generateId } from '../../lib/utils.js'; -import type { SymbolTableReader, SymbolTableWriter } from './model/index.js'; +import type { SymbolTableWriter } from './model/index.js'; import { ASTCache } from './ast-cache.js'; -import { getLanguageFromFilename, SupportedLanguages } from 'gitnexus-shared'; -import { extractVueScript, isVueSetupTopLevel } from './vue-sfc-extractor.js'; -import { yieldToEventLoop } from './utils/event-loop.js'; -import { parseSourceSafe } from '../tree-sitter/safe-parse.js'; -import { isVerboseIngestionEnabled } from './utils/verbose.js'; -import { - buildConcreteTypedefDefinitionRanges, - getDefinitionNodeFromCaptures, - findEnclosingClassInfo, - findObjectLiteralBindingInfo, - getLabelFromCaptures, - isSuppressedConcreteTypedefDuplicate, - isQualifiableScopeLabel, - qualifyRustImplTargetByModScope, - CLASS_CONTAINER_TYPES, - type SyntaxNode, - type EnclosingClassInfo, -} from './utils/ast-helpers.js'; -import { detectFrameworkFromAST } from './framework-detection.js'; -import { buildTypeEnv } from './type-env.js'; +import { getLanguageFromFilename } from 'gitnexus-shared'; + import { accumulateExportedTypesFromParsedNode, type ExportedTypeMap } from './call-processor.js'; -import type { FieldInfo, FieldExtractorContext } from './field-types.js'; -import type { MethodInfo } from './method-types.js'; -import { - buildMethodProps, - arityForIdFromInfo, - typeTagForId, - constTagForId, - buildCollisionGroups, - parameterShapeIdTag, -} from './utils/method-props.js'; -import { - extractTemplateArguments, - templateArgumentsIdTag, - templateConstraintsIdTag, -} from './utils/template-arguments.js'; -import type { LanguageProvider } from './language-provider.js'; + import type { ParsedFile } from 'gitnexus-shared'; import { WorkerPool } from './workers/worker-pool.js'; import { logger } from '../logger.js'; @@ -65,16 +28,6 @@ import type { ExtractedRouterInclude, ExtractedRouterModuleAlias, } from './route-extractors/fastapi-router-bindings.js'; -import { - getTreeSitterBufferSize, - getTreeSitterContentByteLength, - TREE_SITTER_MAX_BUFFER, -} from './constants.js'; -import { - ARRAY_METHOD_HOC_BLOCKLIST_SET, - DEFAULT_EXPORT_IDENTIFIER_BLOCKLIST_SET, - deriveDefaultExportHocName, -} from './ts-js-hoc-utils.js'; export type FileProgressCallback = (current: number, total: number, filePath: string) => void; @@ -196,6 +149,60 @@ export const mergeChunkResults = ( }; }; +/** + * Dispatch a chunk's files to the worker pool and return the RAW per-worker + * results, WITHOUT merging them into the graph. Split out from + * {@link processParsingWithWorkers} so the parse loop can overlap one chunk's + * merge (main-thread, via {@link mergeChunkResults}) with the NEXT chunk's + * worker parse — the merge is the only remaining serial main-thread step once + * ParsedFile serialization moved into the workers (#worker-idle pipelining). + * Returns `[]` for an all-unparseable chunk (the caller merges `[]` → empty). + */ +export const dispatchChunkParse = async ( + files: { path: string; content: string }[], + workerPool: WorkerPool, + onFileProgress?: FileProgressCallback, + /** Populated in-place with the raw results (parse-cache capture). */ + outRawResults?: ParseWorkerResult[], +): Promise => { + const parseableFiles: ParseWorkerInput[] = []; + for (const file of files) { + const lang = getLanguageFromFilename(file.path); + if (lang) parseableFiles.push({ path: file.path, content: file.content }); + } + if (parseableFiles.length === 0) return []; + + const total = files.length; + const chunkResults = await workerPool.dispatch( + parseableFiles, + (filesProcessed) => { + onFileProgress?.(Math.min(filesProcessed, total), total, 'Parsing...'); + }, + ); + + // Capture raw results for the incremental parse cache before merging. + if (outRawResults) { + for (const r of chunkResults) outRawResults.push(r); + } + + // Skipped-language telemetry (worker output, independent of the merge). + const skippedLanguages = new Map(); + for (const result of chunkResults) { + for (const [lang, count] of Object.entries(result.skippedLanguages)) { + skippedLanguages.set(lang, (skippedLanguages.get(lang) || 0) + count); + } + } + if (skippedLanguages.size > 0) { + const summary = Array.from(skippedLanguages.entries()) + .map(([lang, count]) => `${lang}: ${count}`) + .join(', '); + logger.warn(` Skipped unsupported languages: ${summary}`); + } + + onFileProgress?.(total, total, 'done'); + return chunkResults; +}; + const processParsingWithWorkers = async ( graph: KnowledgeGraph, files: { path: string; content: string }[], @@ -213,791 +220,9 @@ const processParsingWithWorkers = async ( outRawResults?: ParseWorkerResult[], exportedTypeMap?: ExportedTypeMap, ): Promise => { - // Filter to parseable files only - const parseableFiles: ParseWorkerInput[] = []; - for (const file of files) { - const lang = getLanguageFromFilename(file.path); - if (lang) parseableFiles.push({ path: file.path, content: file.content }); - } - - if (parseableFiles.length === 0) - return { - calls: [], - assignments: [], - routes: [], - fetchCalls: [], - fetchWrapperDefs: [], - decoratorRoutes: [], - routerIncludes: [], - routerImports: [], - routerModuleAliases: [], - toolDefs: [], - ormQueries: [], - constructorBindings: [], - fileScopeBindings: [], - parsedFiles: [], - }; - - const total = files.length; - - // Dispatch to worker pool — pool handles splitting into chunks and sub-batching - const chunkResults = await workerPool.dispatch( - parseableFiles, - (filesProcessed) => { - onFileProgress?.(Math.min(filesProcessed, total), total, 'Parsing...'); - }, - ); - - // Capture the raw chunk results for the incremental parse cache before - // merging — the cache stores the unmerged worker output so a future run - // can re-merge them into a fresh graph state. - if (outRawResults) { - for (const r of chunkResults) outRawResults.push(r); - } - - // Merge results from all workers into graph and symbol table. - const merged = mergeChunkResults(graph, symbolTable, chunkResults, exportedTypeMap); - - // Merge and log skipped languages from workers - const skippedLanguages = new Map(); - for (const result of chunkResults) { - for (const [lang, count] of Object.entries(result.skippedLanguages)) { - skippedLanguages.set(lang, (skippedLanguages.get(lang) || 0) + count); - } - } - if (skippedLanguages.size > 0) { - const summary = Array.from(skippedLanguages.entries()) - .map(([lang, count]) => `${lang}: ${count}`) - .join(', '); - logger.warn(` Skipped unsupported languages: ${summary}`); - } - - // Final progress - onFileProgress?.(total, total, 'done'); - return merged; -}; - -// ============================================================================ -// Sequential fallback (original implementation) -// ============================================================================ - -// Inline caches to avoid repeated parent-walks per node (same pattern as parse-worker.ts). -// Keyed by tree-sitter node reference — cleared at the start of each file. -const classInfoCache = new Map(); -const exportCache = new Map(); - -const cachedFindEnclosingClassInfo = ( - node: SyntaxNode, - filePath: string, - resolveEnclosingOwner?: (node: SyntaxNode) => SyntaxNode | null, - getQualifiedOwnerName?: (node: SyntaxNode, simpleName: string) => string | null, -): EnclosingClassInfo | null => { - const cached = classInfoCache.get(node); - if (cached !== undefined) return cached; - const result = findEnclosingClassInfo( - node, - filePath, - resolveEnclosingOwner, - getQualifiedOwnerName, - ); - classInfoCache.set(node, result); - return result; -}; - -const cachedExportCheck = ( - checker: (node: SyntaxNode, name: string) => boolean, - node: SyntaxNode, - name: string, -): boolean => { - const cached = exportCache.get(node); - if (cached !== undefined) return cached; - const result = checker(node, name); - exportCache.set(node, result); - return result; -}; - -// FieldExtractor cache for sequential path — same pattern as parse-worker.ts -const seqFieldInfoCache = new Map>(); - -// MethodExtractor cache for sequential path — avoids re-traversing the same class -// body once per method. Keyed on classNode.id (tree-sitter node identity number). -const seqMethodExtractCache = new Map< - number, - { ownerName: string | undefined; methods: MethodInfo[] } | null ->(); -// Derived method map + collision groups cache — avoids rebuilding per method. -const seqMethodMapCache = new Map< - number, - { map: Map; groups: Map } ->(); - -/** Provider-aware enclosing container lookup. - * Walks up from `node` until a CLASS_CONTAINER_TYPES node is found. - * When `resolveEnclosingOwner` is provided, delegates language-specific - * container remapping (e.g., Ruby singleton_class → enclosing class). - * Without the hook, returns the first matching container directly (raw lookup). */ -function seqFindEnclosingOwnerNode( - node: SyntaxNode, - resolveEnclosingOwner?: (node: SyntaxNode) => SyntaxNode | null, -): SyntaxNode | null { - let current = node.parent; - while (current) { - if (CLASS_CONTAINER_TYPES.has(current.type)) { - if (resolveEnclosingOwner) { - const resolved = resolveEnclosingOwner(current); - if (resolved === null) { - // Provider says skip this container — keep walking up. - current = current.parent; - continue; - } - return resolved; - } - return current; - } - current = current.parent; - } - return null; -} - -/** Minimal no-op SymbolTable stub for sequential extractor contexts. The real - * SymbolTable is not fully populated yet at this stage, so use the stub for safety. - * Implements the full {@link SymbolTableReader} surface so future extractor additions - * don't silently fall off an `as unknown as` cast. */ -const NOOP_SYMBOL_TABLE_SEQ: SymbolTableReader = { - lookupExact: () => undefined, - lookupExactFull: () => undefined, - lookupExactAll: () => [], - lookupCallableByName: () => [], - getFiles: () => [][Symbol.iterator](), - getStats: () => ({ fileCount: 0 }), -}; - -function seqGetFieldInfo( - classNode: SyntaxNode, - provider: LanguageProvider, - context: FieldExtractorContext, -): Map | undefined { - if (!provider.fieldExtractor) return undefined; - const cacheKey = classNode.startIndex; - let cached = seqFieldInfoCache.get(cacheKey); - if (cached) return cached; - const extracted = provider.fieldExtractor.extract(classNode, context); - if (!extracted?.fields?.length) return undefined; - cached = new Map(); - for (const field of extracted.fields) cached.set(field.name, field); - seqFieldInfoCache.set(cacheKey, cached); - return cached; -} - -const processParsingSequential = async ( - graph: KnowledgeGraph, - files: { path: string; content: string }[], - symbolTable: SymbolTableWriter, - astCache: ASTCache, - scopeTreeCache: ASTCache | undefined, - onFileProgress?: FileProgressCallback, - exportedTypeMap?: ExportedTypeMap, -) => { - const parser = await loadParser(); - const total = files.length; - const logSkipped = isVerboseIngestionEnabled(); - const skippedByLang = logSkipped ? new Map() : null; - - for (let i = 0; i < files.length; i++) { - const file = files[i]; - - // Reset memoization before each new file (node refs are per-tree) - classInfoCache.clear(); - exportCache.clear(); - seqFieldInfoCache.clear(); - seqMethodExtractCache.clear(); - seqMethodMapCache.clear(); - - onFileProgress?.(i + 1, total, file.path); - - if (i % 20 === 0) await yieldToEventLoop(); - - const language = getLanguageFromFilename(file.path); - - if (!language) continue; - if (!isLanguageAvailable(language)) { - if (skippedByLang) { - skippedByLang.set(language, (skippedByLang.get(language) ?? 0) + 1); - } - continue; - } - - // Skip files larger than the max tree-sitter buffer (32 MB) - if (getTreeSitterContentByteLength(file.content) > TREE_SITTER_MAX_BUFFER) continue; - - // Vue SFC preprocessing: extract