diff --git a/gitnexus/src/core/ingestion/language-config.ts b/gitnexus/src/core/ingestion/language-config.ts index 4657e275e..4776d897e 100644 --- a/gitnexus/src/core/ingestion/language-config.ts +++ b/gitnexus/src/core/ingestion/language-config.ts @@ -1,7 +1,9 @@ import fs from 'fs/promises'; +import { createReadStream } from 'fs'; +import { createInterface } from 'readline'; import path from 'path'; import type { ImportConfigs } from './import-resolvers/types.js'; -import type { CsharpFileStructure } from './languages/csharp/namespace-siblings.js'; +import type { CsharpStructureLineScanner } from './languages/csharp/namespace-siblings.js'; import { isDev } from './utils/env.js'; import { getMaxFileSizeBytes } from './utils/max-file-size.js'; @@ -203,14 +205,14 @@ const CSHARP_ROOT_NAMESPACE_RE = /\s*([^<]+)\s*<\/RootNamespace>/ // with phantom namespaces. Imported lazily (and memoized) so the always-on // `loadImportConfigs` path — every repo, every language — doesn't eagerly // pull tree-sitter-c-sharp in via `namespace-siblings.ts` → `query.ts`. -let csharpScannerPromise: Promise<(content: string) => CsharpFileStructure> | undefined; -function getCsharpStructureScanner(): Promise<(content: string) => CsharpFileStructure> { - if (csharpScannerPromise === undefined) { - csharpScannerPromise = import('./languages/csharp/namespace-siblings.js').then( - (mod) => mod.extractCsharpStructureViaScanner, +let csharpScannerFactoryPromise: Promise<() => CsharpStructureLineScanner> | undefined; +function getCsharpStructureScannerFactory(): Promise<() => CsharpStructureLineScanner> { + if (csharpScannerFactoryPromise === undefined) { + csharpScannerFactoryPromise = import('./languages/csharp/namespace-siblings.js').then( + (mod) => mod.createCsharpStructureScanner, ); } - return csharpScannerPromise; + return csharpScannerFactoryPromise; } /** @@ -298,29 +300,25 @@ export async function scanCSharpProject(repoRoot: string): Promise - collectDeclaredNamespaces( - path.join(dir, name), - declaredNamespaces, - rootNamespaces, - maxFileSizeBytes, - ), + collectDeclaredNamespaces(path.join(dir, name), declaredNamespaces, rootNamespaces), ), ); - // A `.cs` that was skipped (oversized), unreadable, or whose read/scan - // unexpectedly rejected leaves its namespaces uncollected → mark truncated - // to fail the #1881 gate OPEN rather than wrongly suppress an import. + // A `.cs` that was unreadable (or whose read/scan unexpectedly rejected) + // leaves its namespaces uncollected → mark truncated to fail the #1881 + // gate OPEN rather than wrongly suppress an import. The scan streams each + // file, so file size no longer trips truncation. for (const r of settled) { - if (r.status !== 'fulfilled' || r.value) truncated = true; + if (r.status !== 'fulfilled' || r.value === 'truncated') truncated = true; } } } if (truncated) { - // Surface the fail-open so a too-small cap (or an unreadable subtree) - // silently disabling the #1881 gate repo-wide is observable (#4) rather - // than a mystery edge regression. + // Surface the fail-open so an incomplete scan (dir/depth cap, or an + // unreadable directory or `.cs` file) silently disabling the #1881 gate + // repo-wide is observable (#4) rather than a mystery edge regression. logger.warn( - `[csharp] namespace scan of ${repoRoot} truncated (dir cap ${CSHARP_SCAN_MAX_DIRS}, depth cap ${CSHARP_SCAN_MAX_DEPTH}, or an unreadable directory); the #1881 suffix-fallback gate fails open for unmatched usings`, + `[csharp] namespace scan of ${repoRoot} truncated (dir cap ${CSHARP_SCAN_MAX_DIRS}, depth cap ${CSHARP_SCAN_MAX_DEPTH}, an unreadable directory, or an unreadable .cs file); the #1881 suffix-fallback gate fails open for unmatched usings`, ); } return { configs, declaredNamespaces, rootNamespaces, truncated }; @@ -352,36 +350,47 @@ async function readCsprojConfig( } /** - * Collect declared `namespace` names from one `.cs` file into the shared Sets. + * Stream one `.cs` file line-by-line and collect its declared `namespace` names + * into the shared Sets. * - * Returns `true` when the file was skipped (oversized) or could not be read, so - * the caller can mark the scan truncated — its namespaces are missing, and the - * #1881 gate must fail OPEN rather than wrongly suppress an import declared in - * the unread file. Returns `false` on a successful read. + * Streaming (rather than reading the whole file into a string) keeps memory + * constant regardless of file size, so a large generated `.cs` (`*.g.cs`, EF / + * gRPC output) is fully scanned instead of skipped by a per-file size cap — + * which would otherwise trip `truncated` and disable the #1881 gate repo-wide. + * Only the cheap line scan streams here; the tree-sitter PARSE path keeps its + * own size cap. + * + * Returns `'truncated'` when the file could not be read, so the caller marks the + * scan truncated and the #1881 gate fails OPEN rather than wrongly suppress an + * import declared in the unread file. Returns `'ok'` on a complete read. */ async function collectDeclaredNamespaces( filePath: string, declaredNamespaces: Set, rootNamespaces: Set, - maxFileSizeBytes: number, -): Promise { - let content: string; +): Promise<'ok' | 'truncated'> { + const createScanner = await getCsharpStructureScannerFactory(); + const scanner = createScanner(); try { - const stat = await fs.stat(filePath); - if (stat.size > maxFileSizeBytes) { - return true; // oversized source → skip unread, signal truncation + // `crlfDelay: Infinity` treats every `\r\n` as a single break; the line + // scanner is terminator-agnostic, so a streamed scan yields the same + // namespaces as scanning the whole file content at once. + const lines = createInterface({ + input: createReadStream(filePath, { encoding: 'utf-8' }), + crlfDelay: Infinity, + }); + for await (const line of lines) { + scanner.pushLine(line); } - content = await fs.readFile(filePath, 'utf-8'); } catch { - return true; // unreadable source → signal truncation (was a silent skip) + return 'truncated'; // unreadable source → signal truncation (fail open) } - const scan = await getCsharpStructureScanner(); - for (const ns of scan(content).namespaces) { + for (const ns of scanner.result().namespaces) { declaredNamespaces.add(ns); const dot = ns.indexOf('.'); rootNamespaces.add(dot === -1 ? ns : ns.slice(0, dot)); } - return false; + return 'ok'; } export async function loadSwiftPackageConfig(repoRoot: string): Promise { diff --git a/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts b/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts index f4d29d7e7..fdf79a4de 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts @@ -182,26 +182,51 @@ function advanceCsScanState( * AST is a declaration whose keyword is not at the start of a code line * (split across lines, or sharing a line with a comment/string closer). * Mirrors PHP's `extractNamespaceViaScanner` (issue #1741). */ -export function extractCsharpStructureViaScanner(content: string): CsharpFileStructure { +/** Incremental form of {@link extractCsharpStructureViaScanner}: feed lines one + * at a time via `pushLine` (in source order), then read the accumulated + * structure with `result()`. Lets a caller stream a file off disk + * (`createReadStream` + `readline`) and scan it for `namespace` / `using + * static` declarations in CONSTANT memory rather than buffering the whole file + * into a string — the line splitting and per-line matching are identical, so a + * streamed scan yields the same result as scanning the full content. The line + * terminator must be stripped (as `readline` does, or `String.split('\n')`); a + * trailing `\r` on a CRLF line is inert to both the matchers and the lexer. */ +export interface CsharpStructureLineScanner { + pushLine(line: string): void; + result(): CsharpFileStructure; +} + +/** Create a fresh stateful line scanner — see {@link CsharpStructureLineScanner}. */ +export function createCsharpStructureScanner(): CsharpStructureLineScanner { const namespaces: string[] = []; const usingStaticPaths: string[] = []; let state: CsScanState = 'code'; let rawFence = 0; - for (const line of content.split('\n')) { - // Only match when the line START is real code — keywords reached while - // inside a block comment / multi-line string are skipped. - if (state === 'code') { - const ns = CS_NAMESPACE_RE.exec(line); - if (ns !== null) { - namespaces.push(ns[1]!); - } else { - const us = CS_USING_STATIC_RE.exec(line); - if (us !== null) usingStaticPaths.push(us[1]!); + return { + pushLine(line: string): void { + // Only match when the line START is real code — keywords reached while + // inside a block comment / multi-line string are skipped. + if (state === 'code') { + const ns = CS_NAMESPACE_RE.exec(line); + if (ns !== null) { + namespaces.push(ns[1]!); + } else { + const us = CS_USING_STATIC_RE.exec(line); + if (us !== null) usingStaticPaths.push(us[1]!); + } } - } - [state, rawFence] = advanceCsScanState(line, state, rawFence); - } - return { namespaces, usingStaticPaths }; + [state, rawFence] = advanceCsScanState(line, state, rawFence); + }, + result(): CsharpFileStructure { + return { namespaces, usingStaticPaths }; + }, + }; +} + +export function extractCsharpStructureViaScanner(content: string): CsharpFileStructure { + const scanner = createCsharpStructureScanner(); + for (const line of content.split('\n')) scanner.pushLine(line); + return scanner.result(); } /** Build a structural view of a C# file. Prefers `cachedTree` (handed in diff --git a/gitnexus/test/unit/scope-resolution/csharp/csharp-imports.test.ts b/gitnexus/test/unit/scope-resolution/csharp/csharp-imports.test.ts index 552f61888..361bb663b 100644 --- a/gitnexus/test/unit/scope-resolution/csharp/csharp-imports.test.ts +++ b/gitnexus/test/unit/scope-resolution/csharp/csharp-imports.test.ts @@ -609,24 +609,27 @@ describe('loadCsharpResolutionConfig — one-pass namespace scan (#1881)', () => } }); - it('skips an oversized .cs file and fails open via truncation (#1881)', async () => { - // A .cs file larger than the per-file size cap is skipped unread so the - // always-on scan can't pull an unbounded buffer into memory. Its namespace - // is then missing, so `truncated` trips and the gate fails OPEN rather than - // wrongly suppress an import declared there. Sized to the real cap — do NOT - // lower the production cap for the test. + it('streams a large .cs file end-to-end, collecting namespaces past the old size cap (#1881)', async () => { + // The namespace scan streams each file, so a `.cs` far larger than the old + // per-file size cap is read end-to-end in constant memory instead of being + // skipped. A namespace at the START and one at the very END (well past the + // old cap boundary) must BOTH be collected, and `truncated` must stay false + // — a big generated file no longer disables the #1881 gate repo-wide. const cap = getMaxFileSizeBytes(); - const oversized = `namespace Deep.Ns;\n${'// pad\n'.repeat(Math.ceil(cap / 7) + 1)}`; + const padLine = '// pad pad pad pad pad pad\n'; + const padding = padLine.repeat(Math.ceil((cap * 3) / padLine.length)); + const huge = `namespace Generated.Head;\n${padding}namespace Generated.Tail { }\n`; const root = await makeTempRepo({ - 'Shallow.cs': 'namespace Shallow.Ns;', - 'Huge.cs': oversized, + 'Hand.cs': 'namespace Hand.Written;', + 'Generated.cs': huge, }); try { const config = await loadCsharpResolutionConfig(root); const ns = config.namespaces!; - expect(ns.truncated).toBe(true); - expect(ns.declaredNamespaces!.has('Shallow.Ns')).toBe(true); - expect(ns.declaredNamespaces!.has('Deep.Ns')).toBe(false); + expect(ns.truncated).toBe(false); + expect(ns.declaredNamespaces!.has('Hand.Written')).toBe(true); + expect(ns.declaredNamespaces!.has('Generated.Head')).toBe(true); + expect(ns.declaredNamespaces!.has('Generated.Tail')).toBe(true); } finally { await fsp.rm(root, { recursive: true, force: true }); }