diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 74fb9623c..c949b2f44 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -311,7 +311,10 @@ const FILESYSTEM = [ // The deletion-guard cases in this file require real Windows path semantics. 'test/unit/canonicalize-path-long-path-prefix.test.ts', 'test/unit/storage-resolver.test.ts', + // Dart package metadata is captured during the shared scan on every platform. 'test/unit/dart-package-imports.test.ts', + // Verify the same capture is reused by native parsing and scope resolution. + 'test/integration/resolvers/dart.test.ts', // Cargo membership uses path normalization, descriptor validation, symlinks, // and Rust native parsing (including long Windows source strings). 'test/unit/scope-resolution/rust-cargo-targets.test.ts', diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index a4a04e342..bfca597b3 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -1,6 +1,7 @@ import { isVerboseIngestionEnabled } from './utils/verbose.js'; import { DEFAULT_MAX_FILE_SIZE_BYTES, getMaxFileSizeBytes } from './utils/max-file-size.js'; import fs from 'fs/promises'; +import { readdir } from 'node:fs'; import path from 'path'; import { glob } from 'glob'; import { createIgnoreFilter } from '../../config/ignore-service.js'; @@ -58,6 +59,8 @@ const warnLargeFileSkip = (message: string): void => { }; export interface WalkRepositoryOptions { + /** Capture required metadata before stat/size filtering can omit its inputs. */ + onPathsDiscovered?: (paths: readonly string[]) => Promise; /** * Suppress the operator-facing large-file notice. Set by read-only callers * such as `status`, which reuse this scan purely to learn which files the @@ -73,7 +76,7 @@ export interface WalkRepositoryOptions { } /** - * Phase 1: Scan repository — stat files to get paths + sizes, no content loaded. + * Phase 1: Scan repository — capture required metadata, then stat paths + sizes. * Memory: ~10MB for 100K files vs ~1GB+ with content. */ const assertWalkRootIsDirectory = async (repoPath: string): Promise => { @@ -100,13 +103,31 @@ export const walkRepositoryPaths = async ( await assertWalkRootIsDirectory(repoPath); const ignoreFilter = await createIgnoreFilter(repoPath); const maxFileSizeBytes = options.maxFileSizeBytes ?? getMaxFileSizeBytes(); + let enumerationError: NodeJS.ErrnoException | undefined; - const filtered = await glob('**/*', { - cwd: repoPath, - nodir: true, - dot: false, - ignore: ignoreFilter, - }); + const filtered = ( + await glob('**/*', { + cwd: repoPath, + nodir: true, + dot: false, + ignore: ignoreFilter, + // glob treats unreadable directories as empty. Required metadata must + // not silently lose declarations, so retain the first enumeration error. + fs: + options.onPathsDiscovered === undefined + ? undefined + : { + readdir(directory, readOptions, callback) { + readdir(directory, readOptions, (error, entries) => { + if (error) enumerationError ??= error; + callback(error, entries); + }); + }, + }, + }) + ).map((filePath) => filePath.replace(/\\/g, '/')); + if (enumerationError !== undefined) throw enumerationError; + await options.onPathsDiscovered?.(filtered); const entries: ScannedFile[] = []; let processed = 0; let skippedLarge = 0; @@ -120,10 +141,10 @@ export const walkRepositoryPaths = async ( const stat = await fs.stat(fullPath); if (stat.size > maxFileSizeBytes) { skippedLarge++; - skippedLargePaths.push(relativePath.replace(/\\/g, '/')); + skippedLargePaths.push(relativePath); return null; } - return { path: relativePath.replace(/\\/g, '/'), size: stat.size }; + return { path: relativePath, size: stat.size }; }), ); diff --git a/gitnexus/src/core/ingestion/languages/dart/package-config.ts b/gitnexus/src/core/ingestion/languages/dart/package-config.ts index 2040d6d94..f374a34eb 100644 --- a/gitnexus/src/core/ingestion/languages/dart/package-config.ts +++ b/gitnexus/src/core/ingestion/languages/dart/package-config.ts @@ -1,9 +1,10 @@ -import type { BigIntStats, Dirent } from 'node:fs'; -import { constants, lstat, open, opendir, type FileHandle } from 'node:fs/promises'; +import type { BigIntStats } from 'node:fs'; +import { constants, lstat, open, type FileHandle } from 'node:fs/promises'; import path from 'node:path'; import { JSON_SCHEMA, load } from 'js-yaml'; -import { createWatchIgnorePredicate } from '../../../../config/ignore-service.js'; +import { loadIgnoreRules } from '../../../../config/ignore-service.js'; import { logger } from '../../../logger.js'; +import { walkRepositoryPaths } from '../../filesystem-walker.js'; import { getMaxFileSizeBytes } from '../../utils/max-file-size.js'; export interface DartPackageConfig { @@ -11,60 +12,60 @@ export interface DartPackageConfig { readonly manifestsByName: ReadonlyMap; } -/** An incomplete walk cannot prove package names are unique. */ -const DART_PUBSPEC_DIRECTORY_LIMIT = 20_000; - -/** - * Directory descriptors one walk may hold at once. The visit budget is far - * above a process file-descriptor limit, so a deep chain is refused before - * the next open instead of failing later with EMFILE. - */ -const DART_PUBSPEC_OPEN_DIRECTORY_LIMIT = 64; - -/** - * Names read from one directory. `readdir` would retain every entry before the - * visit budget can run, so the walk counts names as it reads and stops there. - */ -const DART_PUBSPEC_DIRECTORY_ENTRY_LIMIT = 100_000; +const DART_PUBSPEC_MANIFEST_LIMIT = 20_000; export interface DartPackageConfigOptions { - /** Test seam. Production calls omit it and use the module directory limit. */ - readonly directoryLimit?: number; - /** Test seam. Production calls omit it and use the open-directory cap. */ - readonly directoryDepthLimit?: number; - /** Test seam. Production calls omit it and use the per-directory entry cap. */ - readonly directoryEntryLimit?: number; - /** - * Test seam. Production calls omit it. Invoked after the directory inode - * is listed and before its entries are opened. - */ - readonly beforeEntryOpen?: (relativePath: string) => void | Promise; - /** - * Test seam. Production calls omit it. Invoked after the directory is - * opened and before it is listed. - */ - readonly beforeDirectoryList?: (relativePath: string) => void | Promise; + /** Test seam. Production calls use the manifest-count limit. */ + readonly manifestLimit?: number; +} + +function warn(reason: string, relativePath: string): void { + logger.warn( + { reason, relativePath }, + 'Dart pubspec discovery could not read a valid package declaration.', + ); +} + +function incomplete(reason: string, relativePath = '.'): never { + warn(reason, relativePath); + throw new Error(`Dart pubspec discovery failed (${reason}): ${relativePath}`); +} + +function manifestIdentity(info: BigIntStats): string { + return `${info.dev}:${info.ino}:${info.mode}:${info.size}:${info.mtimeNs}:${info.ctimeNs}`; } type ManifestRead = | { readonly ok: true; readonly content: string } | { readonly ok: false; readonly reason: 'manifest-size' | 'read-pubspec' }; -/** - * Read at most `maxManifestSize` bytes from a descriptor already opened - * with `O_NOFOLLOW`. A size check after the read rejects a file that grew - * past the captured bytes, including past the cap. The caller closes nothing; - * this function owns `handle`. - */ +/** Open once, validate and read that descriptor. Null excludes a symbolic link. */ async function readManifestBounded( - handle: FileHandle, + absolute: string, maxManifestSize: number, -): Promise { +): Promise { + let handle: FileHandle; try { - const info = await handle.stat(); - if (!info.isFile()) return { ok: false, reason: 'read-pubspec' }; - if (info.size > maxManifestSize) return { ok: false, reason: 'manifest-size' }; - const toRead = Math.min(maxManifestSize + 1, info.size + 1); + handle = await open( + absolute, + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0), + ); + } catch { + // O_NOFOLLOW rejects leaf symlinks on POSIX. Keep static links excluded, + // including dangling links, without retrying the pathname open. + if ((await lstat(absolute).catch(() => null))?.isSymbolicLink()) return null; + return { ok: false, reason: 'read-pubspec' }; + } + try { + const before = await handle.stat({ bigint: true }); + const current = await lstat(absolute, { bigint: true }); + // Also exclude links before reading on platforms without O_NOFOLLOW. + if (current.isSymbolicLink()) return null; + if (!before.isFile() || manifestIdentity(before) !== manifestIdentity(current)) { + return { ok: false, reason: 'read-pubspec' }; + } + if (before.size > BigInt(maxManifestSize)) return { ok: false, reason: 'manifest-size' }; + const toRead = Math.min(maxManifestSize + 1, Number(before.size) + 1); const buffer = Buffer.allocUnsafe(toRead); let bytesRead = 0; while (bytesRead < toRead) { @@ -73,10 +74,11 @@ async function readManifestBounded( bytesRead += chunk.bytesRead; } if (bytesRead > maxManifestSize) return { ok: false, reason: 'manifest-size' }; - const after = await handle.stat(); - if (!after.isFile()) return { ok: false, reason: 'read-pubspec' }; - if (after.size > maxManifestSize) return { ok: false, reason: 'manifest-size' }; - if (after.size !== bytesRead) return { ok: false, reason: 'read-pubspec' }; + const after = await handle.stat({ bigint: true }); + if (after.size > BigInt(maxManifestSize)) return { ok: false, reason: 'manifest-size' }; + if (manifestIdentity(after) !== manifestIdentity(before) || after.size !== BigInt(bytesRead)) { + return { ok: false, reason: 'read-pubspec' }; + } return { ok: true, content: buffer.subarray(0, bytesRead).toString('utf8') }; } catch { return { ok: false, reason: 'read-pubspec' }; @@ -85,409 +87,122 @@ async function readManifestBounded( } } -function requireNoFollowFlag(): number { - const noFollow = constants.O_NOFOLLOW; - if (typeof noFollow !== 'number' || noFollow === 0) { - throw Object.assign(new Error('O_NOFOLLOW is unavailable'), { code: 'ENOTSUP' }); - } - return noFollow; -} - -/** Linux anchors child opens. macOS verifies them. Every other platform does neither. */ -export function pubspecWalkAnchored(): boolean { - const noFollow = constants.O_NOFOLLOW; - return ( - (process.platform === 'linux' || process.platform === 'darwin') && - typeof noFollow === 'number' && - noFollow !== 0 - ); -} - -export function directoryOpenFlags(): number { - let flags = constants.O_RDONLY | requireNoFollowFlag(); - if (typeof constants.O_DIRECTORY === 'number') flags |= constants.O_DIRECTORY; - return flags; -} - -/** - * Read-only, no-follow, and non-blocking. `O_NONBLOCK` does not change a - * regular-file read. Without it, a FIFO blocks inside `open` until a writer - * connects, so the later file-type check never runs. - */ -function fileOpenFlags(): number { - const nonBlock = constants.O_NONBLOCK; - if (typeof nonBlock !== 'number' || nonBlock === 0) { - throw Object.assign(new Error('O_NONBLOCK is unavailable'), { code: 'ENOTSUP' }); - } - return constants.O_RDONLY | requireNoFollowFlag() | nonBlock; -} - -function directoryIdentity(stat: BigIntStats): string { - return `${stat.dev}:${stat.ino}:${stat.mode}`; -} - -/** - * Path that lists the directory inode already open on `fd`. - * Only Linux has one: `/proc/self/fd/N`. macOS refuses `opendir` on - * `/dev/fd/N` for a directory (ENOTDIR) and Node has no `fdopendir`, so the - * walker lists the lexical path and verifies the pinned chain around it. - * Every other platform returns null and is not walked. - */ -export function descriptorDirectoryPath(fd: number): string | null { - if (process.platform === 'linux') return `/proc/self/fd/${fd}`; - return null; -} - -/** - * One entry of the directory inode open on `fd`. - * Linux looks up `/proc/self/fd/N/` in that inode. macOS `/dev/fd/N/` - * does not resolve a child, so this returns null and the walker verifies the - * pinned parent chain instead. Every other platform returns null and is not walked. - */ -export function descriptorEntryPath(fd: number, name: string): string | null { - if (process.platform !== 'linux') return null; - if (!isSingleDirectoryEntry(name)) return null; - return `/proc/self/fd/${fd}/${name}`; -} - -function isSingleDirectoryEntry(name: string): boolean { - return ( - name !== '' && name !== '.' && name !== '..' && !name.includes('/') && !name.includes('\0') - ); -} - -interface OpenedDirectory { - readonly handle: FileHandle; - readonly identity: string; -} - -async function openVerifiedDirectory(directory: string): Promise { - const handle = await open(directory, directoryOpenFlags()); +/** Package identity is a pure function of captured manifest text. */ +function packageName(content: string, manifestPath: string): string | null { try { - const info = await handle.stat({ bigint: true }); - if (!info.isDirectory()) { - throw Object.assign(new Error('not a directory'), { code: 'ENOTDIR' }); - } - return { handle, identity: directoryIdentity(info) }; - } catch (error) { - await handle.close(); - throw error; + const manifest: unknown = load(content, { schema: JSON_SCHEMA }); + if (manifest === null || typeof manifest !== 'object' || Array.isArray(manifest)) return null; + const name = (manifest as Record).name; + return typeof name === 'string' && /^[a-z_][a-z0-9_]*$/.test(name) ? name : null; + } catch { + warn('invalid-yaml', manifestPath); + return null; } } -interface WalkFrame { - relative: string; - absolute: string; - handle: FileHandle; - identity: string; - entries: Dirent[]; - next: number; -} - -/** Re-stat each pinned directory and its path. A replaced inode or a symlink fails the walk. */ -async function assertPinnedChain(frames: readonly WalkFrame[]): Promise { - for (const frame of frames) { - const pinned = await frame.handle.stat({ bigint: true }); - if (!pinned.isDirectory() || directoryIdentity(pinned) !== frame.identity) { - throw Object.assign(new Error('parent descriptor changed'), { code: 'ELOOP' }); - } - let lexical: BigIntStats; - try { - lexical = await lstat(frame.absolute, { bigint: true }); - } catch { - throw Object.assign(new Error('parent path changed'), { code: 'ELOOP' }); - } +/** + * Capture declarations from the shared scan, before source stat/size filtering. + * No directory enumeration occurs here. Every enumerated regular manifest must + * be captured successfully before this config can be published. + * + * Acquisition assumes a stable, trusted workspace, like the source-file reader. + * Static symlinks/junctions are excluded and observed file changes are rejected; + * these checks do not provide a sandbox or an atomic view of a mutating tree. + * Once returned, resolution uses only this compact map, on every operating system. + */ +export async function captureDartPackageConfig( + repoPath: string, + filePaths: readonly string[], + options?: DartPackageConfigOptions, +): Promise { + const manifests = new Set(); + const limit = options?.manifestLimit ?? DART_PUBSPEC_MANIFEST_LIMIT; + for (const filePath of filePaths) { + if (filePath !== 'pubspec.yaml' && !filePath.endsWith('/pubspec.yaml')) continue; if ( - lexical.isSymbolicLink() || - !lexical.isDirectory() || - directoryIdentity(lexical) !== frame.identity + /[\\\0]/.test(filePath) || + /^[a-zA-Z]:/.test(filePath) || + filePath.split('/').some((part) => part === '' || part === '.' || part === '..') ) { - throw Object.assign(new Error('parent path changed'), { code: 'ELOOP' }); + return incomplete('manifest-path'); } + manifests.add(filePath); + if (manifests.size > limit) return incomplete('manifest-limit'); } -} - -/** - * macOS has no descriptor-relative child lookup. Re-check the pinned parents, - * open the child with O_NOFOLLOW, then re-check the parents. The opened - * descriptor is the only child metadata consulted. - */ -async function openLexicalDirectory( - frames: readonly WalkFrame[], - lexicalPath: string, -): Promise { - await assertPinnedChain(frames); - const opened = await openVerifiedDirectory(lexicalPath); try { - await assertPinnedChain(frames); - return opened; - } catch (error) { - await opened.handle.close(); - throw error; + await loadIgnoreRules(repoPath, { strictRepoControlFiles: true, noGlobalIgnore: true }); + } catch { + return incomplete('scan-inputs'); } -} + const packages = new Map(); + const manifestsByName = new Map(); + if (manifests.size === 0) return { packages, manifestsByName }; -async function openLexicalFile( - frames: readonly WalkFrame[], - lexicalPath: string, -): Promise { - await assertPinnedChain(frames); - const handle = await open(lexicalPath, fileOpenFlags()); - try { - const opened = await handle.stat({ bigint: true }); - if (!opened.isFile()) { - throw Object.assign(new Error('not a file'), { code: 'ELOOP' }); - } - await assertPinnedChain(frames); - return handle; - } catch (error) { - await handle.close(); - throw error; + // Cache static parent checks once per directory, not once per import/file. + // This cache belongs only to this capture and never survives another analysis. + const directories = new Map(); + const root = await lstat(repoPath).catch(() => null); + if (root === null || !root.isDirectory() || root.isSymbolicLink()) { + return incomplete('read-directory'); } -} - -async function openChildDirectory( - frames: readonly WalkFrame[], - parentFd: number, - name: string, - lexicalPath: string, -): Promise { - const anchored = descriptorEntryPath(parentFd, name); - if (anchored !== null) return openVerifiedDirectory(anchored); - if (process.platform === 'darwin') return openLexicalDirectory(frames, lexicalPath); - throw Object.assign(new Error('no descriptor anchor'), { code: 'ENOTSUP' }); -} - -async function openChildFile( - frames: readonly WalkFrame[], - parentFd: number, - name: string, - lexicalPath: string, -): Promise { - const anchored = descriptorEntryPath(parentFd, name); - if (anchored !== null) return open(anchored, fileOpenFlags()); - if (process.platform === 'darwin') return openLexicalFile(frames, lexicalPath); - throw Object.assign(new Error('no descriptor anchor'), { code: 'ENOTSUP' }); -} - -/** - * List the directory open on the last frame of `frames`. Linux lists the - * pinned inode through its descriptor. macOS re-checks the pinned chain, lists - * the lexical path, then re-checks the chain, so a path replaced around the - * listing fails the walk instead of being listed. - */ -async function listOpenedDirectory( - frames: readonly WalkFrame[], - entryLimit: number, - beforeList?: () => void | Promise, -): Promise { - const frame = frames[frames.length - 1]; - if (frame === undefined) { - throw Object.assign(new Error('no directory to list'), { code: 'EINVAL' }); - } - const anchored = descriptorDirectoryPath(frame.handle.fd); - if (anchored === null && process.platform !== 'darwin') { - throw Object.assign(new Error('no descriptor listing'), { code: 'ENOTSUP' }); - } - if (anchored === null) await assertPinnedChain(frames); - if (beforeList) await beforeList(); - const entries = await readDirectoryBounded(anchored ?? frame.absolute, entryLimit); - if (anchored === null) await assertPinnedChain(frames); - return entries; -} - -async function readDirectoryBounded(listing: string, entryLimit: number): Promise { - const dir = await opendir(listing); - const entries: Dirent[] = []; - try { - let count = 0; - while (true) { - const entry = await dir.read(); - if (entry === null) break; - count += 1; - if (count > entryLimit) { - throw Object.assign(new Error('directory entry limit'), { code: 'E2BIG' }); + const maxManifestSize = Math.min(1024 * 1024, getMaxFileSizeBytes()); + for (const manifestPath of manifests) { + const parts = manifestPath.split('/'); + let parent = ''; + let linked = false; + for (const part of parts.slice(0, -1)) { + parent = parent ? `${parent}/${part}` : part; + let accepted = directories.get(parent); + if (accepted === undefined) { + const info = await lstat(path.join(repoPath, parent)).catch(() => null); + if (info === null || (!info.isDirectory() && !info.isSymbolicLink())) { + return incomplete('read-directory', parent); + } + accepted = !info.isSymbolicLink(); + directories.set(parent, accepted); + } + if (!accepted) { + linked = true; + break; } - entries.push(entry); } - } finally { - await dir.close().catch(() => undefined); + if (linked) continue; + + const read = await readManifestBounded(path.join(repoPath, manifestPath), maxManifestSize); + if (read === null) continue; + if (read.ok === false) return incomplete(read.reason, manifestPath); + const name = packageName(read.content, manifestPath); + if (name === null) continue; + + const witnesses = manifestsByName.get(name) ?? []; + witnesses.push(manifestPath); + witnesses.sort(); + // Two deterministic witnesses prove ambiguity without duplicate fanout. + if (witnesses.length > 2) witnesses.length = 2; + manifestsByName.set(name, witnesses); + if (witnesses.length > 1) packages.delete(name); + else packages.set(name, parent ? `${parent}/lib` : 'lib'); } - return entries; + return { packages, manifestsByName }; } -/** - * Open `directory` without following a final symlink, then list that inode. - * A path swapped for a symlink after the parent listing fails this open - * (`ENOTDIR` / `ELOOP`) instead of being traversed. - */ -export async function readDirectoryNoFollow(directory: string): Promise { - const opened = await openVerifiedDirectory(directory); - const frame: WalkFrame = { - relative: '', - absolute: directory, - handle: opened.handle, - identity: opened.identity, - entries: [], - next: 0, - }; - try { - return await listOpenedDirectory([frame], DART_PUBSPEC_DIRECTORY_ENTRY_LIMIT); - } finally { - await opened.handle.close(); - } -} - -/** Discover only in-repository packages; never follow dependency paths or symlinks. */ +/** Standalone callers use the same scanner/capture boundary as the pipeline. */ export async function loadDartPackageConfig( repoPath: string, options?: DartPackageConfigOptions, ): Promise { - const warn = (reason: string, relativePath = '.'): void => { - logger.warn( - { reason, relativePath }, - 'Dart pubspec discovery could not read a valid package declaration.', - ); - }; - const incomplete = (reason: string, relativePath = '.'): never => { - warn(reason, relativePath); - throw new Error(`Dart pubspec discovery failed (${reason}): ${relativePath}`); - }; - if (!pubspecWalkAnchored()) { - warn('nofollow-anchor'); - return { packages: new Map(), manifestsByName: new Map() }; - } - let isIgnored; + let captured: DartPackageConfig | undefined; try { - isIgnored = await createWatchIgnorePredicate(repoPath); - } catch { - return incomplete('ignore-rules'); + await walkRepositoryPaths(repoPath, undefined, { + onPathsDiscovered: async (paths) => { + captured = await captureDartPackageConfig(repoPath, paths, options); + }, + }); + } catch (error) { + if (error instanceof Error && error.message.startsWith('Dart pubspec discovery failed')) { + throw error; + } + return incomplete('scan-inputs'); } - const packages = new Map(); - const manifestsByName = new Map(); - const maxManifestSize = Math.min(1024 * 1024, getMaxFileSizeBytes()); - const directoryLimit = options?.directoryLimit ?? DART_PUBSPEC_DIRECTORY_LIMIT; - const directoryDepthLimit = options?.directoryDepthLimit ?? DART_PUBSPEC_OPEN_DIRECTORY_LIMIT; - const directoryEntryLimit = options?.directoryEntryLimit ?? DART_PUBSPEC_DIRECTORY_ENTRY_LIMIT; - const ambiguous = new Set(); - const stack: WalkFrame[] = []; - let visited = 0; - - const closeStack = async (): Promise => { - const frames = stack.splice(0); - await Promise.all(frames.map((frame) => frame.handle.close().catch(() => undefined))); - }; - - const fillFrame = async (frame: WalkFrame): Promise => { - if (++visited > directoryLimit) return incomplete('directory-limit', frame.relative || '.'); - try { - const beforeList = options?.beforeDirectoryList; - frame.entries = await listOpenedDirectory( - stack, - directoryEntryLimit, - beforeList ? () => beforeList(frame.relative) : undefined, - ); - } catch (error) { - const reason = - (error as NodeJS.ErrnoException).code === 'E2BIG' ? 'directory-entries' : 'read-directory'; - return incomplete(reason, frame.relative || '.'); - } - if (options?.beforeEntryOpen) await options.beforeEntryOpen(frame.relative); - }; - - try { - let rootOpened: OpenedDirectory; - try { - rootOpened = await openVerifiedDirectory(repoPath); - } catch { - return incomplete('read-directory', '.'); - } - const root: WalkFrame = { - relative: '', - absolute: repoPath, - handle: rootOpened.handle, - identity: rootOpened.identity, - entries: [], - next: 0, - }; - stack.push(root); - await fillFrame(root); - - while (stack.length > 0) { - const frame = stack[stack.length - 1]; - if (frame === undefined) break; - if (frame.next >= frame.entries.length) { - stack.pop(); - await frame.handle.close().catch(() => undefined); - continue; - } - const entry = frame.entries[frame.next]; - frame.next += 1; - if (entry === undefined || !isSingleDirectoryEntry(entry.name) || entry.isSymbolicLink()) { - continue; - } - const childRelative = frame.relative ? `${frame.relative}/${entry.name}` : entry.name; - const entryPath = path.join(repoPath, childRelative); - if (entry.isDirectory()) { - if (entry.name.startsWith('.') || isIgnored(entryPath, true)) continue; - if (stack.length >= directoryDepthLimit) { - return incomplete('directory-depth', childRelative); - } - let childOpened: OpenedDirectory; - try { - childOpened = await openChildDirectory(stack, frame.handle.fd, entry.name, entryPath); - } catch { - return incomplete('read-directory', childRelative); - } - const child: WalkFrame = { - relative: childRelative, - absolute: entryPath, - handle: childOpened.handle, - identity: childOpened.identity, - entries: [], - next: 0, - }; - stack.push(child); - await fillFrame(child); - } else if (entry.isFile() && entry.name === 'pubspec.yaml' && !isIgnored(entryPath, false)) { - const manifestPath = frame.relative ? `${frame.relative}/pubspec.yaml` : 'pubspec.yaml'; - let manifestHandle: FileHandle; - try { - manifestHandle = await openChildFile(stack, frame.handle.fd, entry.name, entryPath); - } catch { - return incomplete('read-pubspec', manifestPath); - } - const read = await readManifestBounded(manifestHandle, maxManifestSize); - if (read.ok === false) return incomplete(read.reason, manifestPath); - try { - const manifest: unknown = load(read.content, { - schema: JSON_SCHEMA, - }); - if (manifest === null || typeof manifest !== 'object' || Array.isArray(manifest)) - continue; - const name = (manifest as Record).name; - if (typeof name !== 'string' || !/^[a-z_][a-z0-9_]*$/.test(name)) continue; - const manifests = manifestsByName.get(name) ?? []; - manifests.push(manifestPath); - // Two deterministic witnesses suffice to prove ambiguity. Retaining - // more would create unbounded duplicate-package dependency fanout. - manifests.sort(); - if (manifests.length > 2) manifests.length = 2; - manifestsByName.set(name, manifests); - if (packages.has(name) || ambiguous.has(name)) { - packages.delete(name); - ambiguous.add(name); - } else { - packages.set(name, frame.relative ? `${frame.relative}/lib` : 'lib'); - } - } catch { - // Invalid YAML cannot declare a package. Other valid packages remain usable. - warn('invalid-yaml', manifestPath); - } - } - } - } finally { - await closeStack(); - } - return { packages, manifestsByName }; + return captured ?? incomplete('scan-inputs'); } diff --git a/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts index 84a1ad8bf..bc286179e 100644 --- a/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts @@ -42,7 +42,7 @@ import { typeApplicationArguments } from '../../utils/template-arguments.js'; import type { HeritageTypeArgumentSink } from '../../scope-resolution/utils/generic-instantiation.js'; import { expandDartWildcardNames } from './expand-wildcards.js'; import { dartIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; -import { loadDartPackageConfig } from './package-config.js'; +import { captureDartPackageConfig, loadDartPackageConfig } from './package-config.js'; import { emitDartPackageDependencies } from './package-dependencies.js'; interface ClassDefRef { @@ -202,6 +202,7 @@ export const dartScopeResolver: ScopeResolver = { importEdgeReason: 'dart-scope: import', loadResolutionConfig: loadDartPackageConfig, + captureResolutionConfig: captureDartPackageConfig, // Package dependencies use cached ParsedFile facts, never raw source text. postExtractSourceTextPolicy: 'uncached-files', emitPostResolutionEdges: (graph, parsedFiles, _nodeLookup, _indexes, ctx) => diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index f745ea16a..600ee3b91 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -451,6 +451,7 @@ export async function runChunkedParseAndResolve( pipelineStart: number, onProgress: ProgressFn, options?: PipelineOptions, + capturedResolutionConfigs?: ReadonlyMap, ): Promise<{ exportedTypeMap: ExportedTypeMap; allFetchCalls: ExtractedFetchCall[]; @@ -1853,9 +1854,11 @@ export async function runChunkedParseAndResolve( const resolver = SCOPE_RESOLVERS.get(language); routeResolutionConfigs.set( language, - resolver?.loadResolutionConfig === undefined - ? undefined - : await resolver.loadResolutionConfig(repoPath), + capturedResolutionConfigs?.has(language) + ? capturedResolutionConfigs.get(language) + : resolver?.loadResolutionConfig === undefined + ? undefined + : await resolver.loadResolutionConfig(repoPath), ); } let routeResolutionFiles = allParsedFiles; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index 503f3be75..29c62777b 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -125,10 +125,8 @@ export const parsePhase: PipelinePhase = { // nothing before parse produces bulk edge volume anyway. ctx.graphEmit?.beginStreaming(); - const { scannedFiles, allPaths, allPathSet, totalFiles } = getPhaseOutput( - deps, - 'structure', - ); + const { scannedFiles, allPaths, allPathSet, totalFiles, resolutionConfigs } = + getPhaseOutput(deps, 'structure'); const result = await runChunkedParseAndResolve( ctx.graph, @@ -139,6 +137,7 @@ export const parsePhase: PipelinePhase = { ctx.pipelineStart, ctx.onProgress, ctx.options, + resolutionConfigs, ); return { diff --git a/gitnexus/src/core/ingestion/pipeline-phases/scan.ts b/gitnexus/src/core/ingestion/pipeline-phases/scan.ts index f41c4f84e..289ad8300 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/scan.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/scan.ts @@ -2,7 +2,7 @@ * Phase: scan * * Walks the repository filesystem and collects file paths + sizes. - * Does NOT read file contents — that happens in downstream phases. + * Captures required provider metadata; source contents are read downstream. * * @deps (none — this is the pipeline root) * @reads repoPath (filesystem) @@ -14,11 +14,14 @@ import type { PipelinePhase, PipelineContext } from './types.js'; import { walkRepositoryPaths } from '../filesystem-walker.js'; import fs from 'node:fs/promises'; import path from 'node:path'; +import { getLanguageFromFilename, type SupportedLanguages } from 'gitnexus-shared'; +import { SCOPE_RESOLVERS } from '../scope-resolution/pipeline/registry.js'; export interface ScanOutput { scannedFiles: { path: string; size: number }[]; allPaths: string[]; totalFiles: number; + resolutionConfigs?: ReadonlyMap; } const SPRING_ACTUATOR_ENDPOINT_FILES = new Set([ @@ -136,26 +139,47 @@ export const scanPhase: PipelinePhase = { ...(ctx.options?.springActuatorScanExclusions ?? []), ]); let scannedFiles; + const resolutionConfigs = new Map(); try { - scannedFiles = await walkRepositoryPaths(ctx.repoPath, (current, total, filePath) => { - const scanProgress = Math.round((current / total) * 15); - const isRuntimeInput = matchesActuatorExclusion( - canonicalRepoPath, - filePath, - actuatorExclusions, - ); - ctx.onProgress({ - phase: 'extracting', - percent: scanProgress, - message: 'Scanning repository...', - ...(isRuntimeInput ? {} : { detail: filePath }), - stats: { - filesProcessed: current, - totalFiles: total, - nodesCreated: ctx.graph.nodeCount, + scannedFiles = await walkRepositoryPaths( + ctx.repoPath, + (current, total, filePath) => { + const scanProgress = Math.round((current / total) * 15); + const isRuntimeInput = matchesActuatorExclusion( + canonicalRepoPath, + filePath, + actuatorExclusions, + ); + ctx.onProgress({ + phase: 'extracting', + percent: scanProgress, + message: 'Scanning repository...', + ...(isRuntimeInput ? {} : { detail: filePath }), + stats: { + filesProcessed: current, + totalFiles: total, + nodesCreated: ctx.graph.nodeCount, + }, + }); + }, + { + onPathsDiscovered: async (paths) => { + const inputs = paths.filter( + (filePath) => + !matchesActuatorExclusion(canonicalRepoPath, filePath, actuatorExclusions), + ); + const languages = new Set(inputs.map(getLanguageFromFilename)); + for (const [language, provider] of SCOPE_RESOLVERS) { + if (languages.has(language) && provider.captureResolutionConfig !== undefined) { + resolutionConfigs.set( + language, + await provider.captureResolutionConfig(ctx.repoPath, inputs), + ); + } + } }, - }); - }); + }, + ); } catch (err) { // Missing roots throw so status cannot treat an empty glob as "every // covered file was deleted". The pipeline still reports an empty scan @@ -182,6 +206,6 @@ export const scanPhase: PipelinePhase = { stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: ctx.graph.nodeCount }, }); - return { scannedFiles, allPaths, totalFiles }; + return { scannedFiles, allPaths, totalFiles, resolutionConfigs }; }, }; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/structure.ts b/gitnexus/src/core/ingestion/pipeline-phases/structure.ts index 35ca10c1e..11bb6d139 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/structure.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/structure.ts @@ -12,6 +12,7 @@ import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js'; import { getPhaseOutput } from './types.js'; import { processStructure } from '../structure-processor.js'; import type { ScanOutput } from './scan.js'; +import type { SupportedLanguages } from 'gitnexus-shared'; /** Structure phase produces no additional data — it writes directly to the graph. */ export interface StructureOutput { @@ -25,6 +26,7 @@ export interface StructureOutput { */ allPathSet: ReadonlySet; totalFiles: number; + resolutionConfigs?: ReadonlyMap; } export const structurePhase: PipelinePhase = { @@ -35,7 +37,10 @@ export const structurePhase: PipelinePhase = { ctx: PipelineContext, deps: ReadonlyMap>, ): Promise { - const { scannedFiles, allPaths, totalFiles } = getPhaseOutput(deps, 'scan'); + const { scannedFiles, allPaths, totalFiles, resolutionConfigs } = getPhaseOutput( + deps, + 'scan', + ); ctx.onProgress({ phase: 'structure', @@ -57,6 +62,6 @@ export const structurePhase: PipelinePhase = { // can all reuse it instead of re-materializing `new Set(allPaths)` each. const allPathSet: ReadonlySet = new Set(allPaths); - return { scannedFiles, allPaths, allPathSet, totalFiles }; + return { scannedFiles, allPaths, allPathSet, totalFiles, resolutionConfigs }; }, }; diff --git a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts index 22a7a2d41..749665b73 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/contract/scope-resolver.ts @@ -471,6 +471,19 @@ export interface ScopeResolver { */ loadResolutionConfig?(repoPath: string): Promise | unknown; + /** + * Capture metadata from the scan's complete, nonignored candidate paths, + * before unreadable/large source files are filtered out. Called once for a + * language present in the scan. The compact result is shared by every + * resolution pass instead of calling loadResolutionConfig again. + * Capture failures must throw; consumers must not mutate the result. + * This is a stable-workspace input boundary, not an atomic filesystem snapshot. + */ + captureResolutionConfig?( + repoPath: string, + filePaths: readonly string[], + ): Promise | unknown; + /** * Per-scope binding-merge precedence. The shared finalize pass * collects bindings from multiple sources (local declarations, diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts index 5602dcb8d..f7afa4fde 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts @@ -176,7 +176,7 @@ export const scopeResolutionPhase: PipelinePhase = { deps: ReadonlyMap>, ): Promise { logHeapProbe('scopeResolution-enter'); - const { scannedFiles } = getPhaseOutput(deps, 'structure'); + const { scannedFiles, resolutionConfigs } = getPhaseOutput(deps, 'structure'); const parseOutput = getPhaseOutput(deps, 'parse'); const { model, parsedFiles: workerParsedFiles, contentLanguageByPath } = parseOutput; const scopeExtractionFailures = new Set(parseOutput.scopeExtractionFailures); @@ -382,8 +382,9 @@ export const scopeResolutionPhase: PipelinePhase = { // composer.json autoload, go.mod, ...). One I/O round trip per // workspace pass — cached implicitly by the result handed to // every `resolveImportTarget` call below. - const resolutionConfig = - provider.loadResolutionConfig !== undefined + const resolutionConfig = resolutionConfigs?.has(lang) + ? resolutionConfigs.get(lang) + : provider.loadResolutionConfig !== undefined ? await provider.loadResolutionConfig(ctx.repoPath) : undefined; diff --git a/gitnexus/test/integration/resolvers/dart.test.ts b/gitnexus/test/integration/resolvers/dart.test.ts index 2f7b00c04..9f36a33d0 100644 --- a/gitnexus/test/integration/resolvers/dart.test.ts +++ b/gitnexus/test/integration/resolvers/dart.test.ts @@ -6,7 +6,7 @@ * All Dart pipeline features are covered: Property nodes, HAS_PROPERTY edges, * CALLS chain resolution, IMPORTS, call attribution, and ACCESSES field reads. */ -import { describe, it, expect, beforeAll } from 'vitest'; +import { describe, it, expect, beforeAll, vi } from 'vitest'; import path from 'path'; import { FIXTURES, @@ -23,7 +23,7 @@ import { loadLanguage, } from '../../../src/core/tree-sitter/parser-loader.js'; import { SupportedLanguages } from '../../../src/config/supported-languages.js'; -import { pubspecWalkAnchored } from '../../../src/core/ingestion/languages/dart/package-config.js'; +import { dartScopeResolver } from '../../../src/core/ingestion/languages/dart/scope-resolver.js'; // isLanguageAvailable only checks whether the module loaded — it does NOT verify // that the native binary works at runtime (tree-sitter-dart can fail on setLanguage). @@ -38,42 +38,55 @@ if (dartAvailable) { } } -describe.skipIf(!dartAvailable || !pubspecWalkAnchored())( - 'Dart pubspec package identity (#2963)', - () => { - let result: PipelineResult; +describe.skipIf(!dartAvailable)('Dart pubspec package identity (#2963)', () => { + let result: PipelineResult; + let captureCount = 0; + let reloadCount = 0; - beforeAll(async () => { + beforeAll(async () => { + const capture = vi.spyOn(dartScopeResolver, 'captureResolutionConfig'); + const reload = vi.spyOn(dartScopeResolver, 'loadResolutionConfig'); + try { result = await runPipelineFromRepo(path.join(FIXTURES, 'dart-package-imports'), () => {}); - }, 60000); + captureCount = capture.mock.calls.length; + reloadCount = reload.mock.calls.length; + } finally { + capture.mockRestore(); + reload.mockRestore(); + } + }, 60000); - it('emits declared imports and their package identity dependencies', () => { - const imports = getRelationships(result, 'IMPORTS') - .filter((edge) => edge.sourceFilePath === 'lib/main.dart') - .map((edge) => edge.targetFilePath) - .sort(); - expect(imports).toEqual([ - 'lib/models.dart', - 'lib/relative.dart', - 'packages/data/lib/models.dart', - 'packages/data/pubspec.yaml', - 'pubspec.yaml', - ]); - }); + it('captures package metadata once and reuses it through parsing and resolution', () => { + expect(captureCount).toBe(1); + expect(reloadCount).toBe(0); + }); - it.each([ - ['loadOwn', 'lib/models.dart'], - ['loadData', 'packages/data/lib/models.dart'], - ['loadRelative', 'lib/relative.dart'], - ])('resolves %s in the correct library', (name, file) => { - const calls = getRelationships(result, 'CALLS').filter( - (edge) => edge.sourceFilePath === 'lib/main.dart' && edge.target === name, - ); - expect(calls).toHaveLength(1); - expect(calls[0]?.targetFilePath).toBe(file); - }); - }, -); + it('emits declared imports and their package identity dependencies', () => { + const imports = getRelationships(result, 'IMPORTS') + .filter((edge) => edge.sourceFilePath === 'lib/main.dart') + .map((edge) => edge.targetFilePath) + .sort(); + expect(imports).toEqual([ + 'lib/models.dart', + 'lib/relative.dart', + 'packages/data/lib/models.dart', + 'packages/data/pubspec.yaml', + 'pubspec.yaml', + ]); + }); + + it.each([ + ['loadOwn', 'lib/models.dart'], + ['loadData', 'packages/data/lib/models.dart'], + ['loadRelative', 'lib/relative.dart'], + ])('resolves %s in the correct library', (name, file) => { + const calls = getRelationships(result, 'CALLS').filter( + (edge) => edge.sourceFilePath === 'lib/main.dart' && edge.target === name, + ); + expect(calls).toHaveLength(1); + expect(calls[0]?.targetFilePath).toBe(file); + }); +}); // ── Phase 8: Field-type resolution ────────────────────────────────────── diff --git a/gitnexus/test/unit/dart-package-imports.test.ts b/gitnexus/test/unit/dart-package-imports.test.ts index c127838d1..6320c4c3c 100644 --- a/gitnexus/test/unit/dart-package-imports.test.ts +++ b/gitnexus/test/unit/dart-package-imports.test.ts @@ -1,28 +1,18 @@ import { execFileSync } from 'node:child_process'; +import nodeFs from 'node:fs'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { constants } from 'node:fs'; -import { - mkdtemp, - mkdir, - rm, - symlink, - writeFile, - chmod, - rename, - open, - readdir, -} from 'node:fs/promises'; +import fs, { mkdtemp, mkdir, rm, symlink, writeFile, chmod, rename } from 'node:fs/promises'; +import { syncBuiltinESMExports } from 'node:module'; import os from 'node:os'; import path from 'node:path'; +import { SupportedLanguages } from 'gitnexus-shared'; import { dartScopeResolver } from '../../src/core/ingestion/languages/dart/scope-resolver.js'; import { loadDartPackageConfig, - readDirectoryNoFollow, - directoryOpenFlags, - descriptorDirectoryPath, - descriptorEntryPath, - pubspecWalkAnchored, + captureDartPackageConfig, } from '../../src/core/ingestion/languages/dart/package-config.js'; +import { scanPhase } from '../../src/core/ingestion/pipeline-phases/scan.js'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; import { CountingSet } from '../helpers/counting-file-set.js'; import { _captureLogger } from '../../src/core/logger.js'; @@ -183,25 +173,7 @@ describe('Dart package identity (#2963)', () => { }); }); -describe('directory no-follow flags', () => { - it('does not drop O_NOFOLLOW from directory opens', () => { - if (typeof constants.O_NOFOLLOW !== 'number' || constants.O_NOFOLLOW === 0) { - expect(() => directoryOpenFlags()).toThrow(/O_NOFOLLOW is unavailable/); - return; - } - expect(directoryOpenFlags() & constants.O_NOFOLLOW).toBe(constants.O_NOFOLLOW); - }); - - it('anchors pubspec discovery only where a no-follow walk exists', () => { - const canAnchor = - (process.platform === 'linux' || process.platform === 'darwin') && - typeof constants.O_NOFOLLOW === 'number' && - constants.O_NOFOLLOW !== 0; - expect(pubspecWalkAnchored()).toBe(canAnchor); - }); -}); - -describe.skipIf(!pubspecWalkAnchored())('Dart pubspec package discovery', () => { +describe('Dart pubspec package discovery', () => { const roots: string[] = []; afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); @@ -378,223 +350,391 @@ describe.skipIf(!pubspecWalkAnchored())('Dart pubspec package discovery', () => ); }); - it('refuses an incomplete scan rather than persisting untracked identity changes', async () => { + it('discovers declared packages on Windows', async () => { + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + const platform = vi.spyOn(process, 'platform', 'get').mockReturnValue('win32'); + try { + expect((await loadDartPackageConfig(root)).packages).toEqual(config.packages); + } finally { + platform.mockRestore(); + } + }); + + it('fails when repository ignore rules cannot be read', async () => { + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + await mkdir(path.join(root, '.gitignore')); + await expect(loadDartPackageConfig(root)).rejects.toThrow('(scan-inputs)'); + }); + + for (const ignoreFile of ['.gitignore', '.gitnexusignore']) { + it(`applies linked ${ignoreFile} to TypeScript scans while keeping Dart capture strict`, async (context) => { + const root = await fixture({ 'main.ts': 'export {};', 'ignored.ts': 'export {};' }); + const outside = await fixture({ rules: 'ignored.ts\n' }); + try { + await symlink(path.join(outside, 'rules'), path.join(root, ignoreFile), 'file'); + } catch (error) { + if ( + process.platform === 'win32' && + ['EPERM', 'EACCES', 'ENOSYS'].includes((error as NodeJS.ErrnoException).code ?? '') + ) { + context.skip('Windows file symlinks are unavailable'); + } + throw error; + } + const scanContext = { + repoPath: root, + graph: createKnowledgeGraph(), + onProgress: () => {}, + pipelineStart: Date.now(), + }; + const scanned = await scanPhase.execute(scanContext, new Map()); + expect(scanned.allPaths).toEqual(['main.ts']); + expect(scanned.resolutionConfigs?.has(SupportedLanguages.Dart)).toBe(false); + + // Dart capture validates its scan inputs even when no pubspec was discovered. + await writeFile(path.join(root, 'main.dart'), 'void main() {}'); + await expect(scanPhase.execute(scanContext, new Map())).rejects.toThrow('(scan-inputs)'); + }); + } + + it('fails when the repository root is missing', async () => { const root = await fixture({}); await expect(loadDartPackageConfig(path.join(root, 'missing'))).rejects.toThrow( - 'Dart pubspec discovery failed (read-directory)', + '(scan-inputs)', ); }); - it('fails closed when the directory walk exceeds its budget', async () => { + it('fails metadata capture when glob cannot enumerate a nonignored directory', async () => { + const root = await fixture({ + 'pubspec.yaml': 'name: app', + 'packages/duplicate/pubspec.yaml': 'name: app', + }); + const realReaddir = nodeFs.readdir; + let denied = 0; + const spy = vi.spyOn(nodeFs, 'readdir').mockImplementation((directory, options, callback) => { + if (directory === path.join(root, 'packages')) { + denied++; + callback(Object.assign(new Error('directory denied'), { code: 'EACCES' }), []); + } else { + realReaddir(directory, options, callback); + } + }); + syncBuiltinESMExports(); + try { + await expect(loadDartPackageConfig(root)).rejects.toThrow('(scan-inputs)'); + expect(denied).toBe(1); + } finally { + spy.mockRestore(); + syncBuiltinESMExports(); + } + }); + + it('fails instead of returning a partial map when a captured candidate is missing', async () => { + const root = await fixture({ 'pubspec.yaml': 'name: app', 'nested/keep.txt': '' }); + await expect( + captureDartPackageConfig(root, ['pubspec.yaml', 'nested/pubspec.yaml']), + ).rejects.toThrow('Dart pubspec discovery failed (read-pubspec): nested/pubspec.yaml'); + }); + + it('fails closed at the manifest budget', async () => { const root = await fixture({ 'pubspec.yaml': 'name: app', 'nested/pubspec.yaml': 'name: data', }); - await expect(loadDartPackageConfig(root, { directoryLimit: 1 })).rejects.toThrow( - 'Dart pubspec discovery failed (directory-limit)', + await expect(loadDartPackageConfig(root, { manifestLimit: 1 })).rejects.toThrow( + 'Dart pubspec discovery failed (manifest-limit)', ); }); - it('fails closed before one directory listing is unbounded', async () => { - const root = await fixture({ 'pubspec.yaml': 'name: app', 'extra.txt': 'x' }); - await expect(loadDartPackageConfig(root, { directoryEntryLimit: 1 })).rejects.toThrow( - 'Dart pubspec discovery failed (directory-entries)', + it('captures a deep package without holding directory descriptors', async () => { + const manifest = `${'a/'.repeat(70)}pubspec.yaml`; + const root = await fixture({ [manifest]: 'name: data' }); + expect((await captureDartPackageConfig(root, [manifest])).packages.get('data')).toBe( + `${'a/'.repeat(70)}lib`, ); }); - it('fails closed before a deep chain holds one descriptor per level', async () => { - const root = await fixture({ 'a/b/pubspec.yaml': 'name: data' }); - await expect(loadDartPackageConfig(root, { directoryDepthLimit: 2 })).rejects.toThrow( - 'Dart pubspec discovery failed (directory-depth): a/b', - ); - }); - - it('still reads a manifest when the open-directory cap is exactly the nesting', async () => { - const root = await fixture({ 'a/pubspec.yaml': 'name: data' }); - expect((await loadDartPackageConfig(root, { directoryDepthLimit: 2 })).packages).toEqual( - new Map([['data', 'a/lib']]), - ); - }); - - it('fails closed when ignore rules cannot be read', async () => { + it('does not treat the same candidate twice as a duplicate package', async () => { const root = await fixture({ 'pubspec.yaml': 'name: app' }); - await mkdir(path.join(root, '.gitignore')); - await expect(loadDartPackageConfig(root)).rejects.toThrow( - 'Dart pubspec discovery failed (ignore-rules)', + expect( + (await captureDartPackageConfig(root, ['pubspec.yaml', 'pubspec.yaml'])).packages, + ).toEqual(config.packages); + }); + + it.each(['../pubspec.yaml', '/pubspec.yaml', 'C:/pubspec.yaml', 'a/../pubspec.yaml'])( + 'rejects a non-repository manifest path: %s', + async (candidate) => { + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + await expect(captureDartPackageConfig(root, [candidate])).rejects.toThrow('(manifest-path)'); + }, + ); + + it('rejects an enumerated nonregular manifest', async () => { + const root = await fixture({}); + await mkdir(path.join(root, 'pubspec.yaml')); + await expect(captureDartPackageConfig(root, ['pubspec.yaml'])).rejects.toThrow( + '(read-pubspec)', ); }); - // Windows does not enforce POSIX mode bits, and root bypasses them, so chmod - // cannot make this read fail on either. + it.skipIf(process.platform === 'win32')( + 'does not block on a FIFO manifest', + async () => { + const root = await fixture({}); + execFileSync('mkfifo', [path.join(root, 'pubspec.yaml')]); + await expect(captureDartPackageConfig(root, ['pubspec.yaml'])).rejects.toThrow( + '(read-pubspec)', + ); + }, + 3_000, + ); + it.skipIf(process.platform === 'win32' || process.getuid?.() === 0)( 'fails closed when a pubspec cannot be read', async () => { const root = await fixture({ 'pubspec.yaml': 'name: app' }); await chmod(path.join(root, 'pubspec.yaml'), 0o000); - await expect(loadDartPackageConfig(root)).rejects.toThrow( - 'Dart pubspec discovery failed (read-pubspec)', - ); + await expect(loadDartPackageConfig(root)).rejects.toThrow('(read-pubspec)'); }, ); - it('does not block when a listed pubspec is replaced by a fifo', async () => { - const root = await fixture({ 'pubspec.yaml': 'name: app' }); - const manifest = path.join(root, 'pubspec.yaml'); - await expect( - loadDartPackageConfig(root, { - beforeEntryOpen: async (relative) => { - if (relative !== '') return; - await rm(manifest); - execFileSync('mkfifo', [manifest]); - }, - }), - ).rejects.toThrow('Dart pubspec discovery failed (read-pubspec)'); - }, 3_000); - - it.skipIf(process.platform === 'win32')( - 'does not follow a symlinked pubspec into another tree', - async () => { - const outside = await fixture({ 'pubspec.yaml': 'name: foreign' }); - const root = await fixture({ 'packages/data/pubspec.yaml': 'name: data' }); - await symlink(path.join(outside, 'pubspec.yaml'), path.join(root, 'pubspec.yaml')); - expect((await loadDartPackageConfig(root)).packages).toEqual( - new Map([['data', 'packages/data/lib']]), - ); - }, - ); - - it.skipIf(process.platform !== 'darwin')( - 'does not follow a listed directory replaced by a symlink on macOS', - async () => { - const outside = await fixture({ - 'pubspec.yaml': 'name: foreign', - 'nested/pubspec.yaml': 'name: foreign', - }); - const root = await fixture({ - 'pkg/pubspec.yaml': 'name: data', - 'pkg/nested/pubspec.yaml': 'name: nested_data', - }); - const pkg = path.join(root, 'pkg'); - await expect( - loadDartPackageConfig(root, { - beforeEntryOpen: async (relative) => { - if (relative !== 'pkg') return; - await rename(pkg, path.join(root, 'pkg-moved')); - await symlink(outside, pkg, 'dir'); - }, - }), - ).rejects.toThrow(/Dart pubspec discovery failed \((read-directory|read-pubspec)\)/); - }, - ); - - // Linux lists the opened inode through /proc/self/fd/N. macOS cannot list a - // descriptor (opendir on /dev/fd/N is ENOTDIR), so it lists the path and - // re-checks the pinned chain afterwards; the swap must fail that check. - it('lists the opened directory, or refuses, when its path is replaced before listing', async () => { + it.skipIf(process.platform === 'win32')('does not follow a symlinked manifest', async () => { const outside = await fixture({ 'pubspec.yaml': 'name: foreign' }); - const root = await fixture({ - 'pkg/pubspec.yaml': 'name: data', - 'pkg/nested/pubspec.yaml': 'name: nested_data', - }); - const pkg = path.join(root, 'pkg'); - const load = loadDartPackageConfig(root, { - beforeDirectoryList: async (relative) => { - if (relative !== 'pkg') return; - await rename(pkg, path.join(root, 'pkg-moved')); - await symlink(outside, pkg, 'dir'); - }, - }); - if (process.platform === 'darwin') { - await expect(load).rejects.toThrow('Dart pubspec discovery failed (read-directory): pkg'); - return; - } - expect((await load).packages).toEqual( - new Map([ - ['data', 'pkg/lib'], - ['nested_data', 'pkg/nested/lib'], - ]), - ); + const root = await fixture({ 'nested/pubspec.yaml': 'name: data' }); + await symlink(path.join(outside, 'pubspec.yaml'), path.join(root, 'pubspec.yaml')); + expect( + (await captureDartPackageConfig(root, ['pubspec.yaml', 'nested/pubspec.yaml'])).packages, + ).toEqual(new Map([['data', 'nested/lib']])); }); - it.skipIf(descriptorEntryPath(0, 'pubspec.yaml') === null)( - 'reads listed manifests from the opened directory inode after that path is replaced', + it.skipIf(process.platform === 'win32')( + 'closes a symlinked manifest without reading when no-follow is unavailable', async () => { - const outside = await fixture({ - 'pubspec.yaml': 'name: foreign', - 'nested/pubspec.yaml': 'name: foreign', + const outside = await fixture({ 'pubspec.yaml': 'name: foreign' }); + const root = await fixture({ 'nested/pubspec.yaml': 'name: data' }); + const manifest = path.join(root, 'pubspec.yaml'); + await symlink(path.join(outside, 'pubspec.yaml'), manifest); + const realOpen = fs.open; + let opened: Awaited> | undefined; + const read = vi.fn(); + let restoreRead = () => {}; + const spy = vi.spyOn(fs, 'open').mockImplementation(async (file, flags, mode) => { + if (file !== manifest) return realOpen(file, flags, mode); + const handle = await realOpen( + file, + typeof flags === 'number' ? flags & ~(fs.constants.O_NOFOLLOW ?? 0) : flags, + mode, + ); + opened = handle; + const readSpy = vi.spyOn(handle, 'read').mockImplementation(read); + restoreRead = () => readSpy.mockRestore(); + return handle; }); - const root = await fixture({ - 'pkg/pubspec.yaml': 'name: data', - 'pkg/nested/pubspec.yaml': 'name: nested_data', - }); - const pkg = path.join(root, 'pkg'); - const loaded = await loadDartPackageConfig(root, { - beforeEntryOpen: async (relative) => { - if (relative !== 'pkg') return; - await rename(pkg, path.join(root, 'pkg-moved')); - await symlink(outside, pkg, 'dir'); - }, - }); - expect(loaded.packages).toEqual( - new Map([ - ['data', 'pkg/lib'], - ['nested_data', 'pkg/nested/lib'], - ]), - ); - }, - ); - - it.skipIf(descriptorDirectoryPath(0) === null)( - 'lists the opened directory inode after its path becomes a symlink', - async () => { - const outside = await fixture({ 'outside.txt': 'out' }); - const root = await fixture({}); - const real = path.join(root, 'real'); - await mkdir(real); - await writeFile(path.join(real, 'inside.txt'), 'in'); - const handle = await open(real, directoryOpenFlags()); + syncBuiltinESMExports(); try { - const listed = descriptorDirectoryPath(handle.fd); - if (listed === null) throw new Error('descriptor listing path missing'); - await rename(real, path.join(root, 'real-moved')); - await symlink(outside, real, process.platform === 'win32' ? 'junction' : 'dir'); - await expect(readDirectoryNoFollow(real)).rejects.toThrow(); - expect(await readdir(listed)).toEqual(['inside.txt']); + expect( + (await captureDartPackageConfig(root, ['pubspec.yaml', 'nested/pubspec.yaml'])).packages, + ).toEqual(new Map([['data', 'nested/lib']])); + expect(read).not.toHaveBeenCalled(); + expect(spy.mock.calls.filter(([file]) => file === manifest)).toHaveLength(1); + expect(opened?.fd).toBe(-1); } finally { - await handle.close(); + restoreRead(); + spy.mockRestore(); + syncBuiltinESMExports(); } }, ); -}); -describe('directory symlink refusal', () => { - const roots: string[] = []; - afterEach(async () => { - for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); + it('excludes candidates below directory symlinks or Windows junctions', async () => { + const outside = await fixture({ 'nested/pubspec.yaml': 'name: foreign' }); + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + await symlink( + outside, + path.join(root, 'linked'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + expect( + (await captureDartPackageConfig(root, ['pubspec.yaml', 'linked/nested/pubspec.yaml'])) + .packages, + ).toEqual(config.packages); }); - it('refuses a directory symlink instead of listing its target', async () => { - const outside = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-dart-pubspec-')); - const root = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-dart-pubspec-')); - roots.push(outside, root); - await writeFile(path.join(outside, 'secret.txt'), 'name: foreign'); - const link = path.join(root, 'linked'); - await symlink(outside, link, process.platform === 'win32' ? 'junction' : 'dir'); - await expect(readDirectoryNoFollow(link)).rejects.toThrow(); - }); - - it('does not discover packages when the walk cannot honor no-follow', async () => { - const root = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-dart-pubspec-')); - roots.push(root); - await writeFile(path.join(root, 'pubspec.yaml'), 'name: app\n'); - // Exercise the unsupported-platform branch on every host. The real - // capability check must refuse before attempting any directory walk. - const platform = vi.spyOn(process, 'platform', 'get').mockReturnValue('win32'); + it('pins the manifest before pathname validation can race with replacement', async () => { + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + const manifest = path.join(root, 'pubspec.yaml'); + const original = await fs.lstat(manifest, { bigint: true }); + const realOpen = fs.open; + const realLstat = fs.lstat; + let replaced = false; + let openedInode: bigint | undefined; + const statSpy = vi.spyOn(fs, 'lstat').mockImplementation(async (file, options) => { + const info = await realLstat(file, options); + if (file === manifest && !replaced) { + replaced = true; + await rename(manifest, path.join(root, 'old.yaml')); + await fs.utimes(path.join(root, 'old.yaml'), 1, 1); + await writeFile(manifest, 'name: foreign'); + } + return info; + }); + const spy = vi.spyOn(fs, 'open').mockImplementation(async (file, flags, mode) => { + const handle = await realOpen(file, flags, mode); + if (file === manifest) openedInode = (await handle.stat({ bigint: true })).ino; + return handle; + }); + syncBuiltinESMExports(); try { - expect(pubspecWalkAnchored()).toBe(false); - expect((await loadDartPackageConfig(root)).packages.size).toBe(0); + await expect(captureDartPackageConfig(root, ['pubspec.yaml'])).rejects.toThrow( + '(read-pubspec)', + ); + expect(replaced).toBe(true); + expect(openedInode).toBe(original.ino); } finally { - platform.mockRestore(); + spy.mockRestore(); + statSpy.mockRestore(); + syncBuiltinESMExports(); + } + }); + + it('closes a descriptor without reading when its pathname is replaced after open', async () => { + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + const manifest = path.join(root, 'pubspec.yaml'); + const realOpen = fs.open; + let opened: Awaited> | undefined; + const read = vi.fn(); + const spy = vi.spyOn(fs, 'open').mockImplementation(async (file, flags, mode) => { + const handle = await realOpen(file, flags, mode); + if (file === manifest) { + opened = handle; + vi.spyOn(handle, 'read').mockImplementation(read); + await rename(manifest, path.join(root, 'old.yaml')); + await writeFile(manifest, 'name: foreign'); + } + return handle; + }); + syncBuiltinESMExports(); + try { + await expect(captureDartPackageConfig(root, ['pubspec.yaml'])).rejects.toThrow( + '(read-pubspec)', + ); + expect(read).not.toHaveBeenCalled(); + expect(opened?.fd).toBe(-1); + } finally { + spy.mockRestore(); + syncBuiltinESMExports(); + } + }); + + it('rejects a same-size manifest edit after the initial descriptor check', async () => { + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + const manifest = path.join(root, 'pubspec.yaml'); + const realOpen = fs.open; + let changed = false; + const spy = vi.spyOn(fs, 'open').mockImplementation(async (file, flags, mode) => { + const handle = await realOpen(file, flags, mode); + if (file === manifest) { + const realStat = handle.stat.bind(handle); + vi.spyOn(handle, 'stat').mockImplementationOnce(async () => { + const before = await realStat({ bigint: true }); + await writeFile(manifest, 'name: new'); + // Force an observable change even on a filesystem with coarse timestamps. + await fs.utimes(manifest, 1, 1); + changed = true; + return before; + }); + } + return handle; + }); + syncBuiltinESMExports(); + try { + await expect(captureDartPackageConfig(root, ['pubspec.yaml'])).rejects.toThrow( + '(read-pubspec)', + ); + expect(changed).toBe(true); + } finally { + spy.mockRestore(); + syncBuiltinESMExports(); + } + }); + + it('uses only captured package identity after the repository path is replaced', async () => { + const root = await fixture({ + 'pubspec.yaml': 'name: app', + 'lib/models.dart': 'class Model {}', + }); + const captured = await scanPhase.execute( + { + repoPath: root, + graph: createKnowledgeGraph(), + onProgress: () => {}, + pipelineStart: Date.now(), + }, + new Map(), + ); + const resolutionConfig = captured.resolutionConfigs?.get(SupportedLanguages.Dart); + expect(resolutionConfig).toBeDefined(); + const moved = `${root}-moved`; + await rename(root, moved); + roots.push(moved); + await mkdir(root); + await writeFile(path.join(root, 'pubspec.yaml'), 'name: foreign'); + expect( + dartScopeResolver.resolveImportTarget( + 'package:app/models.dart', + 'lib/main.dart', + new Set(captured.allPaths), + resolutionConfig, + ), + ).toBe('lib/models.dart'); + expect( + dartScopeResolver.resolveImportTarget( + 'package:foreign/models.dart', + 'lib/main.dart', + new Set(captured.allPaths), + resolutionConfig, + ), + ).toBeNull(); + }); + + it('rejects oversized captured metadata before the shared scanner can filter it away', async () => { + const root = await fixture({ + 'pubspec.yaml': `name: app\n#${'x'.repeat(1024 * 1024)}`, + 'lib/main.dart': 'void main() {}', + }); + await expect( + scanPhase.execute( + { + repoPath: root, + graph: createKnowledgeGraph(), + onProgress: () => {}, + pipelineStart: Date.now(), + }, + new Map(), + ), + ).rejects.toThrow('(manifest-size)'); + }); + + it('opens only manifest candidates and checks each shared parent once', async () => { + const root = await fixture({ + 'packages/a/pubspec.yaml': 'name: a', + 'packages/b/pubspec.yaml': 'name: b', + }); + const paths = Array.from({ length: 10_000 }, (_, i) => `other/file${i}.dart`); + paths.push('packages/a/pubspec.yaml', 'packages/b/pubspec.yaml'); + const openSpy = vi.spyOn(fs, 'open'); + const statSpy = vi.spyOn(fs, 'lstat'); + syncBuiltinESMExports(); + try { + expect((await captureDartPackageConfig(root, paths)).packages.size).toBe(2); + expect(openSpy).toHaveBeenCalledTimes(2); + expect( + statSpy.mock.calls.filter(([file]) => file === path.join(root, 'packages')), + ).toHaveLength(1); + } finally { + openSpy.mockRestore(); + statSpy.mockRestore(); + syncBuiltinESMExports(); } }); }); diff --git a/gitnexus/test/unit/filesystem-walker-order.test.ts b/gitnexus/test/unit/filesystem-walker-order.test.ts index 7b75128a4..bdbdaba97 100644 --- a/gitnexus/test/unit/filesystem-walker-order.test.ts +++ b/gitnexus/test/unit/filesystem-walker-order.test.ts @@ -22,6 +22,24 @@ afterEach(async () => { }); describe('walkRepositoryPaths ordering', () => { + it('preserves required candidates before size and stat failures can omit them', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-scan-capture-')); + temporaryRoots.push(root); + await fs.writeFile(path.join(root, 'large.yaml'), 'x'.repeat(100)); + vi.mocked(glob).mockResolvedValue(['large.yaml', 'missing.yaml']); + const capture = vi.fn(async () => { + throw new Error('required metadata unavailable'); + }); + + await expect( + walkRepositoryPaths(root, undefined, { + maxFileSizeBytes: 10, + onPathsDiscovered: capture, + }), + ).rejects.toThrow('required metadata unavailable'); + expect(capture).toHaveBeenCalledExactlyOnceWith(['large.yaml', 'missing.yaml']); + }); + it('returns accepted files in canonical path order when glob order is unstable', async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-scan-order-')); temporaryRoots.push(root);