diff --git a/gitnexus/bench/import-target/baselines.json b/gitnexus/bench/import-target/baselines.json index 7b9a9932d..317f56898 100644 --- a/gitnexus/bench/import-target/baselines.json +++ b/gitnexus/bench/import-target/baselines.json @@ -298,42 +298,42 @@ "imports": 3200, "resolved": 1153, "distinct_outcomes": 2987, - "fingerprint": "318084f48ffa4eeae4a5b7fc25916d4ad78673d92dba62b7e462d1bb87ca553a" + "fingerprint": "eb36bd1ccab5c4405cc79d48732b7aa272a90351db60664137663dde7b2db73c" }, "large": { "files": 1600, "imports": 12800, "resolved": 4681, "distinct_outcomes": 11875, - "fingerprint": "5151cd2498bd4b7698dc9309e2539977d306f9ba82a388c630c89b51fc4a3187" + "fingerprint": "08c288bdeaa6f4518e7f5e32e3a8ac6265a5147ec366d37789db6cb573ed2bbe" }, "deep": { "files": 400, "imports": 3200, "resolved": 1153, "distinct_outcomes": 2987, - "fingerprint": "79776ec1c22afa619fd31aeb05dcac567723b436d0461a5782693b9e929f6f74" + "fingerprint": "909c8444e0004692be0d0a8179f56173e87d790ad7aeea38bb54c647ce0d638f" }, "collide": { "files": 400, "imports": 3200, "resolved": 1153, "distinct_outcomes": 2999, - "fingerprint": "1b145a4c3b41ffc4efa26f74449c3d44646d6d59728163ac896fc0ee25c6d608" + "fingerprint": "4625f0310442e60b4291fd9f5c9b90333f6e9175ab5b4c7c7eb2ecbd5def500d" }, "collide_large": { "files": 1600, "imports": 12800, "resolved": 4681, "distinct_outcomes": 11948, - "fingerprint": "b7e5303220b8fa64e85c7e17622961018316a10c5ef921a02584864309748b52" + "fingerprint": "9e7aa3a503352b1f437a70e51ccfe020dff65da319c6082fb25b1ae4417c3531" }, - "fingerprint": "5151cd2498bd4b7698dc9309e2539977d306f9ba82a388c630c89b51fc4a3187", + "fingerprint": "08c288bdeaa6f4518e7f5e32e3a8ac6265a5147ec366d37789db6cb573ed2bbe", "heap": { "files_small": 8000, "files_large": 32000, "path_segments": 11, - "probe": "package:ext0/src/thing.dart" + "probe": "./src/thing.dart" }, "_measured": { "collide_ms": 1.511, diff --git a/gitnexus/bench/import-target/measure.mjs b/gitnexus/bench/import-target/measure.mjs index b66932d77..6290a8893 100644 --- a/gitnexus/bench/import-target/measure.mjs +++ b/gitnexus/bench/import-target/measure.mjs @@ -208,12 +208,14 @@ * workload — identical file, import and resolved counts — laid out the way * these languages are actually written: `svcN/internal/`, `SrcN/Models/`, a * `mod0.dart`/`mod0.rb` in every package. Measured on that shape the per-import - * cost is NOT corpus-size-independent for the four resolvers that scan a + * cost is NOT corpus-size-independent for the three resolvers that scan a * bucket: * * - go, csharp and java walk `PackageDirIndex.dirsByLastSegment[seg]`, which * now holds every directory; - * - dart walks its basename bucket, which now holds every same-named file; + * - dart formerly walked its basename bucket. Since #2963 its package-URI + * arms use exact declared-package paths; the separate heap probe and + * scan-count regressions still exercise its relative-path suffix index; * - ruby, kotlin, php and cobol answer from keyed maps and are collision- * IMMUNE, so their collide budgets are the linear ones — that immunity is * the assertion, and for cobol the arm is also the only one that reaches @@ -849,7 +851,8 @@ function uniqueDir(lang, d, i) { // and breaks its same-workload invariant. C# therefore exercises the removed // rule through progressive stripping rather than through its primary query. if (lang === 'csharp') return d % 7 === 0 ? `src/Ns${d}/Sub/Ns${d}` : `src/Ns${d}`; - if (lang === 'dart') return d % 3 === 0 ? `lib/feature${d}` : `pkg/feature${d}`; + // Keep every synthetic local import valid under app's pubspec lib root. + if (lang === 'dart') return `lib/feature${d}`; if (lang === 'kotlin') { return d % 7 === 0 ? `mod${d}/src/main/kotlin/com/example/pkg${d}/inner/pkg${d}` @@ -1500,11 +1503,10 @@ function collideTarget(lang, { local, r, d, j, dirs }) { } if (lang === 'dart') { return local - ? `package:app/pkg${j % dirs}/lib/src/mod${Math.floor(j / dirs)}.dart` + ? `package:pkg${j % dirs}/src/mod${Math.floor(j / dirs)}.dart` : (r >>> 3) % 3 === 0 ? ['dart:core', 'dart:async', 'dart:io'][(r >>> 4) % 3] - : // A repeated basename under a directory nothing carries: both - // candidates walk the whole basename bucket and miss. + : // Foreign package names must miss despite repeated local basenames. `package:ext${(r >>> 4) % 97}/other/mod${(r >>> 4) % 8}.dart`; } if (lang === 'kotlin') { @@ -1740,6 +1742,16 @@ function buildRepo(lang, fileCount, pad = 0, shape = 'unique') { * hide their build from rep 2 onward and `fastest()` reports the minimum. */ function newPass(lang, files, pad = 0) { + if (lang === 'dart') { + // Synthetic pubspec declarations: app at the root, one package per + // collision directory. Package imports no longer suffix-match (#2963). + const packages = new Map([['app', joinBase(tsBaseUrlFor(pad), 'lib')]]); + for (const file of files) { + const match = /(?:^|\/)(pkg\d+)\/lib\//.exec(file); + if (match) packages.set(match[1], joinBase(tsBaseUrlFor(pad), `${match[1]}/lib`)); + } + return { allFilePaths: new Set(files), config: { packages } }; + } if (HEADER_EXTENSION[lang] !== undefined) { const sources = []; const headers = []; @@ -1803,7 +1815,7 @@ function resolveAll(lang, files, imports, pad = 0) { function resolveOne(lang, from, target, pass) { const allFilePaths = pass.allFilePaths; if (lang === 'go') return resolveGoImportTarget(target, from, allFilePaths, GO_MODULE); - if (lang === 'dart') return resolveDartImportTarget(target, from, allFilePaths); + if (lang === 'dart') return resolveDartImportTarget(target, from, allFilePaths, pass.config); if (lang === 'ruby') return resolveRubyImportTarget(target, from, allFilePaths); if (lang === 'kotlin') { const parsedImport = { @@ -2131,6 +2143,10 @@ const HEAP_RETAINED = []; * #2903 made lazy. It is the witness that the read pattern IS the * footprint: same corpus and same `getWorkspaceFileIndex` as `csharp`, * three times the retained bytes. + * + * `dart` is the exception (`./src/thing.dart` below). `uniqueTarget` never + * emits a relative path, and no timing arm exercises the relative suffix + * fallback, so that probe does. */ const HEAP_PROBE_TARGET = { csharp: 'Ghost0.Deep.Missing', @@ -2149,13 +2165,15 @@ const HEAP_PROBE_TARGET = { objc: 'vendor0/missing.m', // The entries below cover the BOUNDED tier — see `HEAP_BOUNDED`, which // derives to cobol, swift and rust; the rest were promoted. Same rule as the - // budgeted ones above: a spelling `uniqueTarget` already mints for that language, and + // budgeted ones above, except `dart`: a spelling `uniqueTarget` already mints, and // one that MISSES, so the reading is the index and the cascade runs to the // end. Chosen from the miss family that reaches furthest into each cascade: // - `go` names a missing package inside GO_MODULE, which reaches the // package-directory lookup and forces `PackageDirIndex`; - // - `dart` is an external package, so BOTH candidate paths miss and both - // walk the basename bucket to completion; + // - `dart` is a relative miss (`./src/thing.dart`), not a `uniqueTarget` + // spelling. No timing arm exercises the relative suffix fallback; this + // probe does, so the basename index stays in the heap reading. Package + // imports use exact membership and allocate no file index; // - `kotlin` misses after building its declared-package/module-binding index; // - `cobol` misses in both tier maps, `swift` in `byModule`, and `rust` // probes candidate paths and builds nothing — that last is the reading @@ -2165,7 +2183,7 @@ const HEAP_PROBE_TARGET = { // bound compares like with like. `vue`'s is bare rather than `@/…` // because the alias branch rewrites to `src/` and would resolve. go: 'example.com/mod/repo0/pkg/util', - dart: 'package:ext0/src/thing.dart', + dart: './src/thing.dart', kotlin: 'com.ghost0.deep.Missing', cobol: 'VENDOR0', swift: 'ExternalPkg0', diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index 1993458d8..5b11e77c3 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -136,8 +136,9 @@ "_rebaselined_3354_callable_alternatives": "#3354 callable alternatives: a callable chosen by a value-selecting source now flows every branch it can yield (`a ?? b`, `a || b`, `a or b`, `c ? a : b`, statement `if`, elvis), and an operator branch (`x == f || g`) stays opaque instead of seeding a qualified name. Verified by running the BASE (merge-base 233ca2849) and HEAD emitters over the SAME HEAD fixture corpus: every added or removed match is an `@callable-flow.*` match on one of those sources, and the pre-existing corpus is byte-identical (all other languages: zero delta). The rest of the drift is corpus growth from this PR's regression fixture, which this bench globs. Corpus growth: swift-callable-alternatives/App.swift (+1 file, +36 groups). Emitter delta: +6 / -3: `??` and ternary seeds and copies now name each operand; the 3 removed matches are the whole-expression seeds and copy that carried a `target-qualified-name` / `source-qualified-name` of the compound source. capture_groups_fp 1297 -> 1333, fixture_count 67 -> 68; synthetic counts unchanged; scaling 1.057 < 1.5. Prior decf74c01af0c7f403e203b19c2bd92dbf95b4562f6da2b0ac5117ef872204da -> c9fc553662f0db18027fba882e3ff730744f6153cc46eca7b00667fabd6a0df8." }, "dart": { - "fingerprint": "795290a9524ee0dc38af635536e9744c401ec13828ec77822d39e59e1cd6e649", + "fingerprint": "f4edcfedb6e97def5ffbb9d3227fce2549e5c6c6cc876a05bc3b68a46ed30c24", "scaling_budget": 1.5, + "_rebaselined_2963_package_imports": "#3369 package-import fixtures: dart-package-imports adds six .dart files. CORPUS GROWTH ONLY, NOT A CAPTURE CHANGE: parking that directory restores 795290a9524ee0dc38af635536e9744c401ec13828ec77822d39e59e1cd6e649 byte-for-byte. The six files contribute 33 capture groups (862 -> 895) and fixture_count 37 -> 43. Synthetic scale counts stay 5011/16011. Local scaling 1.091 < 1.5. Prior 795290a9524ee0dc38af635536e9744c401ec13828ec77822d39e59e1cd6e649 -> f4edcfedb6e97def5ffbb9d3227fce2549e5c6c6cc876a05bc3b68a46ed30c24.", "_rebaselined_generic_instantiation_2912": "#2912: the Dart heritage marker carries a fourth field \u2014 the type arguments the clause was written with (`implements Validator`) \u2014 so interface dispatch can prune implementors of a mismatched instantiation. Additive marker text on existing heritage matches rather than a new match, so this is digest drift only; a marker from a pre-#2912 cache simply has no fourth field and reads as unknown. Prior ba93c90dcd341259e8e088816bc8c76ad27882419f665e35c056dc22fa54cf73 -> 3a8ddabbeb1cba47a4757451d4f79d726ca230fd15e860772b11526fbb1c6687; scaling 1.027 < 1.5.", "_rebaselined_2538": "#2538: Dart extension type headers are preprocessed into normal extension declarations before scope capture, so extension type symbols and their methods are now emitted. Intentional Dart-only capture fingerprint drift; CI measured scaling 1.042 < 1.5.", "_rebaselined_2538_implements": "#2538 tri-review follow-up: Dart extension type implements clauses now emit heritage markers and fixture coverage asserts IMPLEMENTS edges, including multi-arg generic interfaces. Prior committed baseline 66a46d5ff09f3d11b2771db0f48596fe7057e95c5bc8f56241fdb911137298c3 -> ba93c90dcd341259e8e088816bc8c76ad27882419f665e35c056dc22fa54cf73; scaling 0.945 < 1.5.", diff --git a/gitnexus/src/core/ingestion/languages/dart/import-target.ts b/gitnexus/src/core/ingestion/languages/dart/import-target.ts index fd6c5224a..602ea5ce4 100644 --- a/gitnexus/src/core/ingestion/languages/dart/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/dart/import-target.ts @@ -1,30 +1,28 @@ /** * `resolveImportTarget` adapter for the Dart `ScopeResolver`. Ports the - * legacy-DAG Dart import logic (`import-resolvers/configs/dart.ts`): + * Dart import logic: * * - `dart:` SDK imports → `null` (external, no edge) - * - `package:pkg/path` → `lib/path` (or bare `path`) matched - * against the workspace file set + * - `package:pkg/path` → declared package's exact `lib/path` * - relative `'foo/bar.dart'` → resolved against the importer's dir * - `__heritage__:` markers → `null` (synthetic heritage carrier, * consumed by `emitDartHeritageEdges`) * - * The `ScopeResolver` hook signature is `(targetRaw, fromFile, allFilePaths)`; + * Package identity comes from the workspace's pubspec resolution config; * `targetRaw` arrives already quote-stripped from `interpretDartImport`. */ import { perFileSet } from '../../import-resolvers/per-file-set.js'; import { DART_HERITAGE_PREFIX } from './interpret.js'; +import type { DartPackageConfig } from './package-config.js'; +import { DART_PACKAGE_SCHEME, dartPackageImportName } from './package-uri.js'; /** * Basename → files carrying it, in `allFilePaths` iteration order, memoized on * the Set's identity (#2879). * - * Both resolution legs answered `fp === candidate || fp.endsWith('/' + candidate)` - * with a full workspace scan, and the `package:` leg ran one scan PER candidate - * — for an external package both candidates miss, so both scans always ran to - * completion. The orchestrator passes the same Set to every import in a pass, - * so the index is built once per run. + * Relative-path suffix fallback uses this index instead of scanning the + * workspace for each import. Package imports use exact Set membership only. * * Bucketing by basename is exact rather than a heuristic: a path satisfying * either arm of the match ends with `candidate`, so its last `/`-delimited @@ -89,6 +87,7 @@ export function resolveDartImportTarget( targetRaw: string, fromFile: string, allFilePaths: ReadonlySet, + resolutionConfig?: unknown, ): string | readonly string[] | null { if (targetRaw.startsWith(DART_HERITAGE_PREFIX)) return null; // `targetRaw` already arrives quote-stripped from `interpretDartImport`. @@ -97,17 +96,21 @@ export function resolveDartImportTarget( // Dart SDK imports never resolve to a repo file. if (targetRaw.startsWith('dart:')) return null; - // `package:pkg/path.dart` → `lib/path.dart` (or bare `path.dart`). - if (targetRaw.startsWith('package:')) { - const slash = targetRaw.indexOf('/'); - if (slash === -1) return null; - const relPath = targetRaw.slice(slash + 1); - // Candidate priority is load-bearing: `lib/` before bare ``. - for (const candidate of [`lib/${relPath}`, relPath]) { - const hit = findByPathSuffix(allFilePaths, candidate); - if (hit !== null) return hit; - } - return null; // external package + // A package URI never falls back to another package's same-named file. + if (targetRaw.startsWith(DART_PACKAGE_SCHEME)) { + const packageName = dartPackageImportName(targetRaw); + if (packageName === null) return null; + const config = resolutionConfig as DartPackageConfig | undefined; + const lib = config?.packages?.get(packageName); + if (lib === undefined) return null; + const relPath = targetRaw.slice(DART_PACKAGE_SCHEME.length + packageName.length + 1); + if ( + /[\\%?#:]/.test(relPath) || + relPath.split('/').some((part) => part === '' || part === '.' || part === '..') + ) + return null; + const candidate = `${lib}/${relPath}`; + return allFilePaths.has(candidate) ? candidate : null; } // Relative import. diff --git a/gitnexus/src/core/ingestion/languages/dart/index.ts b/gitnexus/src/core/ingestion/languages/dart/index.ts index 9059cf108..3cb814cde 100644 --- a/gitnexus/src/core/ingestion/languages/dart/index.ts +++ b/gitnexus/src/core/ingestion/languages/dart/index.ts @@ -9,7 +9,10 @@ * - `query.ts` — tree-sitter scope query + lazy parser/query * - `captures.ts` — `emitDartScopeCaptures` orchestrator * - `interpret.ts` — capture-match → `ParsedImport` / `ParsedTypeBinding` - * - `import-target.ts` — `(targetRaw, fromFile, allFilePaths) → file path` + * - `import-target.ts` — exact declared-package / relative import targets + * - `package-uri.ts` — shared `package:name/...` parse + * - `package-config.ts` — bounded, ignore-aware in-repo pubspec discovery + * - `package-dependencies.ts` — manifest dependencies for incremental writeback * - `receiver-binding.ts` — synthesize `this` / `super` type-bindings * - `signature-bindings.ts`— synthesize parameter / return type-bindings * - `arity.ts` — Dart arity compatibility (count-primary) diff --git a/gitnexus/src/core/ingestion/languages/dart/package-config.ts b/gitnexus/src/core/ingestion/languages/dart/package-config.ts new file mode 100644 index 000000000..4503a57bd --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/dart/package-config.ts @@ -0,0 +1,452 @@ +import type { BigIntStats, Dirent } from 'node:fs'; +import { constants, lstat, open, opendir, type FileHandle } from 'node:fs/promises'; +import path from 'node:path'; +import { JSON_SCHEMA, load } from 'js-yaml'; +import { createWatchIgnorePredicate } from '../../../../config/ignore-service.js'; +import { logger } from '../../../logger.js'; +import { getMaxFileSizeBytes } from '../../utils/max-file-size.js'; + +export interface DartPackageConfig { + readonly packages: ReadonlyMap; + readonly manifestsByName: ReadonlyMap; +} + +/** An incomplete walk cannot prove package names are unique. */ +const DART_PUBSPEC_DIRECTORY_LIMIT = 20_000; + +/** + * Directory descriptors one walk may hold at once. The visit budget is far + * above a process file-descriptor limit, so a deep chain is refused before + * the next open instead of failing later with EMFILE. + */ +const DART_PUBSPEC_OPEN_DIRECTORY_LIMIT = 64; + +/** + * Names read from one directory. `readdir` would retain every entry before the + * visit budget can run, so the walk counts names as it reads and stops there. + */ +const DART_PUBSPEC_DIRECTORY_ENTRY_LIMIT = 100_000; + +export interface DartPackageConfigOptions { + /** Test seam. Production calls omit it and use the module directory limit. */ + readonly directoryLimit?: number; + /** Test seam. Production calls omit it and use the open-directory cap. */ + readonly directoryDepthLimit?: number; + /** Test seam. Production calls omit it and use the per-directory entry cap. */ + readonly directoryEntryLimit?: number; + /** + * Test seam. Production calls omit it. Invoked after the directory inode + * is listed and before its entries are opened. + */ + readonly beforeEntryOpen?: (relativePath: string) => void | Promise; +} + +type ManifestRead = + | { readonly ok: true; readonly content: string } + | { readonly ok: false; readonly reason: 'manifest-size' | 'read-pubspec' }; + +/** + * Read at most `maxManifestSize` bytes from a descriptor already opened + * with `O_NOFOLLOW`. A size check after the read rejects a file that grew + * past the captured bytes, including past the cap. The caller closes nothing; + * this function owns `handle`. + */ +async function readManifestBounded( + handle: FileHandle, + maxManifestSize: number, +): Promise { + try { + const info = await handle.stat(); + if (!info.isFile()) return { ok: false, reason: 'read-pubspec' }; + if (info.size > maxManifestSize) return { ok: false, reason: 'manifest-size' }; + const toRead = Math.min(maxManifestSize + 1, info.size + 1); + const buffer = Buffer.allocUnsafe(toRead); + let bytesRead = 0; + while (bytesRead < toRead) { + const chunk = await handle.read(buffer, bytesRead, toRead - bytesRead, bytesRead); + if (chunk.bytesRead === 0) break; + bytesRead += chunk.bytesRead; + } + if (bytesRead > maxManifestSize) return { ok: false, reason: 'manifest-size' }; + const after = await handle.stat(); + if (!after.isFile()) return { ok: false, reason: 'read-pubspec' }; + if (after.size > maxManifestSize) return { ok: false, reason: 'manifest-size' }; + if (after.size !== bytesRead) return { ok: false, reason: 'read-pubspec' }; + return { ok: true, content: buffer.subarray(0, bytesRead).toString('utf8') }; + } catch { + return { ok: false, reason: 'read-pubspec' }; + } finally { + await handle.close(); + } +} + +function requireNoFollowFlag(): number { + const noFollow = constants.O_NOFOLLOW; + if (typeof noFollow !== 'number' || noFollow === 0) { + throw Object.assign(new Error('O_NOFOLLOW is unavailable'), { code: 'ENOTSUP' }); + } + return noFollow; +} + +/** Linux anchors child opens. macOS verifies them. Every other platform does neither. */ +export function pubspecWalkAnchored(): boolean { + const noFollow = constants.O_NOFOLLOW; + return ( + (process.platform === 'linux' || process.platform === 'darwin') && + typeof noFollow === 'number' && + noFollow !== 0 + ); +} + +export function directoryOpenFlags(): number { + let flags = constants.O_RDONLY | requireNoFollowFlag(); + if (typeof constants.O_DIRECTORY === 'number') flags |= constants.O_DIRECTORY; + return flags; +} + +/** + * Read-only, no-follow, and non-blocking. `O_NONBLOCK` does not change a + * regular-file read. Without it, a FIFO blocks inside `open` until a writer + * connects, so the later file-type check never runs. + */ +function fileOpenFlags(): number { + const nonBlock = constants.O_NONBLOCK; + if (typeof nonBlock !== 'number' || nonBlock === 0) { + throw Object.assign(new Error('O_NONBLOCK is unavailable'), { code: 'ENOTSUP' }); + } + return constants.O_RDONLY | requireNoFollowFlag() | nonBlock; +} + +function directoryIdentity(stat: BigIntStats): string { + return `${stat.dev}:${stat.ino}:${stat.mode}`; +} + +/** + * Path that lists the directory inode already open on `fd`. + * Linux uses `/proc/self/fd/N` and macOS uses `/dev/fd/N`. + * Other platforms have no such path; callers refuse instead of listing by name. + */ +export function descriptorDirectoryPath(fd: number): string | null { + if (process.platform === 'linux') return `/proc/self/fd/${fd}`; + if (process.platform === 'darwin') return `/dev/fd/${fd}`; + return null; +} + +/** + * One entry of the directory inode open on `fd`. + * Linux looks up `/proc/self/fd/N/` in that inode. macOS `/dev/fd/N/` + * does not resolve a child, so this returns null and the walker verifies the + * pinned parent chain instead. Every other platform returns null and is not walked. + */ +export function descriptorEntryPath(fd: number, name: string): string | null { + if (process.platform !== 'linux') return null; + if (!isSingleDirectoryEntry(name)) return null; + return `/proc/self/fd/${fd}/${name}`; +} + +function isSingleDirectoryEntry(name: string): boolean { + return ( + name !== '' && name !== '.' && name !== '..' && !name.includes('/') && !name.includes('\0') + ); +} + +interface OpenedDirectory { + readonly handle: FileHandle; + readonly identity: string; +} + +async function openVerifiedDirectory(directory: string): Promise { + const handle = await open(directory, directoryOpenFlags()); + try { + const info = await handle.stat({ bigint: true }); + if (!info.isDirectory()) { + throw Object.assign(new Error('not a directory'), { code: 'ENOTDIR' }); + } + return { handle, identity: directoryIdentity(info) }; + } catch (error) { + await handle.close(); + throw error; + } +} + +interface WalkFrame { + relative: string; + absolute: string; + handle: FileHandle; + identity: string; + entries: Dirent[]; + next: number; +} + +/** Re-stat each pinned directory and its path. A replaced inode or a symlink fails the walk. */ +async function assertPinnedChain(frames: readonly WalkFrame[]): Promise { + for (const frame of frames) { + const pinned = await frame.handle.stat({ bigint: true }); + if (!pinned.isDirectory() || directoryIdentity(pinned) !== frame.identity) { + throw Object.assign(new Error('parent descriptor changed'), { code: 'ELOOP' }); + } + let lexical: BigIntStats; + try { + lexical = await lstat(frame.absolute, { bigint: true }); + } catch { + throw Object.assign(new Error('parent path changed'), { code: 'ELOOP' }); + } + if ( + lexical.isSymbolicLink() || + !lexical.isDirectory() || + directoryIdentity(lexical) !== frame.identity + ) { + throw Object.assign(new Error('parent path changed'), { code: 'ELOOP' }); + } + } +} + +/** + * macOS has no descriptor-relative child lookup. Re-check the pinned parents, + * open the child with O_NOFOLLOW, then re-check the parents. The opened + * descriptor is the only child metadata consulted. + */ +async function openLexicalDirectory( + frames: readonly WalkFrame[], + lexicalPath: string, +): Promise { + await assertPinnedChain(frames); + const opened = await openVerifiedDirectory(lexicalPath); + try { + await assertPinnedChain(frames); + return opened; + } catch (error) { + await opened.handle.close(); + throw error; + } +} + +async function openLexicalFile( + frames: readonly WalkFrame[], + lexicalPath: string, +): Promise { + await assertPinnedChain(frames); + const handle = await open(lexicalPath, fileOpenFlags()); + try { + const opened = await handle.stat({ bigint: true }); + if (!opened.isFile()) { + throw Object.assign(new Error('not a file'), { code: 'ELOOP' }); + } + await assertPinnedChain(frames); + return handle; + } catch (error) { + await handle.close(); + throw error; + } +} + +async function openChildDirectory( + frames: readonly WalkFrame[], + parentFd: number, + name: string, + lexicalPath: string, +): Promise { + const anchored = descriptorEntryPath(parentFd, name); + if (anchored !== null) return openVerifiedDirectory(anchored); + if (process.platform === 'darwin') return openLexicalDirectory(frames, lexicalPath); + throw Object.assign(new Error('no descriptor anchor'), { code: 'ENOTSUP' }); +} + +async function openChildFile( + frames: readonly WalkFrame[], + parentFd: number, + name: string, + lexicalPath: string, +): Promise { + const anchored = descriptorEntryPath(parentFd, name); + if (anchored !== null) return open(anchored, fileOpenFlags()); + if (process.platform === 'darwin') return openLexicalFile(frames, lexicalPath); + throw Object.assign(new Error('no descriptor anchor'), { code: 'ENOTSUP' }); +} + +async function listOpenedDirectory(handle: FileHandle, entryLimit: number): Promise { + const listing = descriptorDirectoryPath(handle.fd); + if (listing === null) { + throw Object.assign(new Error('no descriptor listing'), { code: 'ENOTSUP' }); + } + const dir = await opendir(listing); + const entries: Dirent[] = []; + try { + let count = 0; + while (true) { + const entry = await dir.read(); + if (entry === null) break; + count += 1; + if (count > entryLimit) { + throw Object.assign(new Error('directory entry limit'), { code: 'E2BIG' }); + } + entries.push(entry); + } + } finally { + await dir.close().catch(() => undefined); + } + return entries; +} + +/** + * Open `directory` without following a final symlink, then list that inode. + * A path swapped for a symlink after the parent listing fails this open + * (`ENOTDIR` / `ELOOP`) instead of being traversed. + */ +export async function readDirectoryNoFollow(directory: string): Promise { + const opened = await openVerifiedDirectory(directory); + try { + return await listOpenedDirectory(opened.handle, DART_PUBSPEC_DIRECTORY_ENTRY_LIMIT); + } finally { + await opened.handle.close(); + } +} + +/** Discover only in-repository packages; never follow dependency paths or symlinks. */ +export async function loadDartPackageConfig( + repoPath: string, + options?: DartPackageConfigOptions, +): Promise { + const warn = (reason: string, relativePath = '.'): void => { + logger.warn( + { reason, relativePath }, + 'Dart pubspec discovery could not read a valid package declaration.', + ); + }; + const incomplete = (reason: string, relativePath = '.'): never => { + warn(reason, relativePath); + throw new Error(`Dart pubspec discovery failed (${reason}): ${relativePath}`); + }; + if (!pubspecWalkAnchored()) { + warn('nofollow-anchor'); + return { packages: new Map(), manifestsByName: new Map() }; + } + let isIgnored; + try { + isIgnored = await createWatchIgnorePredicate(repoPath); + } catch { + return incomplete('ignore-rules'); + } + const packages = new Map(); + const manifestsByName = new Map(); + const maxManifestSize = Math.min(1024 * 1024, getMaxFileSizeBytes()); + const directoryLimit = options?.directoryLimit ?? DART_PUBSPEC_DIRECTORY_LIMIT; + const directoryDepthLimit = options?.directoryDepthLimit ?? DART_PUBSPEC_OPEN_DIRECTORY_LIMIT; + const directoryEntryLimit = options?.directoryEntryLimit ?? DART_PUBSPEC_DIRECTORY_ENTRY_LIMIT; + const ambiguous = new Set(); + const stack: WalkFrame[] = []; + let visited = 0; + + const closeStack = async (): Promise => { + const frames = stack.splice(0); + await Promise.all(frames.map((frame) => frame.handle.close().catch(() => undefined))); + }; + + const fillFrame = async (frame: WalkFrame): Promise => { + if (++visited > directoryLimit) return incomplete('directory-limit', frame.relative || '.'); + try { + frame.entries = await listOpenedDirectory(frame.handle, directoryEntryLimit); + } catch (error) { + const reason = + (error as NodeJS.ErrnoException).code === 'E2BIG' ? 'directory-entries' : 'read-directory'; + return incomplete(reason, frame.relative || '.'); + } + if (options?.beforeEntryOpen) await options.beforeEntryOpen(frame.relative); + }; + + try { + let rootOpened: OpenedDirectory; + try { + rootOpened = await openVerifiedDirectory(repoPath); + } catch { + return incomplete('read-directory', '.'); + } + const root: WalkFrame = { + relative: '', + absolute: repoPath, + handle: rootOpened.handle, + identity: rootOpened.identity, + entries: [], + next: 0, + }; + stack.push(root); + await fillFrame(root); + + while (stack.length > 0) { + const frame = stack[stack.length - 1]; + if (frame === undefined) break; + if (frame.next >= frame.entries.length) { + stack.pop(); + await frame.handle.close().catch(() => undefined); + continue; + } + const entry = frame.entries[frame.next]; + frame.next += 1; + if (entry === undefined || !isSingleDirectoryEntry(entry.name) || entry.isSymbolicLink()) { + continue; + } + const childRelative = frame.relative ? `${frame.relative}/${entry.name}` : entry.name; + const entryPath = path.join(repoPath, childRelative); + if (entry.isDirectory()) { + if (entry.name.startsWith('.') || isIgnored(entryPath, true)) continue; + if (stack.length >= directoryDepthLimit) { + return incomplete('directory-depth', childRelative); + } + let childOpened: OpenedDirectory; + try { + childOpened = await openChildDirectory(stack, frame.handle.fd, entry.name, entryPath); + } catch { + return incomplete('read-directory', childRelative); + } + const child: WalkFrame = { + relative: childRelative, + absolute: entryPath, + handle: childOpened.handle, + identity: childOpened.identity, + entries: [], + next: 0, + }; + stack.push(child); + await fillFrame(child); + } else if (entry.isFile() && entry.name === 'pubspec.yaml' && !isIgnored(entryPath, false)) { + const manifestPath = frame.relative ? `${frame.relative}/pubspec.yaml` : 'pubspec.yaml'; + let manifestHandle: FileHandle; + try { + manifestHandle = await openChildFile(stack, frame.handle.fd, entry.name, entryPath); + } catch { + return incomplete('read-pubspec', manifestPath); + } + const read = await readManifestBounded(manifestHandle, maxManifestSize); + if (read.ok === false) return incomplete(read.reason, manifestPath); + try { + const manifest: unknown = load(read.content, { + schema: JSON_SCHEMA, + }); + if (manifest === null || typeof manifest !== 'object' || Array.isArray(manifest)) + continue; + const name = (manifest as Record).name; + if (typeof name !== 'string' || !/^[a-z_][a-z0-9_]*$/.test(name)) continue; + const manifests = manifestsByName.get(name) ?? []; + manifests.push(manifestPath); + // Two deterministic witnesses suffice to prove ambiguity. Retaining + // more would create unbounded duplicate-package dependency fanout. + manifests.sort(); + if (manifests.length > 2) manifests.length = 2; + manifestsByName.set(name, manifests); + if (packages.has(name) || ambiguous.has(name)) { + packages.delete(name); + ambiguous.add(name); + } else { + packages.set(name, frame.relative ? `${frame.relative}/lib` : 'lib'); + } + } catch { + // Invalid YAML cannot declare a package. Other valid packages remain usable. + warn('invalid-yaml', manifestPath); + } + } + } + } finally { + await closeStack(); + } + return { packages, manifestsByName }; +} diff --git a/gitnexus/src/core/ingestion/languages/dart/package-dependencies.ts b/gitnexus/src/core/ingestion/languages/dart/package-dependencies.ts new file mode 100644 index 000000000..f87489cf3 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/dart/package-dependencies.ts @@ -0,0 +1,93 @@ +import type { ParsedFile } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../../../graph/types.js'; +import { generateId } from '../../../../lib/utils.js'; +import { DART_PACKAGE_IDENTITY_REASON } from '../../scope-resolution/graph-bridge/imports-to-edges.js'; +import type { DartPackageConfig } from './package-config.js'; +import { dartPackageImportName } from './package-uri.js'; + +export { DART_PACKAGE_IDENTITY_REASON }; + +const MAX_DEPENDENCIES = 100_000; +const MAX_TRAVERSALS = 1_000_000; + +/** Package identity is an input dependency, including unresolved/ambiguous imports. */ +export function emitDartPackageDependencies( + graph: KnowledgeGraph, + parsedFiles: readonly ParsedFile[], + resolutionConfig?: unknown, +): void { + const config = resolutionConfig as DartPackageConfig | undefined; + if (!config?.manifestsByName) return; + + const consumersByManifest = new Map>(); + let dependencyCount = 0; + const addConsumer = (consumers: Set, sourceId: string): void => { + if (consumers.has(sourceId)) return; + if (++dependencyCount > MAX_DEPENDENCIES) { + throw new Error( + 'Dart package dependency limit exceeded; narrow the workspace with .gitnexusignore.', + ); + } + consumers.add(sourceId); + }; + const fileIds = new Set(parsedFiles.map((parsed) => generateId('File', parsed.filePath))); + for (const parsed of parsedFiles) { + const sourceId = generateId('File', parsed.filePath); + if (!graph.getNode(sourceId)) continue; + for (const imp of parsed.parsedImports) { + const raw = imp.targetRaw; + if (typeof raw !== 'string') continue; + const packageName = dartPackageImportName(raw); + if (packageName === null) continue; + for (const manifest of config.manifestsByName.get(packageName) ?? []) { + let consumers = consumersByManifest.get(manifest); + if (!consumers) consumersByManifest.set(manifest, (consumers = new Set())); + addConsumer(consumers, sourceId); + } + } + } + if (consumersByManifest.size === 0) return; + + // New manifests are not present in the old DB's importer closure. Carry + // identity dependencies through the freshly resolved imports as well, so + // one-hop incremental boundary expansion rewrites downstream consumers. + const importers = new Map>(); + for (const edge of graph.iterRelationshipsByType('IMPORTS')) { + if (!fileIds.has(edge.sourceId) || !fileIds.has(edge.targetId)) continue; + let sources = importers.get(edge.targetId); + if (!sources) importers.set(edge.targetId, (sources = new Set())); + sources.add(edge.sourceId); + } + let traversals = 0; + for (const [manifest, consumers] of consumersByManifest) { + const targetId = generateId('File', manifest); + if (!graph.getNode(targetId)) { + throw new Error(`Dart package manifest is missing from the indexed file set: ${manifest}`); + } + // Set iteration visits additions and deduplicates cycles without recursion. + for (const sourceId of consumers) { + for (const importer of importers.get(sourceId) ?? []) { + if (++traversals > MAX_TRAVERSALS) { + throw new Error( + 'Dart package dependency traversal limit exceeded; narrow the workspace with .gitnexusignore.', + ); + } + addConsumer(consumers, importer); + } + } + } + // Validate the complete closure before publishing any dependency edges. + for (const [manifest, consumers] of consumersByManifest) { + const targetId = generateId('File', manifest); + for (const sourceId of consumers) { + graph.addRelationship({ + id: generateId('IMPORTS', `${sourceId}->${targetId}`), + sourceId, + targetId, + type: 'IMPORTS', + confidence: 1, + reason: DART_PACKAGE_IDENTITY_REASON, + }); + } + } +} diff --git a/gitnexus/src/core/ingestion/languages/dart/package-uri.ts b/gitnexus/src/core/ingestion/languages/dart/package-uri.ts new file mode 100644 index 000000000..8afda07da --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/dart/package-uri.ts @@ -0,0 +1,13 @@ +export const DART_PACKAGE_SCHEME = 'package:'; + +/** + * Name from `package:name/library.dart`. + * A bare `package:name` or `package:name/` has no library path, so it is not + * an import target and it does not create a package-identity dependency. + */ +export function dartPackageImportName(targetRaw: string): string | null { + if (!targetRaw.startsWith(DART_PACKAGE_SCHEME)) return null; + const slash = targetRaw.indexOf('/', DART_PACKAGE_SCHEME.length); + if (slash <= DART_PACKAGE_SCHEME.length || slash === targetRaw.length - 1) return null; + return targetRaw.slice(DART_PACKAGE_SCHEME.length, slash); +} diff --git a/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts index bc46ad452..84a1ad8bf 100644 --- a/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/dart/scope-resolver.ts @@ -42,6 +42,8 @@ import { typeApplicationArguments } from '../../utils/template-arguments.js'; import type { HeritageTypeArgumentSink } from '../../scope-resolution/utils/generic-instantiation.js'; import { expandDartWildcardNames } from './expand-wildcards.js'; import { dartIsGlobalNameFallbackPlausible } from './name-fallback-visibility.js'; +import { loadDartPackageConfig } from './package-config.js'; +import { emitDartPackageDependencies } from './package-dependencies.js'; interface ClassDefRef { readonly graphId: string; @@ -199,8 +201,13 @@ export const dartScopeResolver: ScopeResolver = { languageProvider: dartProvider, importEdgeReason: 'dart-scope: import', - resolveImportTarget: (targetRaw, fromFile, allFilePaths) => - resolveDartImportTarget(targetRaw, fromFile, allFilePaths), + loadResolutionConfig: loadDartPackageConfig, + // Package dependencies use cached ParsedFile facts, never raw source text. + postExtractSourceTextPolicy: 'uncached-files', + emitPostResolutionEdges: (graph, parsedFiles, _nodeLookup, _indexes, ctx) => + emitDartPackageDependencies(graph, parsedFiles, ctx.resolutionConfig), + resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => + resolveDartImportTarget(targetRaw, fromFile, allFilePaths, resolutionConfig), // Dart `import` is whole-library: every public top-level symbol of the // target enters scope. Enumerating them lets `propagateImportedReturnTypes` diff --git a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/imports-to-edges.ts b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/imports-to-edges.ts index 47c9acf2c..f548393d3 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/imports-to-edges.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/graph-bridge/imports-to-edges.ts @@ -142,6 +142,13 @@ export const DEFERRED_IMPORT_REASON_SUFFIX = ' (deferred)'; */ export const TYPE_ONLY_IMPORT_REASON_SUFFIX = ' (type-only)'; +/** + * Dart pubspec identity edges are incremental metadata, not initialization + * edges. Kept here so the cycle query can name the reason without importing + * a language provider into MCP startup. + */ +export const DART_PACKAGE_IDENTITY_REASON = 'dart-scope: package identity dependency'; + /** * How much of an import survives to run time. Lower is stronger; a pair takes * the minimum over every edge that reaches it. See the precedence section in diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index c26923486..1c5dec3fe 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -147,6 +147,7 @@ import { scopeExtractionFailureTotal } from '../../core/ingestion/scope-resoluti import { lookupCount } from '../../core/ingestion/scope-resolution/summary-maps.js'; import { VALUE_REF_EDGE_REASON } from '../../core/ingestion/scope-resolution/value-ref-edges.js'; import { + DART_PACKAGE_IDENTITY_REASON, DEFERRED_IMPORT_REASON_SUFFIX, TYPE_ONLY_IMPORT_REASON_SUFFIX, } from '../../core/ingestion/scope-resolution/graph-bridge/imports-to-edges.js'; @@ -2944,20 +2945,25 @@ export class LocalBackend { const rows = await executeParameterized( repo.lbugPath, // A cycle here means "these modules cannot be initialized in any order". - // Only edges that force initialization count, so four kinds are excluded: + // Only edges that force initialization count, so five kinds are excluded: // Swift implicit module visibility and markdown links (never code - // dependencies at all); imports reachable only through `import()` or a + // dependencies at all); Dart package-identity edges, which point at + // pubspec.yaml so a manifest edit invalidates importers and cannot + // form an init cycle; imports reachable only through `import()` or a // function body, which are deferred by construction — deferring is the // standard idiom for BREAKING an init cycle, so counting it reports the // fix as the bug; and imports reachable only through TypeScript // `import type`, which `tsc` erases outright, so no module load exists // to order. `imports-to-edges.ts` tags the last two with // DEFERRED_IMPORT_REASON_SUFFIX / TYPE_ONLY_IMPORT_REASON_SUFFIX. + // The Dart exclusion has to sit in this filter, before LIMIT, or the + // identity edges consume the 100000-row admission cap on a cycle-free graph. `MATCH (source:File)-[r:CodeRelation]->(target:File) WHERE r.type = 'IMPORTS' AND (r.reason IS NULL OR ( r.reason <> 'swift-scope: implicit module visibility' AND r.reason <> 'markdown-link' + AND r.reason <> '${DART_PACKAGE_IDENTITY_REASON}' AND NOT r.reason ENDS WITH '${DEFERRED_IMPORT_REASON_SUFFIX}' AND NOT r.reason ENDS WITH '${TYPE_ONLY_IMPORT_REASON_SUFFIX}' )) diff --git a/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/http.dart b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/http.dart new file mode 100644 index 000000000..4393bc505 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/http.dart @@ -0,0 +1,4 @@ +void localHttpDecoy() {} +void loadOwn() {} +void loadData() {} +void loadRelative() {} diff --git a/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/main.dart b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/main.dart new file mode 100644 index 000000000..5612f961e --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/main.dart @@ -0,0 +1,12 @@ +import 'package:app/models.dart'; +import 'package:data/models.dart'; +import 'package:http/http.dart' as http; +import 'package:app/tool/run.dart' as tool; +import 'dart:io'; +import './relative.dart'; + +void main() { + loadOwn(); + loadData(); + loadRelative(); +} diff --git a/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/models.dart b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/models.dart new file mode 100644 index 000000000..edcd68fe2 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/models.dart @@ -0,0 +1 @@ +void loadOwn() {} diff --git a/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/relative.dart b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/relative.dart new file mode 100644 index 000000000..98b0f0f3f --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/lib/relative.dart @@ -0,0 +1 @@ +void loadRelative() {} diff --git a/gitnexus/test/fixtures/lang-resolution/dart-package-imports/packages/data/lib/models.dart b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/packages/data/lib/models.dart new file mode 100644 index 000000000..72512f1ac --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/packages/data/lib/models.dart @@ -0,0 +1 @@ +void loadData() {} diff --git a/gitnexus/test/fixtures/lang-resolution/dart-package-imports/packages/data/pubspec.yaml b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/packages/data/pubspec.yaml new file mode 100644 index 000000000..8eeadd27e --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/packages/data/pubspec.yaml @@ -0,0 +1 @@ +name: data diff --git a/gitnexus/test/fixtures/lang-resolution/dart-package-imports/pubspec.yaml b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/pubspec.yaml new file mode 100644 index 000000000..f628ca137 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/pubspec.yaml @@ -0,0 +1,5 @@ +name: app +dependencies: + http: ^1.0.0 + data: + path: packages/data diff --git a/gitnexus/test/fixtures/lang-resolution/dart-package-imports/tool/run.dart b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/tool/run.dart new file mode 100644 index 000000000..50ca7fe28 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/dart-package-imports/tool/run.dart @@ -0,0 +1 @@ +void nonLibraryDecoy() {} diff --git a/gitnexus/test/integration/dart-import-index-reuse.test.ts b/gitnexus/test/integration/dart-import-index-reuse.test.ts index e868f3d9f..7ca667c5c 100644 --- a/gitnexus/test/integration/dart-import-index-reuse.test.ts +++ b/gitnexus/test/integration/dart-import-index-reuse.test.ts @@ -29,10 +29,7 @@ const { resolveImportTarget } = dartScopeResolver; const FROM_FILE = 'lib/main.dart'; /** - * A synthetic Dart package: many library files under `lib/src/`, plus the two - * targets the `package:` leg addresses — one reachable as `lib/` and one - * only as bare ``, which is the second candidate and therefore the leg - * that used to run a second full scan for every external import. + * A synthetic Dart package with library and non-library targets. */ function buildWorkspace(fileCount: number): CountingSet { const files: string[] = []; @@ -52,13 +49,10 @@ describe('Dart import resolution — index reuse across imports (#2879)', () => const resolved: (string | readonly string[] | null)[] = []; for (let i = 0; i < 200; i++) { - // Three shapes: an in-package hit through `lib/`, a bare-`` hit - // that only the SECOND candidate answers, and an external package whose - // two candidates both miss — the case that used to cost two full - // workspace scans per import. - resolved.push(resolveImportTarget('package:app/models.dart', FROM_FILE, files)); - resolved.push(resolveImportTarget('package:app/tool/generate.dart', FROM_FILE, files)); - resolved.push(resolveImportTarget(`package:vendor${i}/ghost${i}.dart`, FROM_FILE, files)); + // Exact relative hits plus misses that must reuse the suffix index. + resolved.push(resolveImportTarget('./models.dart', FROM_FILE, files)); + resolved.push(resolveImportTarget('../tool/generate.dart', FROM_FILE, files)); + resolved.push(resolveImportTarget(`vendor${i}/ghost${i}.dart`, FROM_FILE, files)); } expect(files.scans).toBe(1); @@ -74,8 +68,8 @@ describe('Dart import resolution — index reuse across imports (#2879)', () => expectDistinctFileSetsGetOwnIndex({ resolveImportTarget, buildWorkspace: () => buildWorkspace(20), - targetRaw: 'package:app/models.dart', - fromFile: FROM_FILE, + targetRaw: 'models.dart', + fromFile: 'main.dart', resolutionConfig: undefined, expected: 'lib/models.dart', expectedScans: 1, @@ -85,15 +79,14 @@ describe('Dart import resolution — index reuse across imports (#2879)', () => it('still resolves real imports correctly (the perf test is not vacuous)', () => { const files = buildWorkspace(5); - // `package:` leg, first candidate: `lib/`. - expect(resolveImportTarget('package:app/models.dart', FROM_FILE, files)).toBe( + const config = { packages: new Map([['app', 'lib']]) }; + expect(resolveImportTarget('package:app/models.dart', FROM_FILE, files, config)).toBe( 'lib/models.dart', ); - // `package:` leg, second candidate: bare ``, reached only after - // `lib/` misses entirely. - expect(resolveImportTarget('package:app/tool/generate.dart', FROM_FILE, files)).toBe( - 'tool/generate.dart', - ); + // Package imports cannot reach files outside their declared lib directory. + expect( + resolveImportTarget('package:app/tool/generate.dart', FROM_FILE, files, config), + ).toBeNull(); // Relative import against the importer's directory. expect(resolveImportTarget('src/util.dart', FROM_FILE, files)).toBe('lib/src/util.dart'); expect(resolveImportTarget('./src/util.dart', FROM_FILE, files)).toBe('lib/src/util.dart'); diff --git a/gitnexus/test/integration/resolvers/dart.test.ts b/gitnexus/test/integration/resolvers/dart.test.ts index e362edc33..2f7b00c04 100644 --- a/gitnexus/test/integration/resolvers/dart.test.ts +++ b/gitnexus/test/integration/resolvers/dart.test.ts @@ -23,6 +23,7 @@ import { loadLanguage, } from '../../../src/core/tree-sitter/parser-loader.js'; import { SupportedLanguages } from '../../../src/config/supported-languages.js'; +import { pubspecWalkAnchored } from '../../../src/core/ingestion/languages/dart/package-config.js'; // isLanguageAvailable only checks whether the module loaded — it does NOT verify // that the native binary works at runtime (tree-sitter-dart can fail on setLanguage). @@ -37,6 +38,43 @@ if (dartAvailable) { } } +describe.skipIf(!dartAvailable || !pubspecWalkAnchored())( + 'Dart pubspec package identity (#2963)', + () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'dart-package-imports'), () => {}); + }, 60000); + + it('emits declared imports and their package identity dependencies', () => { + const imports = getRelationships(result, 'IMPORTS') + .filter((edge) => edge.sourceFilePath === 'lib/main.dart') + .map((edge) => edge.targetFilePath) + .sort(); + expect(imports).toEqual([ + 'lib/models.dart', + 'lib/relative.dart', + 'packages/data/lib/models.dart', + 'packages/data/pubspec.yaml', + 'pubspec.yaml', + ]); + }); + + it.each([ + ['loadOwn', 'lib/models.dart'], + ['loadData', 'packages/data/lib/models.dart'], + ['loadRelative', 'lib/relative.dart'], + ])('resolves %s in the correct library', (name, file) => { + const calls = getRelationships(result, 'CALLS').filter( + (edge) => edge.sourceFilePath === 'lib/main.dart' && edge.target === name, + ); + expect(calls).toHaveLength(1); + expect(calls[0]?.targetFilePath).toBe(file); + }); + }, +); + // ── Phase 8: Field-type resolution ────────────────────────────────────── describe.skipIf(!dartAvailable)('Dart field-type resolution', () => { diff --git a/gitnexus/test/unit/calltool-dispatch.test.ts b/gitnexus/test/unit/calltool-dispatch.test.ts index 85fbbccea..6a0a98d4a 100644 --- a/gitnexus/test/unit/calltool-dispatch.test.ts +++ b/gitnexus/test/unit/calltool-dispatch.test.ts @@ -123,6 +123,7 @@ import { DEFERRED_IMPORT_REASON_SUFFIX, TYPE_ONLY_IMPORT_REASON_SUFFIX, } from '../../src/core/ingestion/scope-resolution/graph-bridge/imports-to-edges.js'; +import { DART_PACKAGE_IDENTITY_REASON } from '../../src/core/ingestion/languages/dart/package-dependencies.js'; // ─── Helpers ───────────────────────────────────────────────────────── @@ -787,6 +788,12 @@ describe('LocalBackend.callTool', () => { `NOT r.reason ENDS WITH '${TYPE_ONLY_IMPORT_REASON_SUFFIX}'`, ); expect(reasonNullAlternativeOf(query)).toContain("r.reason <> 'markdown-link'"); + // Package-identity edges are IMPORTS metadata for incremental + // invalidation. They must be excluded inside this same group, before + // LIMIT, or they fill the 100000-row cap on a cycle-free graph. + expect(reasonNullAlternativeOf(query)).toContain( + `r.reason <> '${DART_PACKAGE_IDENTITY_REASON}'`, + ); expect(query).toContain('LIMIT 100001'); }); diff --git a/gitnexus/test/unit/dart-package-dependencies.test.ts b/gitnexus/test/unit/dart-package-dependencies.test.ts new file mode 100644 index 000000000..1151f284d --- /dev/null +++ b/gitnexus/test/unit/dart-package-dependencies.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, it } from 'vitest'; +import type { ParsedFile, ScopeId } from 'gitnexus-shared'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { generateId } from '../../src/lib/utils.js'; +import { + DART_PACKAGE_IDENTITY_REASON, + emitDartPackageDependencies, +} from '../../src/core/ingestion/languages/dart/package-dependencies.js'; +import { computeEffectiveWriteSet } from '../../src/core/incremental/subgraph-extract.js'; +import { GraphEmitSink } from '../../src/core/lbug/graph-emit-sink.js'; + +function consumer(filePath: string, targets: string[]): ParsedFile { + return { + filePath, + moduleScope: `module:${filePath}` as ScopeId, + scopes: [], + parsedImports: targets.map((targetRaw) => ({ kind: 'wildcard', targetRaw })), + localDefs: [], + referenceSites: [], + }; +} + +function graphWithFiles(paths: string[]) { + const graph = createKnowledgeGraph(); + for (const filePath of paths) { + graph.addNode({ + id: generateId('File', filePath), + label: 'File', + properties: { name: filePath, filePath }, + }); + } + return graph; +} + +describe('Dart package identity dependencies', () => { + it.each([ + { files: 400, packages: 251, dense: false, limit: 'dependency limit' }, + { files: 150, packages: 100, dense: true, limit: 'dependency traversal limit' }, + ])( + 'fails explicitly before partial emission at the $limit', + ({ files, packages, dense, limit }) => { + const paths = Array.from({ length: files }, (_, i) => `f${i}.dart`); + const manifests = Array.from({ length: packages }, (_, i) => `p${i}/pubspec.yaml`); + const graph = graphWithFiles([...paths, ...manifests]); + for (let i = 0; i < files; i++) { + const targets = dense ? paths : [paths[(i + 1) % files]]; + for (const target of targets) { + graph.addRelationship({ + id: `${paths[i]}->${target}`, + sourceId: `File:${paths[i]}`, + targetId: `File:${target}`, + type: 'IMPORTS', + confidence: 1, + reason: 'dart-scope: import', + }); + } + } + const originalCount = graph.relationshipCount; + expect(() => + emitDartPackageDependencies( + graph, + paths.map((file, i) => consumer(file, [`package:p${i % packages}/model.dart`])), + { + packages: new Map(), + manifestsByName: new Map(manifests.map((file, i) => [`p${i}`, [file]])), + }, + ), + ).toThrow(`Dart package ${limit} exceeded`); + expect(graph.relationshipCount).toBe(originalCount); + }, + ); + + it('preserves transitive dependencies and idempotence through the streaming sink', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'dart-package-stream-')); + const graph = graphWithFiles(['a.dart', 'b.dart', 'pubspec.yaml']); + const sink = new GraphEmitSink(graph, path.join(root, 'csv')); + try { + sink.beginStreaming(); + sink.addRelationship({ + id: 'IMPORTS:File:b.dart->File:a.dart', + sourceId: 'File:b.dart', + targetId: 'File:a.dart', + type: 'IMPORTS', + confidence: 1, + reason: 'dart-scope: import', + }); + const parsed = [ + consumer('a.dart', ['package:app/model.dart']), + consumer('b.dart', ['./a.dart']), + ]; + const config = { packages: new Map(), manifestsByName: new Map([['app', ['pubspec.yaml']]]) }; + emitDartPackageDependencies(sink, parsed, config); + emitDartPackageDependencies(sink, parsed, config); + const edges: string[] = []; + sink.forEachRelationshipFields((source, target, type, _confidence, reason) => { + if (type === 'IMPORTS') edges.push(`${source}->${target}:${reason}`); + }); + expect(edges.sort()).toEqual([ + `File:a.dart->File:pubspec.yaml:${DART_PACKAGE_IDENTITY_REASON}`, + 'File:b.dart->File:a.dart:dart-scope: import', + `File:b.dart->File:pubspec.yaml:${DART_PACKAGE_IDENTITY_REASON}`, + ]); + const manifest = sink.finalize(); + expect(manifest.totalRows).toBe(3); + const csv = manifest.relsByPair.get('File|File'); + expect(csv?.rows).toBe(3); + if (!csv) throw new Error('Missing File-to-File CSV'); + expect((await readFile(csv.csvPath, 'utf8')).trim().split('\n')).toHaveLength(4); + expect(graph.relationshipCount).toBe(0); + } finally { + sink.close(); + await rm(root, { recursive: true, force: true }); + } + }); + + it('tracks all duplicate candidates even when no package resolves, idempotently', () => { + const graph = graphWithFiles(['main.dart', 'pubspec.yaml', 'nested/pubspec.yaml']); + const parsed = [consumer('main.dart', ['package:app/a.dart', 'package:app/b.dart'])]; + const config = { + packages: new Map(), + manifestsByName: new Map([['app', ['pubspec.yaml', 'nested/pubspec.yaml']]]), + }; + emitDartPackageDependencies(graph, parsed, config); + emitDartPackageDependencies(graph, parsed, config); + expect(graph.relationships.map((edge) => [edge.targetId, edge.reason])).toEqual([ + ['File:pubspec.yaml', DART_PACKAGE_IDENTITY_REASON], + ['File:nested/pubspec.yaml', DART_PACKAGE_IDENTITY_REASON], + ]); + expect(computeEffectiveWriteSet(graph, new Set(['nested/pubspec.yaml']))).toEqual( + new Set(['nested/pubspec.yaml', 'main.dart']), + ); + }); + + it('ignores a package URI that has no library path', () => { + const graph = graphWithFiles(['main.dart', 'pubspec.yaml']); + emitDartPackageDependencies( + graph, + [consumer('main.dart', ['package:app', 'package:', 'package:app/'])], + { + packages: new Map(), + manifestsByName: new Map([['app', ['pubspec.yaml']]]), + }, + ); + expect(graph.relationships).toEqual([]); + }); + + it('does not connect unrelated packages, relative imports, SDK imports, or files without imports', () => { + const graph = graphWithFiles([ + 'a.dart', + 'b.dart', + 'c.dart', + 'pubspec.yaml', + 'other/pubspec.yaml', + ]); + emitDartPackageDependencies( + graph, + [ + consumer('a.dart', ['package:app/missing.dart', 'package:external/http.dart']), + consumer('b.dart', ['./a.dart', 'dart:core']), + consumer('c.dart', []), + ], + { + packages: new Map(), + manifestsByName: new Map([ + ['app', ['pubspec.yaml']], + ['other', ['other/pubspec.yaml']], + ]), + }, + ); + expect(graph.relationships.map((edge) => [edge.sourceId, edge.targetId])).toEqual([ + ['File:a.dart', 'File:pubspec.yaml'], + ]); + }); + + it('includes transitive consumers in fresh-manifest invalidation and terminates on cycles', () => { + const graph = graphWithFiles(['a.dart', 'b.dart', 'c.dart', 'pubspec.yaml']); + for (const [source, target] of [ + ['b.dart', 'a.dart'], + ['c.dart', 'b.dart'], + ['a.dart', 'c.dart'], + ]) { + graph.addRelationship({ + id: `${source}->${target}`, + sourceId: `File:${source}`, + targetId: `File:${target}`, + type: 'IMPORTS', + confidence: 1, + reason: 'dart-scope: import', + }); + } + emitDartPackageDependencies( + graph, + [ + consumer('a.dart', ['package:app/a.dart']), + consumer('b.dart', ['./a.dart']), + consumer('c.dart', ['./b.dart']), + ], + { packages: new Map(), manifestsByName: new Map([['app', ['pubspec.yaml']]]) }, + ); + expect( + graph.relationships + .filter((edge) => edge.reason === DART_PACKAGE_IDENTITY_REASON) + .map((edge) => [edge.sourceId, edge.targetId]), + ).toEqual([ + ['File:a.dart', 'File:pubspec.yaml'], + ['File:b.dart', 'File:pubspec.yaml'], + ['File:c.dart', 'File:pubspec.yaml'], + ]); + expect(computeEffectiveWriteSet(graph, new Set(['pubspec.yaml']))).toEqual( + new Set(['pubspec.yaml', 'a.dart', 'b.dart', 'c.dart']), + ); + }); + + it('refuses to persist dependencies on manifests absent from the indexed graph', () => { + const graph = graphWithFiles(['a.dart']); + expect(() => + emitDartPackageDependencies(graph, [consumer('a.dart', ['package:app/a.dart'])], { + packages: new Map(), + manifestsByName: new Map([['app', ['pubspec.yaml']]]), + }), + ).toThrow('Dart package manifest is missing from the indexed file set: pubspec.yaml'); + expect(graph.relationships).toEqual([]); + }); + + it('emits only matching dependencies in a many-package workspace', () => { + const paths = Array.from({ length: 200 }, (_, i) => `p${i}/pubspec.yaml`); + const graph = graphWithFiles([...paths, 'main.dart']); + emitDartPackageDependencies(graph, [consumer('main.dart', ['package:p199/a.dart'])], { + packages: new Map(), + manifestsByName: new Map(paths.map((filePath, i) => [`p${i}`, [filePath]])), + }); + expect(graph.relationships.map((edge) => edge.targetId)).toEqual(['File:p199/pubspec.yaml']); + }); +}); diff --git a/gitnexus/test/unit/dart-package-imports.test.ts b/gitnexus/test/unit/dart-package-imports.test.ts new file mode 100644 index 000000000..0e33bc463 --- /dev/null +++ b/gitnexus/test/unit/dart-package-imports.test.ts @@ -0,0 +1,561 @@ +import { execFileSync } from 'node:child_process'; +import { afterEach, describe, expect, it } from 'vitest'; +import { constants } from 'node:fs'; +import { + mkdtemp, + mkdir, + rm, + symlink, + writeFile, + chmod, + rename, + open, + readdir, +} from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { dartScopeResolver } from '../../src/core/ingestion/languages/dart/scope-resolver.js'; +import { + loadDartPackageConfig, + readDirectoryNoFollow, + directoryOpenFlags, + descriptorDirectoryPath, + descriptorEntryPath, + pubspecWalkAnchored, +} from '../../src/core/ingestion/languages/dart/package-config.js'; +import { CountingSet } from '../helpers/counting-file-set.js'; +import { _captureLogger } from '../../src/core/logger.js'; + +const files = new Set(['lib/main.dart', 'lib/http.dart', 'lib/models.dart', 'tool/run.dart']); +const config = { packages: new Map([['app', 'lib']]) }; + +describe('Dart package identity (#2963)', () => { + it('does not resolve a package URI that has no library path', () => { + expect( + dartScopeResolver.resolveImportTarget('package:app', 'lib/main.dart', files, config), + ).toBeNull(); + expect( + dartScopeResolver.resolveImportTarget('package:app/', 'lib/main.dart', files, config), + ).toBeNull(); + }); + + it('does not resolve a pub dependency to a same-named local file', () => { + expect( + dartScopeResolver.resolveImportTarget( + 'package:http/http.dart', + 'lib/main.dart', + files, + config, + ), + ).toBeNull(); + }); + + it('resolves this package through its declared lib directory', () => { + expect( + dartScopeResolver.resolveImportTarget( + 'package:app/models.dart', + 'lib/main.dart', + files, + config, + ), + ).toBe('lib/models.dart'); + }); + + it('does not guess package identity when no pubspec config is available', () => { + expect( + dartScopeResolver.resolveImportTarget('package:app/models.dart', 'lib/main.dart', files), + ).toBeNull(); + }); + + it('does not fall back to non-library files', () => { + expect( + dartScopeResolver.resolveImportTarget( + 'package:app/tool/run.dart', + 'lib/main.dart', + files, + config, + ), + ).toBeNull(); + }); + + it.each([ + '', + '../http.dart', + 'src/../../http.dart', + '/http.dart', + 'src\\http.dart', + '%2e%2e/http.dart', + 'http.dart?q', + 'http.dart#part', + ])('rejects unsupported package paths: %s', (target) => { + expect( + dartScopeResolver.resolveImportTarget( + `package:app/${target}`, + 'lib/main.dart', + files, + config, + ), + ).toBeNull(); + }); + + it('uses exact package roots even with earlier same-suffix files', () => { + const workspace = new Set([ + 'decoy/lib/models.dart', + 'packages/data/lib/models.dart', + 'lib/models.dart', + ]); + const monorepo = { + packages: new Map([ + ['app', 'lib'], + ['data', 'packages/data/lib'], + ]), + }; + expect( + dartScopeResolver.resolveImportTarget( + 'package:data/models.dart', + 'lib/main.dart', + workspace, + monorepo, + ), + ).toBe('packages/data/lib/models.dart'); + expect( + dartScopeResolver.resolveImportTarget( + 'package:app/models.dart', + 'packages/data/lib/main.dart', + workspace, + monorepo, + ), + ).toBe('lib/models.dart'); + expect( + dartScopeResolver.resolveImportTarget( + 'package:missing/models.dart', + 'lib/main.dart', + workspace, + monorepo, + ), + ).toBeNull(); + }); + + it('does not suffix-match a missing file in a known package', () => { + expect( + dartScopeResolver.resolveImportTarget( + 'package:app/models.dart', + 'lib/main.dart', + new Set(['other/lib/models.dart']), + config, + ), + ).toBeNull(); + }); + + it('uses no workspace scans for package hits or misses', () => { + const workspace = new CountingSet(files); + for (let i = 0; i < 200; i++) { + expect( + dartScopeResolver.resolveImportTarget( + 'package:app/models.dart', + 'lib/main.dart', + workspace, + config, + ), + ).toBe('lib/models.dart'); + expect( + dartScopeResolver.resolveImportTarget( + `package:external${i}/http.dart`, + 'lib/main.dart', + workspace, + config, + ), + ).toBeNull(); + } + expect(workspace.scans).toBe(0); + }); + + it('still ignores SDK imports and resolves relative paths without config', () => { + expect( + dartScopeResolver.resolveImportTarget('dart:core', 'lib/main.dart', files, config), + ).toBeNull(); + expect(dartScopeResolver.resolveImportTarget('./models.dart', 'lib/main.dart', files)).toBe( + 'lib/models.dart', + ); + expect(dartScopeResolver.resolveImportTarget('../tool/run.dart', 'lib/main.dart', files)).toBe( + 'tool/run.dart', + ); + }); +}); + +describe('directory no-follow flags', () => { + it('does not drop O_NOFOLLOW from directory opens', () => { + if (typeof constants.O_NOFOLLOW !== 'number' || constants.O_NOFOLLOW === 0) { + expect(() => directoryOpenFlags()).toThrow(/O_NOFOLLOW is unavailable/); + return; + } + expect(directoryOpenFlags() & constants.O_NOFOLLOW).toBe(constants.O_NOFOLLOW); + }); + + it('anchors pubspec discovery only where a no-follow walk exists', () => { + const canAnchor = + (process.platform === 'linux' || process.platform === 'darwin') && + typeof constants.O_NOFOLLOW === 'number' && + constants.O_NOFOLLOW !== 0; + expect(pubspecWalkAnchored()).toBe(canAnchor); + }); +}); + +describe.skipIf(!pubspecWalkAnchored())('Dart pubspec package discovery', () => { + const roots: string[] = []; + afterEach(async () => { + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); + }); + + async function fixture(manifests: Record): Promise { + const root = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-dart-pubspec-')); + roots.push(root); + for (const [relative, content] of Object.entries(manifests)) { + const destination = path.join(root, relative); + await mkdir(path.dirname(destination), { recursive: true }); + await writeFile(destination, content); + } + return root; + } + + it('loads root and nested package names through the production hook', async () => { + const root = await fixture({ + 'pubspec.yaml': 'name: "app" # root package\ndependencies:\n http: ^1.0.0\n', + 'packages/data/pubspec.yaml': 'name: data\n', + 'packages/unnamed/pubspec.yaml': 'description: no package name\n', + }); + const loaded = await dartScopeResolver.loadResolutionConfig?.(root); + expect(loaded).toMatchObject({ + packages: new Map([ + ['app', 'lib'], + ['data', 'packages/data/lib'], + ]), + }); + expect( + dartScopeResolver.resolveImportTarget( + 'package:data/models.dart', + 'lib/main.dart', + new Set(['packages/data/lib/models.dart']), + loaded, + ), + ).toBe('packages/data/lib/models.dart'); + }); + + it('suppresses duplicate names even across three packages', async () => { + const root = await fixture({ + 'pubspec.yaml': 'name: app', + 'a/pubspec.yaml': 'name: repeated', + 'b/pubspec.yaml': 'name: repeated', + 'c/pubspec.yaml': 'name: repeated', + }); + const loaded = await loadDartPackageConfig(root); + expect(loaded.packages).toEqual(config.packages); + expect(loaded.manifestsByName.get('repeated')).toEqual(['a/pubspec.yaml', 'b/pubspec.yaml']); + }); + + it('accepts an underscore-prefixed package name', async () => { + const root = await fixture({ 'pubspec.yaml': 'name: _app' }); + expect((await loadDartPackageConfig(root)).packages).toEqual(new Map([['_app', 'lib']])); + }); + + it.each(['.gitignore', '.gitnexusignore'])( + 'ignores duplicate names in directories excluded by %s', + async (ignoreFile) => { + const root = await fixture({ + 'pubspec.yaml': 'name: app', + [ignoreFile]: 'backup/\n', + 'backup/pubspec.yaml': 'name: app', + }); + expect((await loadDartPackageConfig(root)).packages).toEqual(config.packages); + }, + ); + + it('honors explicitly re-included package directories', async () => { + const root = await fixture({ + 'pubspec.yaml': 'name: app', + '.gitnexusignore': '!vendor/\n', + 'vendor/pubspec.yaml': 'name: local_vendor', + }); + expect(await loadDartPackageConfig(root)).toMatchObject({ + packages: new Map([ + ['app', 'lib'], + ['local_vendor', 'vendor/lib'], + ]), + }); + }); + + it.each(['.gitignore', '.gitnexusignore'])( + 'does not read manifests excluded individually by %s', + async (ignoreFile) => { + const root = await fixture({ + 'pubspec.yaml': 'name: app', + [ignoreFile]: 'backup/pubspec.yaml\nbroken/pubspec.yaml\n', + 'backup/pubspec.yaml': 'name: app', + 'broken/pubspec.yaml': 'name: [invalid', + }); + expect((await loadDartPackageConfig(root)).packages).toEqual(config.packages); + }, + ); + + it('excludes hidden directories just like the production file scanner', async () => { + const root = await fixture({ + 'pubspec.yaml': 'name: app', + '.backup/pubspec.yaml': 'name: app', + '.broken/pubspec.yaml': 'name: [invalid', + }); + expect((await loadDartPackageConfig(root)).packages).toEqual(config.packages); + }); + + it('reports invalid YAML without exposing contents or discarding valid packages', async () => { + const root = await fixture({ + 'pubspec.yaml': 'name: app', + 'nested/pubspec.yaml': 'name: [private-manifest-content', + }); + const capture = _captureLogger(); + try { + expect((await loadDartPackageConfig(root)).packages).toEqual(config.packages); + expect(capture.records()).toEqual([ + expect.objectContaining({ + level: 40, + reason: 'invalid-yaml', + relativePath: 'nested/pubspec.yaml', + msg: 'Dart pubspec discovery could not read a valid package declaration.', + }), + ]); + expect(capture.text()).not.toContain('private-manifest-content'); + expect(capture.text()).not.toContain(root); + } finally { + capture.restore(); + } + }); + + it.each(['name: [bad', 'name: one\nname: two', '!!js/function function() {}'])( + 'does not interpret invalid YAML as a package declaration: %s', + async (manifest) => { + const root = await fixture({ 'pubspec.yaml': 'name: app', 'nested/pubspec.yaml': manifest }); + expect((await loadDartPackageConfig(root)).packages).toEqual(config.packages); + }, + ); + + it.each(['null', '- name: app', 'name: 42', 'name: ../app', 'description: app'])( + 'does not infer a name from %s', + async (manifest) => { + expect( + (await loadDartPackageConfig(await fixture({ 'pubspec.yaml': manifest }))).packages.size, + ).toBe(0); + }, + ); + + it('does not read generated or installed pubspecs', async () => { + const root = await fixture({ + 'pubspec.yaml': 'name: app', + '.dart_tool/pubspec.yaml': 'name: app', + '.pub-cache/pubspec.yaml': 'name: app', + 'node_modules/dependency/pubspec.yaml': 'name: app', + }); + expect((await loadDartPackageConfig(root)).packages).toEqual(config.packages); + }); + + it('does not follow directory links outside the repository', async () => { + const outside = await fixture({ 'pubspec.yaml': 'name: app' }); + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + await symlink( + outside, + path.join(root, 'linked'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + expect((await loadDartPackageConfig(root)).packages).toEqual(config.packages); + }); + + it('returns no packages when no pubspec is present', async () => { + expect((await loadDartPackageConfig(await fixture({}))).packages.size).toBe(0); + }); + + it('refuses oversized manifests instead of parsing an unbounded document', async () => { + const root = await fixture({ 'pubspec.yaml': `name: app\n#${'x'.repeat(1024 * 1024)}` }); + await expect(loadDartPackageConfig(root)).rejects.toThrow( + 'Dart pubspec discovery failed (manifest-size)', + ); + }); + + it('refuses an incomplete scan rather than persisting untracked identity changes', async () => { + const root = await fixture({}); + await expect(loadDartPackageConfig(path.join(root, 'missing'))).rejects.toThrow( + 'Dart pubspec discovery failed (read-directory)', + ); + }); + + it('fails closed when the directory walk exceeds its budget', async () => { + const root = await fixture({ + 'pubspec.yaml': 'name: app', + 'nested/pubspec.yaml': 'name: data', + }); + await expect(loadDartPackageConfig(root, { directoryLimit: 1 })).rejects.toThrow( + 'Dart pubspec discovery failed (directory-limit)', + ); + }); + + it('fails closed before one directory listing is unbounded', async () => { + const root = await fixture({ 'pubspec.yaml': 'name: app', 'extra.txt': 'x' }); + await expect(loadDartPackageConfig(root, { directoryEntryLimit: 1 })).rejects.toThrow( + 'Dart pubspec discovery failed (directory-entries)', + ); + }); + + it('fails closed before a deep chain holds one descriptor per level', async () => { + const root = await fixture({ 'a/b/pubspec.yaml': 'name: data' }); + await expect(loadDartPackageConfig(root, { directoryDepthLimit: 2 })).rejects.toThrow( + 'Dart pubspec discovery failed (directory-depth): a/b', + ); + }); + + it('still reads a manifest when the open-directory cap is exactly the nesting', async () => { + const root = await fixture({ 'a/pubspec.yaml': 'name: data' }); + expect((await loadDartPackageConfig(root, { directoryDepthLimit: 2 })).packages).toEqual( + new Map([['data', 'a/lib']]), + ); + }); + + it('fails closed when ignore rules cannot be read', async () => { + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + await mkdir(path.join(root, '.gitignore')); + await expect(loadDartPackageConfig(root)).rejects.toThrow( + 'Dart pubspec discovery failed (ignore-rules)', + ); + }); + + // Windows does not enforce POSIX mode bits, and root bypasses them, so chmod + // cannot make this read fail on either. + it.skipIf(process.platform === 'win32' || process.getuid?.() === 0)( + 'fails closed when a pubspec cannot be read', + async () => { + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + await chmod(path.join(root, 'pubspec.yaml'), 0o000); + await expect(loadDartPackageConfig(root)).rejects.toThrow( + 'Dart pubspec discovery failed (read-pubspec)', + ); + }, + ); + + it('does not block when a listed pubspec is replaced by a fifo', async () => { + const root = await fixture({ 'pubspec.yaml': 'name: app' }); + const manifest = path.join(root, 'pubspec.yaml'); + await expect( + loadDartPackageConfig(root, { + beforeEntryOpen: async (relative) => { + if (relative !== '') return; + await rm(manifest); + execFileSync('mkfifo', [manifest]); + }, + }), + ).rejects.toThrow('Dart pubspec discovery failed (read-pubspec)'); + }, 3_000); + + it.skipIf(process.platform === 'win32')( + 'does not follow a symlinked pubspec into another tree', + async () => { + const outside = await fixture({ 'pubspec.yaml': 'name: foreign' }); + const root = await fixture({ 'packages/data/pubspec.yaml': 'name: data' }); + await symlink(path.join(outside, 'pubspec.yaml'), path.join(root, 'pubspec.yaml')); + expect((await loadDartPackageConfig(root)).packages).toEqual( + new Map([['data', 'packages/data/lib']]), + ); + }, + ); + + it('does not follow a listed directory replaced by a symlink on macOS', async () => { + if (process.platform !== 'darwin') return; + const outside = await fixture({ + 'pubspec.yaml': 'name: foreign', + 'nested/pubspec.yaml': 'name: foreign', + }); + const root = await fixture({ + 'pkg/pubspec.yaml': 'name: data', + 'pkg/nested/pubspec.yaml': 'name: nested_data', + }); + const pkg = path.join(root, 'pkg'); + await expect( + loadDartPackageConfig(root, { + beforeEntryOpen: async (relative) => { + if (relative !== 'pkg') return; + await rename(pkg, path.join(root, 'pkg-moved')); + await symlink(outside, pkg, 'dir'); + }, + }), + ).rejects.toThrow(/Dart pubspec discovery failed \((read-directory|read-pubspec)\)/); + }); + + it('reads listed manifests from the opened directory inode after that path is replaced', async () => { + if (descriptorEntryPath(0, 'pubspec.yaml') === null) return; + const outside = await fixture({ + 'pubspec.yaml': 'name: foreign', + 'nested/pubspec.yaml': 'name: foreign', + }); + const root = await fixture({ + 'pkg/pubspec.yaml': 'name: data', + 'pkg/nested/pubspec.yaml': 'name: nested_data', + }); + const pkg = path.join(root, 'pkg'); + const loaded = await loadDartPackageConfig(root, { + beforeEntryOpen: async (relative) => { + if (relative !== 'pkg') return; + await rename(pkg, path.join(root, 'pkg-moved')); + await symlink(outside, pkg, 'dir'); + }, + }); + expect(loaded.packages).toEqual( + new Map([ + ['data', 'pkg/lib'], + ['nested_data', 'pkg/nested/lib'], + ]), + ); + }); + + it('lists the opened directory inode after its path becomes a symlink', async () => { + const listingRoot = descriptorDirectoryPath(0); + if (listingRoot === null) return; + const outside = await fixture({ 'outside.txt': 'out' }); + const root = await fixture({}); + const real = path.join(root, 'real'); + await mkdir(real); + await writeFile(path.join(real, 'inside.txt'), 'in'); + const handle = await open(real, directoryOpenFlags()); + try { + const listed = descriptorDirectoryPath(handle.fd); + if (listed === null) throw new Error('descriptor listing path missing'); + await rename(real, path.join(root, 'real-moved')); + await symlink(outside, real, process.platform === 'win32' ? 'junction' : 'dir'); + await expect(readDirectoryNoFollow(real)).rejects.toThrow(); + expect(await readdir(listed)).toEqual(['inside.txt']); + } finally { + await handle.close(); + } + }); +}); + +describe('directory symlink refusal', () => { + const roots: string[] = []; + afterEach(async () => { + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); + }); + + it('refuses a directory symlink instead of listing its target', async () => { + const outside = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-dart-pubspec-')); + const root = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-dart-pubspec-')); + roots.push(outside, root); + await writeFile(path.join(outside, 'secret.txt'), 'name: foreign'); + const link = path.join(root, 'linked'); + await symlink(outside, link, process.platform === 'win32' ? 'junction' : 'dir'); + await expect(readDirectoryNoFollow(link)).rejects.toThrow(); + }); + + it.skipIf(pubspecWalkAnchored())( + 'does not discover packages when the walk cannot honor no-follow', + async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'gitnexus-dart-pubspec-')); + roots.push(root); + await writeFile(path.join(root, 'pubspec.yaml'), 'name: app\n'); + expect((await loadDartPackageConfig(root)).packages.size).toBe(0); + }, + ); +}); diff --git a/gitnexus/test/unit/incremental-orchestration.test.ts b/gitnexus/test/unit/incremental-orchestration.test.ts index df24e963d..742aac8d8 100644 --- a/gitnexus/test/unit/incremental-orchestration.test.ts +++ b/gitnexus/test/unit/incremental-orchestration.test.ts @@ -43,6 +43,13 @@ import { } from '../helpers/embedding-seed.js'; import { CLASS_FRAMEWORK_ANNOTATIONS_FEATURE } from '../../src/core/analysis-features.js'; import { SCHEMA_FINGERPRINT } from '../../src/core/lbug/schema.js'; +import { + isLanguageAvailable, + loadParser, + loadLanguage, +} from '../../src/core/tree-sitter/parser-loader.js'; +import { SupportedLanguages } from '../../src/config/supported-languages.js'; +import { DART_PACKAGE_IDENTITY_REASON } from '../../src/core/ingestion/languages/dart/package-dependencies.js'; import { SPRING_AOP_FEATURE, SPRING_BEAN_INVENTORY_FEATURE, @@ -1726,6 +1733,166 @@ describe('runFullAnalysis — incremental orchestration', () => { * 'exact-scan', which the unit-level wiring pin in * run-analyze-fts-repair.test.ts covers platform-independently. */ +let dartAvailable = isLanguageAvailable(SupportedLanguages.Dart); +if (dartAvailable) { + try { + await loadParser(); + await loadLanguage(SupportedLanguages.Dart); + } catch { + dartAvailable = false; + } +} + +describe.skipIf(!dartAvailable)('Dart pubspec-only incremental persistence (#2963)', () => { + it.each([ + { + name: 'rename', + initial: 'name: app', + next: 'name: renamed', + file: 'pubspec.yaml', + nextResolves: false, + }, + { + name: 'addition', + initial: null, + next: 'name: app', + file: 'pubspec.yaml', + nextResolves: true, + }, + { + name: 'duplicate', + initial: 'name: app', + next: 'name: app', + file: 'nested/pubspec.yaml', + nextResolves: false, + }, + { + name: 'repair', + initial: 'name: [invalid', + next: 'name: app', + file: 'pubspec.yaml', + nextResolves: true, + }, + ])( + 'persists $name and its reversal with forced-rebuild parity', + async ({ initial, next, file, nextResolves }) => { + const repo = await createTempDir(); + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const options = { skipAgentsMd: true, skipFts: true }; + const progress = { onProgress: () => {} }; + const readEdges = async () => { + await adapter.initLbug(getStoragePaths(repo.dbPath).lbugPath, { skipFts: true }); + try { + return await adapter.executeQuery( + `MATCH (a)-[r:CodeRelation]->(b) WHERE r.type IN ['IMPORTS', 'CALLS'] ` + + `RETURN a.id AS source, b.id AS target, r.type AS type, r.reason AS reason, ` + + `a.filePath AS sourceFile, b.filePath AS targetFile, b.name AS targetName ` + + `ORDER BY source, target, type, reason`, + ); + } finally { + await adapter.closeLbug(); + } + }; + try { + await mkdir(path.join(repo.dbPath, 'lib'), { recursive: true }); + await mkdir(path.join(repo.dbPath, 'nested'), { recursive: true }); + await writeFile(path.join(repo.dbPath, '.gitignore'), '.gitnexus/\n'); + await writeFile( + path.join(repo.dbPath, 'lib/model.dart'), + 'class Model { void save() {} }\n', + ); + await writeFile( + path.join(repo.dbPath, 'lib/bridge.dart'), + "import 'package:app/model.dart';\nModel buildModel() => Model();\n", + ); + await writeFile( + path.join(repo.dbPath, 'lib/main.dart'), + "import './bridge.dart';\nvoid run() { buildModel().save(); }\n", + ); + await writeFile( + path.join(repo.dbPath, 'nested/decoy.dart'), + 'class Model { void save() {} }\n', + ); + if (initial !== null) await writeFile(path.join(repo.dbPath, 'pubspec.yaml'), initial); + execSync('git init -q', { cwd: repo.dbPath, stdio: 'pipe' }); + gitCommitAll(repo.dbPath, 'Dart package fixture'); + await runFullAnalysis(repo.dbPath, options, progress); + const initialEdges = await readEdges(); + expect( + initialEdges.filter( + (edge) => + edge.type === 'IMPORTS' && + edge.source === 'File:lib/bridge.dart' && + edge.target === 'File:lib/model.dart', + ), + ).toHaveLength(initial === 'name: app' ? 1 : 0); + + for (const [content, resolves] of [ + [next, nextResolves], + [file === 'pubspec.yaml' ? initial : null, initial === 'name: app'], + ] as const) { + const target = path.join(repo.dbPath, file); + if (content === null) await rm(target); + else await writeFile(target, content); + gitCommitAll(repo.dbPath, 'Update package identity'); + const incremental = await runFullAnalysis(repo.dbPath, options, progress); + expect(incremental.incrementalStats?.writeMode).toBe('incremental'); + const incrementalEdges = await readEdges(); + const imports = incrementalEdges.filter( + (edge) => + edge.type === 'IMPORTS' && + edge.source === 'File:lib/bridge.dart' && + edge.target === 'File:lib/model.dart', + ); + expect(imports).toHaveLength(resolves ? 1 : 0); + expect( + incrementalEdges.filter( + (edge) => + edge.type === 'CALLS' && + edge.sourceFile === 'lib/main.dart' && + edge.targetFile === 'lib/model.dart' && + edge.targetName === 'save', + ), + ).toHaveLength(resolves ? 1 : 0); + const meta = await loadMeta(getStoragePaths(repo.dbPath).storagePath); + expect(Object.hasOwn(meta?.fileHashes ?? {}, file)).toBe(content !== null); + const identityToFile = incrementalEdges.filter( + (edge) => + edge.type === 'IMPORTS' && + edge.reason === DART_PACKAGE_IDENTITY_REASON && + edge.source === 'File:lib/main.dart' && + edge.target === `File:${file}`, + ); + if (content === 'name: app') { + expect(identityToFile).toEqual([ + { + source: 'File:lib/main.dart', + target: `File:${file}`, + type: 'IMPORTS', + reason: DART_PACKAGE_IDENTITY_REASON, + sourceFile: 'lib/main.dart', + targetFile: file, + targetName: 'pubspec.yaml', + }, + ]); + } else { + // Rename and deletion must drop the identity edge. A stale edge + // would keep rewriting importers after the package name is gone. + expect(identityToFile).toEqual([]); + } + await runFullAnalysis(repo.dbPath, { ...options, force: true }, progress); + expect(await readEdges()).toEqual(incrementalEdges); + } + } finally { + await adapter.closeLbug(); + await repo.cleanup(); + } + }, + 300_000, + ); +}); + describe('runFullAnalysis — escalated wipe recreates the vector index (#2409, tri-review 4669518496 P1)', () => { let vectorAvailable = false; let skipWarned = false; diff --git a/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts b/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts index 3e56563f1..bf73a6a76 100644 --- a/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts +++ b/gitnexus/test/unit/scope-resolution/external-import-conformance.test.ts @@ -318,7 +318,7 @@ const CASES: ReadonlyMap = new Map([ { files: ['lib/http.dart', 'lib/models.dart', 'lib/main.dart'], fromFile: 'lib/main.dart', - resolutionConfig: undefined, + resolutionConfig: { packages: new Map([['app', 'lib']]) }, external: 'package:http/http.dart', decoy: 'lib/http.dart', reachesDecoy: 'package:app/http.dart', @@ -416,7 +416,6 @@ const CASES: ReadonlyMap = new Map([ * TypeScript one, then deleting its line here. */ const KNOWN_GAPS: ReadonlyMap = new Map([ - [SupportedLanguages.Dart, '`package:http/http.dart` -> `lib/http.dart`'], [SupportedLanguages.C, '`stdio.h` -> `src/stdio.h`'], [SupportedLanguages.CPlusPlus, '`cstdio.h` -> `src/cstdio.h`'], ]); diff --git a/gitnexus/test/unit/scope-resolution/import-target-index-parity.test.ts b/gitnexus/test/unit/scope-resolution/import-target-index-parity.test.ts index a3ecd5693..bc8384257 100644 --- a/gitnexus/test/unit/scope-resolution/import-target-index-parity.test.ts +++ b/gitnexus/test/unit/scope-resolution/import-target-index-parity.test.ts @@ -16,8 +16,8 @@ * - C#'s `resolveDirectMatch` lets a whole-path match win over a suffix match * found EARLIER in iteration order, while `resolveByProgressiveStripping` * takes whichever comes first; - * - Dart tries `lib/` fully before bare ``, and compares raw paths - * (no backslash normalization) on both legs. + * - Dart's relative suffix fallback compares raw workspace paths. Package + * identity changed in #2963 and is covered by separate boundary tests. * * So this file keeps copies of the pre-change implementations and asserts the * new ones agree with them on a deterministic corpus built to force exactly @@ -396,10 +396,12 @@ const GO_TARGETS = [ const DART_TARGETS = [ '', 'dart:core', - 'package:app/models.dart', - 'package:app/src/models.dart', - 'package:app', - 'package:other/lib/util.dart', + // Package identity intentionally changed in #2963; legacy parity applies + // only to relative imports. Package boundaries have dedicated regressions. + '../../lib/models.dart', + '../../lib/src/models.dart', + '../../models.dart', + '../util.dart', 'models.dart', './models.dart', '../models.dart', @@ -621,32 +623,28 @@ describe('import-target index hoist — output parity with the pre-change scans' }, { lang: 'dart', - why: '`lib/` beats bare `` even when the bare hit comes FIRST in Set order', + why: 'relative suffix lookup keeps the importer directory prefix', files: ['a/models.dart', 'z/lib/models.dart'], - target: 'package:app/models.dart', + target: 'models.dart', }, { lang: 'dart', - why: 'bare `` is reached only after `lib/` misses entirely', + why: 'relative root lookup reaches a nested suffix', files: ['a/models.dart'], - target: 'package:app/models.dart', + target: 'models.dart', + fromFile: 'main.dart', }, { lang: 'dart', why: 'paths are matched RAW — a backslash path is not normalized into a hit', files: ['win\\dir\\thing.dart'], - target: 'package:app/dir/thing.dart', + target: 'dir/thing.dart', }, { - // The negative case above pins the guard NEXT DOOR to the one it names: - // the basename bucket lookup misses before the raw comparison is ever - // consulted, so normalizing only the bucket key, or only the comparison, - // still yields null and still matches. This positive twin puts the - // backslashes in the TARGET so a hit depends on both halves staying raw. lang: 'dart', - why: 'a backslash TARGET matches only because neither the bucket key nor the comparison normalizes', - files: ['dir\\thing.dart'], - target: 'package:app/dir\\thing.dart', + why: 'relative targets normalize backslashes before exact membership lookup', + files: ['lib/dir/thing.dart'], + target: 'dir\\thing.dart', }, { lang: 'dart', @@ -745,7 +743,7 @@ describe('import-target index hoist — output parity with the pre-change scans' if (csharp(t, cs) !== null) hits.csharp++; } } - // Measured on this corpus: go 366, dart 75, ruby 259, csharp 220. Ruby and + // Historical corpus counts: go 366, dart 75, ruby 259, csharp 220. Ruby and // C# gained 40 each from the `win\dir\thing.` targets — one per repo, // which is also the floor those two arms now defend. #2881 moved go 364 -> // 366 and csharp 196 -> 220, from the corpus's `pkg/pkg`, `a/pkg/b/pkg` and @@ -804,7 +802,7 @@ describe('import-target index hoist — built once per file set, not once per im it('dart builds one index for many imports (#2879)', () => { const files = countingCorpus(2, '.dart'); for (let i = 0; i < 200; i++) { - resolveDartImportTarget(`package:pkg${i}/ghost${i}.dart`, 'lib/main.dart', files); + resolveDartImportTarget(`pkg${i}/ghost${i}.dart`, 'lib/main.dart', files); } expect(files.scans).toBe(1); }); diff --git a/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts b/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts index 775452ba7..0f2a9f2e3 100644 --- a/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts +++ b/gitnexus/test/unit/scope-resolution/import-target-index-reuse.contract.test.ts @@ -519,10 +519,9 @@ const FIXTURES: ReadonlyMap = new Map< files: ['lib/models.dart', 'tool/generate.dart', 'lib/main.dart'], fromFile: 'lib/main.dart', resolutionConfig: undefined, - // An external package: both `lib/` and bare `` miss, which is - // the two-scan case. - missTarget: (i) => `package:vendor${i}/ghost.dart`, - hitTarget: 'package:app/models.dart', + // Package imports are exact lookups; relative misses exercise the index. + missTarget: (i) => `vendor${i}/ghost.dart`, + hitTarget: './models.dart', parsedImport: IGNORES_CONTEXT, minimumScans: 1, minimumParsedFileReads: 0, diff --git a/gitnexus/test/unit/stream-graph-emit-config.test.ts b/gitnexus/test/unit/stream-graph-emit-config.test.ts index e6311df99..878698b8c 100644 --- a/gitnexus/test/unit/stream-graph-emit-config.test.ts +++ b/gitnexus/test/unit/stream-graph-emit-config.test.ts @@ -168,6 +168,10 @@ describe('RETAINED_REL_TYPES tracks its readers', () => { // CALLS is read by taintSummaries, which is exactly why the sink answers a // COMPLETE read instead of retaining it — so it is a known exemption. readTypes.delete('CALLS'); + // Dart package invalidation reads IMPORTS endpoints through the sink's + // complete typed iterator. dart-package-dependencies.test.ts exercises + // transitive closure and idempotence with actual streamed IMPORTS rows. + readTypes.delete('IMPORTS'); const missing = [...readTypes].filter((t) => !RETAINED_REL_TYPES.has(t as RelationshipType)); expect(missing).toEqual([]);