diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index f12441517..d0ec8caa1 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -164,6 +164,7 @@ const PLATFORM_LOGIC = [ // N-API addon which has known platform-specific behavior (Windows // file-lock lag after close, macOS N-API destructor segfaults) const LBUG_NATIVE = [ + 'test/integration/dangling-owner-edges.test.ts', 'test/integration/impact-private-type-file-seed.test.ts', 'test/integration/impact-private-type-source.test.ts', // Inline-handler findings must survive native persistence and cold explain diff --git a/gitnexus/scripts/owner-edge-guard-scaling.ts b/gitnexus/scripts/owner-edge-guard-scaling.ts new file mode 100644 index 000000000..ad90fb52c --- /dev/null +++ b/gitnexus/scripts/owner-edge-guard-scaling.ts @@ -0,0 +1,321 @@ +/** + * Reproducible scaling probe for the per-file owner-edge guard. + * + * Run alone, after a build, from gitnexus/: + * node --expose-gc --import tsx scripts/owner-edge-guard-scaling.ts > /tmp/owner-edge-scaling.json + * --operations-only skips timings; --sizes=1024,2048,... --samples=11 --warmup=3 override defaults. + * + * Timings include guard lifecycle and appending preconstructed rows, excluding + * fixture creation, prefix copies, assertions, and explicit GC. Prefix-isolation + * timings call the guard directly on a fixed 16-member slice. Proxy counters run + * separately: they are deterministic regression checks, never timed thresholds. + */ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { arch, cpus, platform, release } from 'node:os'; +import { performance } from 'node:perf_hooks'; +import { parseArgs } from 'node:util'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { + createOwnerEdgeGuard, + dropDanglingInFileOwnerEdges, +} from '../src/core/ingestion/workers/owner-edge-guard.js'; +import type { ParseWorkerResult } from '../src/core/ingestion/workers/parse-worker.js'; + +const source = readFileSync( + new URL('../src/core/ingestion/workers/owner-edge-guard.ts', import.meta.url), +); + +type Slice = Pick; +type Rel = Slice['relationships'][number]; +type Shape = 'member-free' | 'clean' | 'dangling' | 'predefined' | 'duplicate' | 'owner-last'; +type FileFixture = Slice & { path: string; expected: Rel[] }; +type Reads = { nodes: number; relationships: number; writes: number }; + +const { values } = parseArgs({ + options: { + sizes: { type: 'string', default: '1024,2048,4096,8192,16384,32768,65536' }, + samples: { type: 'string', default: '11' }, + warmup: { type: 'string', default: '3' }, + 'operations-only': { type: 'boolean', default: false }, + }, +}); +const sizes = values.sizes.split(',').map(Number); +const samples = Number(values.samples); +const warmup = Number(values.warmup); +assert( + sizes.length >= 2 && + sizes.every((n, i) => Number.isSafeInteger(n) && n > 0 && (!i || n > sizes[i - 1])), +); +assert(Number.isSafeInteger(samples) && samples >= 3); +assert(Number.isSafeInteger(warmup) && warmup >= 1); + +const edge = (type: Rel['type'], sourceId: string, targetId: string): Rel => ({ + id: `${type}:${sourceId}->${targetId}`, + sourceId, + targetId, + type, + confidence: 1, + reason: '', +}); + +function fixture(shape: Shape, count: number, file: number): FileFixture { + // Fixed-width ids keep the string-length dimension constant across sizes. + const path = `src/file-${String(file).padStart(8, '0')}.ts`; + const owner = `Class:${path}:Owner`; + const result: FileFixture = { path, nodes: [], relationships: [], expected: [] }; + const addNode = (id: string, label: string): void => { + result.nodes.push({ + id, + label, + properties: { + name: id, + filePath: path, + startLine: 0, + endLine: 0, + language: SupportedLanguages.TypeScript, + isExported: false, + }, + }); + }; + if (shape === 'clean') addNode(owner, 'Class'); + for (let i = 0; i < count; i++) { + const target = `Method:${path}:member-${String(i).padStart(8, '0')}`; + const defined = edge('DEFINES', `File:${path}`, target); + const member = edge(i % 2 ? 'HAS_PROPERTY' : 'HAS_METHOD', owner, target); + addNode(target, 'Method'); + if (shape === 'member-free' || shape === 'predefined') result.relationships.push(defined); + if (shape !== 'member-free') result.relationships.push(member); + if (shape === 'duplicate') result.relationships.push(edge('HAS_PROPERTY', owner, target)); + result.expected.push(shape === 'clean' || shape === 'owner-last' ? member : defined); + } + if (shape === 'owner-last') addNode(owner, 'Class'); + return result; +} + +function files(shape: Shape, count: number, perFile: number): FileFixture[] { + const result: FileFixture[] = []; + for (let i = 0; i < count; i += perFile) + result.push(fixture(shape, Math.min(perFile, count - i), i / perFile)); + return result; +} + +function append(target: Slice, file: Slice): void { + for (const node of file.nodes) target.nodes.push(node); + for (const rel of file.relationships) target.relationships.push(rel); +} + +function verify(result: Slice, expected: Rel[], nodeCount: number): void { + assert.equal(result.nodes.length, nodeCount); + assert.deepEqual(result.relationships, expected); +} + +/** Count numeric array accesses only while the production guard runs. */ +function operations(input: FileFixture[], prefix: Slice): Reads { + const raw: Slice = { nodes: prefix.nodes.slice(), relationships: prefix.relationships.slice() }; + let nodeStart = raw.nodes.length; + let relStart = raw.relationships.length; + const reads: Reads = { nodes: 0, relationships: 0, writes: 0 }; + const watched = (array: T[], kind: 'nodes' | 'relationships'): T[] => + new Proxy(array, { + get(target, property, receiver) { + if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) { + assert( + Number(property) >= (kind === 'nodes' ? nodeStart : relStart), + `rescanned previous ${kind}`, + ); + reads[kind]++; + } + return Reflect.get(target, property, receiver); + }, + set(target, property, value, receiver) { + if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) { + assert( + Number(property) >= (kind === 'nodes' ? nodeStart : relStart), + `rewrote previous ${kind}`, + ); + reads.writes++; + } + return Reflect.set(target, property, value, receiver); + }, + }); + const guarded = { + nodes: watched(raw.nodes, 'nodes'), + relationships: watched(raw.relationships, 'relationships'), + }; + const guard = createOwnerEdgeGuard(guarded); + for (const file of input) { + guard.open(file.path); // Settles the previous slice before the boundaries advance. + nodeStart = raw.nodes.length; + relStart = raw.relationships.length; + append(raw, file); // Fixture appends bypass the access counters. + } + guard.settle(); + const settledReads = { ...reads }; + guard.settle(); + assert.deepEqual(reads, settledReads, 'settling twice rescans a file'); + const n = input.reduce((sum, file) => sum + file.nodes.length, 0); + const e = input.reduce((sum, file) => sum + file.relationships.length, 0); + assert(reads.nodes <= n, 'more than one node pass'); + assert(reads.relationships <= 3 * e, 'more than three relationship passes'); + assert(reads.writes <= e, 'more than one compaction pass'); + verify( + raw, + [...prefix.relationships, ...input.flatMap((file) => file.expected)], + prefix.nodes.length + n, + ); + return reads; +} + +function trial(input: FileFixture[], prefix: Slice, direct: boolean, expected: Rel[]): number { + const result: Slice = { + nodes: prefix.nodes.slice(), + relationships: prefix.relationships.slice(), + }; + const nodeStart = result.nodes.length; + const relStart = result.relationships.length; + if (direct) append(result, input[0]); + globalThis.gc?.(); + const start = performance.now(); + if (direct) { + dropDanglingInFileOwnerEdges(result, input[0].path, nodeStart, relStart); + } else { + const guard = createOwnerEdgeGuard(result); + for (const file of input) { + guard.open(file.path); + append(result, file); + } + guard.settle(); + } + const elapsed = performance.now() - start; + verify( + result, + expected, + prefix.nodes.length + input.reduce((sum, file) => sum + file.nodes.length, 0), + ); + return elapsed; +} + +const percentile = (values: number[], fraction: number): number => { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.ceil(fraction * sorted.length) - 1]; +}; + +function measure(shape: Shape, count: number, perFile: number, prefixCount = 0) { + const input = files(shape, count, perFile); + const prefix = prefixCount + ? fixture('member-free', prefixCount, 99999999) + : { nodes: [], relationships: [] }; + const nodes = input.reduce((sum, file) => sum + file.nodes.length, 0); + const relationships = input.reduce((sum, file) => sum + file.relationships.length, 0); + const counts = operations(input, prefix); + const direct = prefixCount > 0; + const samplesMs: number[] = []; + if (!values['operations-only']) { + for (let i = 0; i < warmup + samples; i++) { + const freshInput = files(shape, count, perFile); + const expected = [...prefix.relationships, ...freshInput.flatMap((file) => file.expected)]; + const elapsed = trial(freshInput, prefix, direct, expected); + if (i >= warmup) samplesMs.push(elapsed); + } + } + const medianMs = samplesMs.length ? percentile(samplesMs, 0.5) : null; + return { + shape, + members: count, + files: input.length, + prefixRows: prefix.nodes.length + prefix.relationships.length, + nodes, + relationships, + indexedReads: counts, + samplesMs, + medianMs, + p95Ms: samplesMs.length ? percentile(samplesMs, 0.95) : null, + nsPerRow: medianMs === null ? null : (medianMs * 1e6) / (nodes + relationships), + }; +} + +function slope(points: { x: number; y: number }[]): number | null { + if (points.length < 2 || points.some(({ x, y }) => x <= 0 || y <= 0)) return null; + const xs = points.map(({ x }) => Math.log(x)); + const ys = points.map(({ y }) => Math.log(y)); + const meanX = xs.reduce((a, b) => a + b, 0) / xs.length; + const meanY = ys.reduce((a, b) => a + b, 0) / ys.length; + return ( + xs.reduce((sum, x, i) => sum + (x - meanX) * (ys[i] - meanY), 0) / + xs.reduce((sum, x) => sum + (x - meanX) ** 2, 0) + ); +} + +const cases: ReturnType[] = []; +const scaling: { shape: Shape; distribution: string; logLogSlope: number | null }[] = []; +const shapes: Shape[] = [ + 'member-free', + 'clean', + 'dangling', + 'predefined', + 'duplicate', + 'owner-last', +]; +// Alternate ascending/descending sizes to avoid tying every large case to late-run thermal drift. +for (const [shapeIndex, shape] of shapes.entries()) { + for (const distribution of ['one-file', 'many-files']) { + const rows = (shapeIndex % 2 ? [...sizes].reverse() : sizes).map((size) => + measure(shape, size, distribution === 'one-file' ? size : 16), + ); + cases.push(...rows); + scaling.push({ + shape, + distribution, + logLogSlope: values['operations-only'] + ? null + : slope( + rows.flatMap((row) => + row.medianMs === null ? [] : [{ x: row.nodes + row.relationships, y: row.medianMs }], + ), + ), + }); + process.stderr.write(`Validated ${shape} / ${distribution}\n`); + } +} +const prefixIsolation = sizes.map((size) => measure('dangling', 16, 16, size)); +process.stdout.write( + `${JSON.stringify( + { + schema: 'gitnexus.owner-edge-guard-scaling/v1', + recordedAt: new Date().toISOString(), + environment: { + node: process.version, + v8: process.versions.v8, + platform: platform(), + arch: arch(), + release: release(), + cpu: cpus()[0]?.model, + explicitGc: typeof globalThis.gc === 'function', + processMaxRssKiB: process.resourceUsage().maxRSS, + }, + sourceSha256: createHash('sha256').update(source).digest('hex'), + config: { sizes, samples, warmup, operationsOnly: values['operations-only'] }, + methodology: { + timed: + 'guard lifecycle + append preconstructed rows; prefixIsolation times only the direct guard', + excluded: + 'fresh fixture generation for every sample, prefix copies, assertions, explicit GC, Proxy instrumentation', + statistics: + 'nearest-rank median/p95; least-squares slope of log(median ms) versus log(active rows)', + operations: + 'at most one indexed node read, three indexed relationship reads, one relationship write per active row; no earlier-slice indexed accesses', + limits: + 'Kernel benchmark, not end-to-end indexing throughput. Timings and slopes are descriptive, not asymptotic proof. maxRSS covers all fixtures and validation and is not guard peak memory. Set operations assume average constant-time hashing; string work scales with id lengths.', + memory: + 'Live guard scratch is O(nodes + edges) of the largest active file; output accumulation and delayed garbage collection are excluded from that bound.', + }, + cases, + scaling, + prefixIsolation, + }, + null, + 2, + )}\n`, +); diff --git a/gitnexus/src/core/ingestion/workers/owner-edge-guard.ts b/gitnexus/src/core/ingestion/workers/owner-edge-guard.ts new file mode 100644 index 000000000..7c7809ccb --- /dev/null +++ b/gitnexus/src/core/ingestion/workers/owner-edge-guard.ts @@ -0,0 +1,109 @@ +/** + * Per-file guard against member edges whose owner node was never emitted. + * + * Owner ids are minted from syntax (the binding an object literal is assigned + * to, the enclosing container), while node labels come from the query captures, + * so the two can disagree (#3747): `const X = z.object({ k: f(() => 0) }).refine(cb)` + * is emitted as `Function::X` but its key's owner is spelled + * `Const::X`, and a TS `declare module 'm' {}` owns its members under a + * `Trait` id no TS capture ever emits. LadybugDB's COPY rejects such an edge and + * the retry silently drops it. + * + * Kept free of worker side effects so it can be unit-tested directly. + */ +import { generateId } from '../../../lib/utils.js'; +import type { ParseWorkerResult } from './parse-worker.js'; + +const isMemberEdge = (type: string): boolean => type === 'HAS_METHOD' || type === 'HAS_PROPERTY'; + +/** + * Remove `HAS_METHOD` / `HAS_PROPERTY` edges appended for `filePath` (from + * `relStart` on) whose owner id names a node in that same file that the file + * did not emit (from `nodeStart` on). The orphaned member is re-attached to its + * File with `DEFINES` unless it already has one. + * + * Owners in other files are left alone: they are resolved across the whole + * graph, not per file. + */ +export const dropDanglingInFileOwnerEdges = ( + result: Pick, + filePath: string, + nodeStart: number, + relStart: number, +): void => { + const rels = result.relationships; + const inFilePrefix = `:${filePath}:`; + let emitted: Set | null = null; + let dangling: Set | null = null; + + for (let i = relStart; i < rels.length; i++) { + const rel = rels[i]; + if (!isMemberEdge(rel.type)) continue; + const labelEnd = rel.sourceId.indexOf(':'); + if (labelEnd < 0 || !rel.sourceId.startsWith(inFilePrefix, labelEnd)) continue; + if (emitted === null) { + emitted = new Set(); + for (let n = nodeStart; n < result.nodes.length; n++) emitted.add(result.nodes[n].id); + } + if (!emitted.has(rel.sourceId)) (dangling ??= new Set()).add(i); + } + if (dangling === null || emitted === null) return; + + const fileId = generateId('File', filePath); + const defined = new Set(); + for (let i = relStart; i < rels.length; i++) { + const rel = rels[i]; + if (rel.type === 'DEFINES' && rel.sourceId === fileId) defined.add(rel.targetId); + } + + let write = relStart; + for (let i = relStart; i < rels.length; i++) { + const rel = rels[i]; + if (!dangling.has(i)) { + rels[write++] = rel; + continue; + } + if (defined.has(rel.targetId) || !emitted.has(rel.targetId)) continue; + defined.add(rel.targetId); + rels[write++] = { + id: generateId('DEFINES', `${fileId}->${rel.targetId}`), + sourceId: fileId, + targetId: rel.targetId, + type: 'DEFINES', + confidence: 1.0, + reason: '', + }; + } + rels.length = write; +}; + +/** Tracks the file whose slice of a worker result is still being appended. */ +export interface OwnerEdgeGuard { + /** Settle the previous file, then start tracking what `filePath` appends. */ + open(filePath: string): void; + /** Settle the tracked file, if any. Safe to call more than once. */ + settle(): void; +} + +export const createOwnerEdgeGuard = ( + result: Pick, +): OwnerEdgeGuard => { + let current: { path: string; nodeStart: number; relStart: number } | null = null; + const settle = (): void => { + if (current === null) return; + const file = current; + current = null; + dropDanglingInFileOwnerEdges(result, file.path, file.nodeStart, file.relStart); + }; + return { + open(filePath: string): void { + settle(); + current = { + path: filePath, + nodeStart: result.nodes.length, + relStart: result.relationships.length, + }; + }, + settle, + }; +}; diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index fb0a43b2b..a2599ba76 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -138,6 +138,7 @@ import { buildTypeEnv } from '../type-env.js'; import type { ConstructorBinding } from '../type-env.js'; import { detectFrameworkFromAST } from '../framework-detection.js'; import { generateId } from '../../../lib/utils.js'; +import { createOwnerEdgeGuard, type OwnerEdgeGuard } from './owner-edge-guard.js'; import { defaultExportNameCollides } from '../languages/typescript/cjs-export-assignment.js'; import { extractVueScript, @@ -1595,6 +1596,23 @@ const processFileGroup = ( queryString: string, result: ParseWorkerResult, onFileProcessed?: () => void, +): void => { + const ownerEdgeGuard = createOwnerEdgeGuard(result); + try { + processFileGroupFiles(files, language, queryString, result, ownerEdgeGuard, onFileProcessed); + } finally { + // Also on a throw: processBatch keeps the partial result of a failed group. + ownerEdgeGuard.settle(); + } +}; + +const processFileGroupFiles = ( + files: ParseWorkerInput[], + language: SupportedLanguages, + queryString: string, + result: ParseWorkerResult, + ownerEdgeGuard: OwnerEdgeGuard, + onFileProcessed?: () => void, ): void => { let query: Parser.Query; try { @@ -1615,6 +1633,8 @@ const processFileGroup = ( } for (const file of files) { + // Settles the previous file, so every `continue` below still gets checked. + ownerEdgeGuard.open(file.path); // Skip files larger than the max tree-sitter buffer (32 MB) if (getTreeSitterContentByteLength(file.content) > TREE_SITTER_MAX_BUFFER) continue; diff --git a/gitnexus/test/integration/dangling-owner-edges.test.ts b/gitnexus/test/integration/dangling-owner-edges.test.ts new file mode 100644 index 000000000..a603bf780 --- /dev/null +++ b/gitnexus/test/integration/dangling-owner-edges.test.ts @@ -0,0 +1,163 @@ +/** + * #3747: a member edge must never name an owner node the file did not emit. + * Both fixtures are the reporter's repro; before the fix each produced one + * HAS_METHOD edge that LadybugDB's COPY rejected with + * "Unable to find primary key value". + */ +import { describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { + closeLbug, + executeQuery, + initLbug, + loadGraphToLbug, +} from '../../src/core/lbug/lbug-adapter.js'; +import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js'; +import { createTempDir } from '../helpers/test-db.js'; +import { DIST_WORKER_URL, parseFilesWithWorkers } from '../helpers/worker-parse.js'; +import { createWorkerPool } from '../../src/core/ingestion/workers/worker-pool.js'; +import type { + ParseWorkerInput, + ParseWorkerResult, +} from '../../src/core/ingestion/workers/parse-worker.js'; +import { findDanglingEdges } from './resolvers/helpers.js'; + +const CHAINED_CONST = `import { z } from 'zod'; + +const KEY = 'BRANCH'; +const Branch = z.string(); + +const Context = z + .object({ + REF: z.string(), + [KEY]: Branch, + REPO: z.string().refine((value) => value.split('/').every((p) => p !== '..')), + }) + .refine((context) => context.REF === \`refs/heads/\${context[KEY]}\`, { + message: 'REF must be the branch head', + }); + +export function run(env) { + const context = Context.parse(env); + return context; +} +`; + +const AMBIENT_MODULE = `declare module 'somelib' { + export function connect(opts: { user: string }): Promise; + const lib: { connect: typeof connect }; + export default lib; +} +`; + +const OBJECT_LITERAL_CONTROL = `export const service = { + load() { return 1; }, + save: () => 2, +}; +`; + +describe('#3747 member edges only name emitted owners', () => { + it('leaves no dangling member edge and keeps every member reachable from its file', async () => { + const { graph } = await parseFilesWithWorkers([ + { path: 'ctx.mjs', content: CHAINED_CONST }, + { path: 'types/lib.d.ts', content: AMBIENT_MODULE }, + ]); + expect(findDanglingEdges({ graph } as never, ['HAS_METHOD', 'HAS_PROPERTY'])).toEqual([]); + + const definedFromFile = (file: string): string[] => + graph.relationships + .filter((edge) => edge.type === 'DEFINES' && edge.sourceId === `File:${file}`) + .map((edge) => edge.targetId); + const repo = graph.nodes.find( + (n) => n.properties.filePath === 'ctx.mjs' && n.properties.name === 'REPO', + ); + const connect = graph.nodes.find( + (n) => n.properties.filePath === 'types/lib.d.ts' && n.properties.name === 'connect', + ); + expect(repo).toBeDefined(); + expect(connect).toBeDefined(); + expect(definedFromFile('ctx.mjs')).toContain(repo?.id); + expect(definedFromFile('types/lib.d.ts')).toContain(connect?.id); + }); + + it('keeps object-literal members owned by their emitted binding', async () => { + const { graph } = await parseFilesWithWorkers([ + { path: 'service.ts', content: OBJECT_LITERAL_CONTROL }, + ]); + const owner = graph.nodes.find( + (n) => n.properties.filePath === 'service.ts' && n.properties.name === 'service', + ); + expect(owner).toBeDefined(); + const members = graph.relationships + .filter((edge) => edge.type === 'HAS_METHOD' && edge.sourceId === owner?.id) + .map((edge) => graph.getNode(edge.targetId)?.properties.name) + .sort(); + expect(members).toEqual(['load', 'save']); + }); + + it('persists every relationship through native COPY', async () => { + const temp = await createTempDir(); + try { + await fs.mkdir(path.join(temp.dbPath, 'types'), { recursive: true }); + await fs.writeFile(path.join(temp.dbPath, 'ctx.mjs'), CHAINED_CONST); + await fs.writeFile(path.join(temp.dbPath, 'types', 'lib.d.ts'), AMBIENT_MODULE); + const result = await runPipelineFromRepo(temp.dbPath, () => {}, { workerPoolSize: 1 }); + expect(result.usedWorkerPool).toBe(true); + expect(findDanglingEdges(result)).toEqual([]); + + await initLbug(path.join(temp.dbPath, 'lbug'), { skipFts: true }); + await loadGraphToLbug(result.graph, temp.dbPath, path.join(temp.dbPath, 'storage')); + const [row] = await executeQuery('MATCH ()-[r:CodeRelation]->() RETURN count(r) AS edges'); + expect(Number(row.edges)).toBe(result.graph.relationships.length); + } finally { + await closeLbug(); + await temp.cleanup(); + } + }); + + it('settles the partial file when a real parser provider throws after emitting members', async () => { + const temp = await createTempDir(); + const bootstrap = path.join(temp.dbPath, 'throwing-provider.mjs'); + const providersUrl = new URL('../languages/index.js', DIST_WORKER_URL).href; + // Inject the failure in the worker isolate, leaving the production loop and + // its finally intact. Removing that finally must make this regression fail. + await fs.writeFile( + bootstrap, + ` + const { getProvider } = await import(${JSON.stringify(providersUrl)}); + getProvider('javascript').extractToolDefinitions = () => { + throw new Error('owner-edge-guard provider fault'); + }; + await import(${JSON.stringify(DIST_WORKER_URL.href)}); + `, + ); + const pool = createWorkerPool(pathToFileURL(bootstrap), 1); + try { + const [result] = await pool.dispatch([ + { path: 'ctx.mjs', content: CHAINED_CONST }, + { path: 'after.mjs', content: 'export function shouldNotParse() {}' }, + ]); + // The first file is counted before the provider throws; the remaining + // file is skipped, proving the injected failure reached processBatch. + expect(result.fileCount).toBe(1); + expect(result.nodes.some((node) => node.properties.name === 'shouldNotParse')).toBe(false); + const member = result.nodes.find((node) => node.properties.name === 'REPO'); + expect(member).toBeDefined(); + expect(result.relationships.filter((edge) => edge.targetId === member?.id)).toEqual([ + { + id: `DEFINES:File:ctx.mjs->${member?.id}`, + sourceId: 'File:ctx.mjs', + targetId: member?.id, + type: 'DEFINES', + confidence: 1, + reason: '', + }, + ]); + } finally { + await pool.terminate(); + await temp.cleanup(); + } + }); +}); diff --git a/gitnexus/test/unit/owner-edge-guard-scaling.test.ts b/gitnexus/test/unit/owner-edge-guard-scaling.test.ts new file mode 100644 index 000000000..471e6d595 --- /dev/null +++ b/gitnexus/test/unit/owner-edge-guard-scaling.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { createOwnerEdgeGuard } from '../../src/core/ingestion/workers/owner-edge-guard.js'; +import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js'; + +type Slice = Pick; +type Rel = Slice['relationships'][number]; + +const memberNode = (path: string, member: number): Slice['nodes'][number] => ({ + id: `Method:${path}:m${member}`, + label: 'Method', + properties: { + name: `m${member}`, + filePath: path, + startLine: 0, + endLine: 0, + language: SupportedLanguages.TypeScript, + isExported: false, + }, +}); + +const memberEdge = (type: Rel['type'], sourceId: string, targetId: string): Rel => ({ + id: `${type}:${sourceId}->${targetId}`, + sourceId, + targetId, + type, + confidence: 1, + reason: '', +}); + +describe('owner-edge guard scaling', () => { + it.each([8, 32, 128])('processes %i files without rescanning completed files', (fileCount) => { + const prefixNode = memberNode('previous.ts', 0); + const prefixEdge = memberEdge('DEFINES', 'File:previous.ts', prefixNode.id); + const raw: Slice = { nodes: [prefixNode], relationships: [prefixEdge] }; + let nodeStart = 1; + let relStart = 1; + let nodeReads = 0; + let relReads = 0; + let relWrites = 0; + const guarded: Slice = { + nodes: new Proxy(raw.nodes, { + get(target, property, receiver) { + if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) { + expect(Number(property)).toBeGreaterThanOrEqual(nodeStart); + nodeReads++; + } + return Reflect.get(target, property, receiver); + }, + }), + relationships: new Proxy(raw.relationships, { + get(target, property, receiver) { + if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) { + expect(Number(property)).toBeGreaterThanOrEqual(relStart); + relReads++; + } + return Reflect.get(target, property, receiver); + }, + set(target, property, value, receiver) { + if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) { + expect(Number(property)).toBeGreaterThanOrEqual(relStart); + relWrites++; + } + return Reflect.set(target, property, value, receiver); + }, + }), + }; + const guard = createOwnerEdgeGuard(guarded); + for (let file = 0; file < fileCount; file++) { + const path = `file-${file}.ts`; + guard.open(path); + nodeStart = raw.nodes.length; + relStart = raw.relationships.length; + for (let member = 0; member < 8; member++) { + const node = memberNode(path, member); + raw.nodes.push(node); + if (member % 2 === 0) + raw.relationships.push(memberEdge('DEFINES', `File:${path}`, node.id)); + raw.relationships.push(memberEdge('HAS_METHOD', `Class:${path}:Missing`, node.id)); + raw.relationships.push(memberEdge('HAS_PROPERTY', `Class:${path}:Missing`, node.id)); + } + } + guard.settle(); + // Bound work by current-file input, including duplicate and already-defined members. + expect(nodeReads).toBeGreaterThan(0); + expect(nodeReads).toBeLessThanOrEqual(fileCount * 8); + expect(relReads).toBeGreaterThan(0); + expect(relReads).toBeLessThanOrEqual(fileCount * 20 * 3); + expect(relWrites).toBeLessThanOrEqual(fileCount * 20); + expect(raw.relationships).toHaveLength(1 + fileCount * 8); + expect(raw.relationships[0]).toBe(prefixEdge); + expect(raw.relationships.every((rel) => rel.type === 'DEFINES')).toBe(true); + expect(new Set(raw.relationships.map((rel) => rel.targetId)).size).toBe( + raw.relationships.length, + ); + }); +}); diff --git a/gitnexus/test/unit/owner-edge-guard.test.ts b/gitnexus/test/unit/owner-edge-guard.test.ts new file mode 100644 index 000000000..2da9ea1a1 --- /dev/null +++ b/gitnexus/test/unit/owner-edge-guard.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from 'vitest'; + +import { + createOwnerEdgeGuard, + dropDanglingInFileOwnerEdges, +} from '../../src/core/ingestion/workers/owner-edge-guard.js'; +import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js'; + +type Slice = Pick; +type Rel = Slice['relationships'][number]; + +const node = (id: string): Slice['nodes'][number] => + ({ id, label: id.slice(0, id.indexOf(':')), properties: {} }) as Slice['nodes'][number]; + +const rel = (type: Rel['type'], sourceId: string, targetId: string): Rel => ({ + id: `${type}:${sourceId}->${targetId}`, + sourceId, + targetId, + type, + confidence: 1, + reason: '', +}); + +const summarize = (slice: Slice): string[] => + slice.relationships.map((r) => `${r.type} ${r.sourceId} -> ${r.targetId}`); + +describe('dropDanglingInFileOwnerEdges', () => { + it('re-attaches a member whose in-file owner was never emitted to its File', () => { + const slice: Slice = { + nodes: [node('Function:types/lib.d.ts:m.connect')], + relationships: [ + rel('HAS_METHOD', "Trait:types/lib.d.ts:'m'", 'Function:types/lib.d.ts:m.connect'), + ], + }; + dropDanglingInFileOwnerEdges(slice, 'types/lib.d.ts', 0, 0); + expect(summarize(slice)).toEqual([ + 'DEFINES File:types/lib.d.ts -> Function:types/lib.d.ts:m.connect', + ]); + }); + + it('drops the edge without duplicating an existing DEFINES', () => { + const slice: Slice = { + nodes: [node('Function:a.mjs:Context'), node('Function:a.mjs:Context.REPO')], + relationships: [ + rel('DEFINES', 'File:a.mjs', 'Function:a.mjs:Context.REPO'), + rel('HAS_METHOD', 'Const:a.mjs:Context', 'Function:a.mjs:Context.REPO'), + rel('HAS_PROPERTY', 'Const:a.mjs:Context', 'Property:a.mjs:Context.REF'), + ], + }; + slice.nodes.push(node('Property:a.mjs:Context.REF')); + dropDanglingInFileOwnerEdges(slice, 'a.mjs', 0, 0); + expect(summarize(slice)).toEqual([ + 'DEFINES File:a.mjs -> Function:a.mjs:Context.REPO', + 'DEFINES File:a.mjs -> Property:a.mjs:Context.REF', + ]); + }); + + it('keeps edges whose owner the file emitted, owners in other files, and non-member edges', () => { + const slice: Slice = { + nodes: [node('Const:a.ts:svc'), node('Method:a.ts:load'), node('Method:a.ts:Server.run')], + relationships: [ + rel('HAS_METHOD', 'Const:a.ts:svc', 'Method:a.ts:load'), + rel('HAS_METHOD', 'Struct:b.ts:Server', 'Method:a.ts:Server.run'), + rel('DECLARES', 'Const:a.ts:gone', 'Method:a.ts:load'), + ], + }; + const before = summarize(slice); + dropDanglingInFileOwnerEdges(slice, 'a.ts', 0, 0); + expect(summarize(slice)).toEqual(before); + }); + + it('does not treat a path that merely shares a prefix as the same file', () => { + const slice: Slice = { + nodes: [node('Method:a.ts:run')], + relationships: [rel('HAS_METHOD', 'Class:a.ts.bak:Foo', 'Method:a.ts:run')], + }; + const before = summarize(slice); + dropDanglingInFileOwnerEdges(slice, 'a.ts', 0, 0); + expect(summarize(slice)).toEqual(before); + }); + + it('only inspects the current file slice', () => { + const slice: Slice = { + nodes: [node('Method:prev.ts:m'), node('Method:cur.ts:m')], + relationships: [ + rel('HAS_METHOD', 'Const:cur.ts:Owner', 'Method:prev.ts:m'), + rel('HAS_METHOD', 'Const:cur.ts:Owner', 'Method:cur.ts:m'), + ], + }; + dropDanglingInFileOwnerEdges(slice, 'cur.ts', 1, 1); + expect(summarize(slice)).toEqual([ + 'HAS_METHOD Const:cur.ts:Owner -> Method:prev.ts:m', + 'DEFINES File:cur.ts -> Method:cur.ts:m', + ]); + }); + + it('treats an owner emitted earlier in the same file as present', () => { + const slice: Slice = { + nodes: [node('Class:cur.ts:Foo'), node('Method:cur.ts:Foo.m')], + relationships: [rel('HAS_METHOD', 'Class:cur.ts:Foo', 'Method:cur.ts:Foo.m')], + }; + dropDanglingInFileOwnerEdges(slice, 'cur.ts', 0, 0); + expect(summarize(slice)).toEqual(['HAS_METHOD Class:cur.ts:Foo -> Method:cur.ts:Foo.m']); + }); +}); + +describe('createOwnerEdgeGuard', () => { + it('settles the previous file when the next one opens', () => { + const slice: Slice = { nodes: [], relationships: [] }; + const guard = createOwnerEdgeGuard(slice); + guard.open('a.ts'); + slice.nodes.push(node('Method:a.ts:m')); + slice.relationships.push(rel('HAS_METHOD', 'Const:a.ts:Gone', 'Method:a.ts:m')); + guard.open('b.ts'); + expect(summarize(slice)).toEqual(['DEFINES File:a.ts -> Method:a.ts:m']); + }); + + it('settles a file whose processing threw', () => { + const slice: Slice = { nodes: [], relationships: [] }; + const guard = createOwnerEdgeGuard(slice); + expect(() => { + try { + guard.open('a.ts'); + slice.nodes.push(node('Function:a.ts:X.k')); + slice.relationships.push(rel('HAS_METHOD', 'Const:a.ts:X', 'Function:a.ts:X.k')); + throw new Error('extractor failed after member edges were appended'); + } finally { + guard.settle(); + } + }).toThrow('extractor failed'); + expect(summarize(slice)).toEqual(['DEFINES File:a.ts -> Function:a.ts:X.k']); + }); + + it('settles each file once', () => { + const slice: Slice = { nodes: [], relationships: [] }; + const guard = createOwnerEdgeGuard(slice); + guard.open('a.ts'); + slice.nodes.push(node('Method:a.ts:m')); + slice.relationships.push(rel('HAS_METHOD', 'Const:a.ts:Gone', 'Method:a.ts:m')); + guard.settle(); + slice.relationships.push(rel('HAS_METHOD', 'Const:a.ts:Late', 'Method:a.ts:m')); + guard.settle(); + expect(summarize(slice)).toEqual([ + 'DEFINES File:a.ts -> Method:a.ts:m', + 'HAS_METHOD Const:a.ts:Late -> Method:a.ts:m', + ]); + }); +}); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 71945fd5b..ba93e6c84 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -72,6 +72,7 @@ export default defineConfig({ test: { name: 'lbug-db', include: [ + 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', 'test/integration/rename-provider-*.test.ts', 'test/integration/pdg-explicit-owner.test.ts', @@ -183,6 +184,7 @@ export default defineConfig({ sequence: { groupOrder: 3 }, include: ['test/**/*.test.ts'], exclude: [ + 'test/integration/dangling-owner-edges.test.ts', 'test/integration/class-expression-owners.test.ts', 'test/integration/rename-provider-*.test.ts', 'test/integration/pdg-explicit-owner.test.ts',