mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-11-11 08:37:35 +00:00
fix(ingestion): drop member edges to owners the file never emitted (#3767)
* fix(ingestion): drop member edges to owners the file never emitted (#3747) The parse worker could emit HAS_METHOD/HAS_PROPERTY edges whose owner id named a node in the same file that was never materialized, so LadybugDB's COPY rejected them with "Unable to find primary key value" and the retry silently dropped the edge: - `const X = z.object({ k: f(cb) }).refine(cb)` is emitted as a Function, but its key's owner was spelled `Const:<file>:X`. - A TS `declare module 'm' {}` parses as a bare `module` node, which the shared container map labels `Trait`; no TS capture emits that node. After each file, the worker now removes member edges whose in-file owner was not emitted and re-attaches the member to its File with DEFINES (unless it already has one). Owners in other files are untouched and no node ids change. * fix(ingestion): settle the owner-edge guard when a file group throws processBatch catches a throw from processFileGroup and still returns the partial result, but the guard only settled the last open file after the loop completed. A throw after a member edge was appended (e.g. from an unguarded provider hook) let that file's dangling owner edge escape the guard and reach COPY. Per-file tracking now lives in createOwnerEdgeGuard, and processFileGroup settles it in a finally block around the per-file loop, so the partially processed file is checked on the error path too. --------- Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com>
This commit is contained in:
parent
bd9586e008
commit
36fefca17e
8 changed files with 861 additions and 0 deletions
|
|
@ -164,6 +164,7 @@ const PLATFORM_LOGIC = [
|
|||
// N-API addon which has known platform-specific behavior (Windows
|
||||
// file-lock lag after close, macOS N-API destructor segfaults)
|
||||
const LBUG_NATIVE = [
|
||||
'test/integration/dangling-owner-edges.test.ts',
|
||||
'test/integration/impact-private-type-file-seed.test.ts',
|
||||
'test/integration/impact-private-type-source.test.ts',
|
||||
// Inline-handler findings must survive native persistence and cold explain
|
||||
|
|
|
|||
321
gitnexus/scripts/owner-edge-guard-scaling.ts
Normal file
321
gitnexus/scripts/owner-edge-guard-scaling.ts
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
/**
|
||||
* Reproducible scaling probe for the per-file owner-edge guard.
|
||||
*
|
||||
* Run alone, after a build, from gitnexus/:
|
||||
* node --expose-gc --import tsx scripts/owner-edge-guard-scaling.ts > /tmp/owner-edge-scaling.json
|
||||
* --operations-only skips timings; --sizes=1024,2048,... --samples=11 --warmup=3 override defaults.
|
||||
*
|
||||
* Timings include guard lifecycle and appending preconstructed rows, excluding
|
||||
* fixture creation, prefix copies, assertions, and explicit GC. Prefix-isolation
|
||||
* timings call the guard directly on a fixed 16-member slice. Proxy counters run
|
||||
* separately: they are deterministic regression checks, never timed thresholds.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { arch, cpus, platform, release } from 'node:os';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { parseArgs } from 'node:util';
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import {
|
||||
createOwnerEdgeGuard,
|
||||
dropDanglingInFileOwnerEdges,
|
||||
} from '../src/core/ingestion/workers/owner-edge-guard.js';
|
||||
import type { ParseWorkerResult } from '../src/core/ingestion/workers/parse-worker.js';
|
||||
|
||||
const source = readFileSync(
|
||||
new URL('../src/core/ingestion/workers/owner-edge-guard.ts', import.meta.url),
|
||||
);
|
||||
|
||||
type Slice = Pick<ParseWorkerResult, 'nodes' | 'relationships'>;
|
||||
type Rel = Slice['relationships'][number];
|
||||
type Shape = 'member-free' | 'clean' | 'dangling' | 'predefined' | 'duplicate' | 'owner-last';
|
||||
type FileFixture = Slice & { path: string; expected: Rel[] };
|
||||
type Reads = { nodes: number; relationships: number; writes: number };
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
sizes: { type: 'string', default: '1024,2048,4096,8192,16384,32768,65536' },
|
||||
samples: { type: 'string', default: '11' },
|
||||
warmup: { type: 'string', default: '3' },
|
||||
'operations-only': { type: 'boolean', default: false },
|
||||
},
|
||||
});
|
||||
const sizes = values.sizes.split(',').map(Number);
|
||||
const samples = Number(values.samples);
|
||||
const warmup = Number(values.warmup);
|
||||
assert(
|
||||
sizes.length >= 2 &&
|
||||
sizes.every((n, i) => Number.isSafeInteger(n) && n > 0 && (!i || n > sizes[i - 1])),
|
||||
);
|
||||
assert(Number.isSafeInteger(samples) && samples >= 3);
|
||||
assert(Number.isSafeInteger(warmup) && warmup >= 1);
|
||||
|
||||
const edge = (type: Rel['type'], sourceId: string, targetId: string): Rel => ({
|
||||
id: `${type}:${sourceId}->${targetId}`,
|
||||
sourceId,
|
||||
targetId,
|
||||
type,
|
||||
confidence: 1,
|
||||
reason: '',
|
||||
});
|
||||
|
||||
function fixture(shape: Shape, count: number, file: number): FileFixture {
|
||||
// Fixed-width ids keep the string-length dimension constant across sizes.
|
||||
const path = `src/file-${String(file).padStart(8, '0')}.ts`;
|
||||
const owner = `Class:${path}:Owner`;
|
||||
const result: FileFixture = { path, nodes: [], relationships: [], expected: [] };
|
||||
const addNode = (id: string, label: string): void => {
|
||||
result.nodes.push({
|
||||
id,
|
||||
label,
|
||||
properties: {
|
||||
name: id,
|
||||
filePath: path,
|
||||
startLine: 0,
|
||||
endLine: 0,
|
||||
language: SupportedLanguages.TypeScript,
|
||||
isExported: false,
|
||||
},
|
||||
});
|
||||
};
|
||||
if (shape === 'clean') addNode(owner, 'Class');
|
||||
for (let i = 0; i < count; i++) {
|
||||
const target = `Method:${path}:member-${String(i).padStart(8, '0')}`;
|
||||
const defined = edge('DEFINES', `File:${path}`, target);
|
||||
const member = edge(i % 2 ? 'HAS_PROPERTY' : 'HAS_METHOD', owner, target);
|
||||
addNode(target, 'Method');
|
||||
if (shape === 'member-free' || shape === 'predefined') result.relationships.push(defined);
|
||||
if (shape !== 'member-free') result.relationships.push(member);
|
||||
if (shape === 'duplicate') result.relationships.push(edge('HAS_PROPERTY', owner, target));
|
||||
result.expected.push(shape === 'clean' || shape === 'owner-last' ? member : defined);
|
||||
}
|
||||
if (shape === 'owner-last') addNode(owner, 'Class');
|
||||
return result;
|
||||
}
|
||||
|
||||
function files(shape: Shape, count: number, perFile: number): FileFixture[] {
|
||||
const result: FileFixture[] = [];
|
||||
for (let i = 0; i < count; i += perFile)
|
||||
result.push(fixture(shape, Math.min(perFile, count - i), i / perFile));
|
||||
return result;
|
||||
}
|
||||
|
||||
function append(target: Slice, file: Slice): void {
|
||||
for (const node of file.nodes) target.nodes.push(node);
|
||||
for (const rel of file.relationships) target.relationships.push(rel);
|
||||
}
|
||||
|
||||
function verify(result: Slice, expected: Rel[], nodeCount: number): void {
|
||||
assert.equal(result.nodes.length, nodeCount);
|
||||
assert.deepEqual(result.relationships, expected);
|
||||
}
|
||||
|
||||
/** Count numeric array accesses only while the production guard runs. */
|
||||
function operations(input: FileFixture[], prefix: Slice): Reads {
|
||||
const raw: Slice = { nodes: prefix.nodes.slice(), relationships: prefix.relationships.slice() };
|
||||
let nodeStart = raw.nodes.length;
|
||||
let relStart = raw.relationships.length;
|
||||
const reads: Reads = { nodes: 0, relationships: 0, writes: 0 };
|
||||
const watched = <T>(array: T[], kind: 'nodes' | 'relationships'): T[] =>
|
||||
new Proxy(array, {
|
||||
get(target, property, receiver) {
|
||||
if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) {
|
||||
assert(
|
||||
Number(property) >= (kind === 'nodes' ? nodeStart : relStart),
|
||||
`rescanned previous ${kind}`,
|
||||
);
|
||||
reads[kind]++;
|
||||
}
|
||||
return Reflect.get(target, property, receiver);
|
||||
},
|
||||
set(target, property, value, receiver) {
|
||||
if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) {
|
||||
assert(
|
||||
Number(property) >= (kind === 'nodes' ? nodeStart : relStart),
|
||||
`rewrote previous ${kind}`,
|
||||
);
|
||||
reads.writes++;
|
||||
}
|
||||
return Reflect.set(target, property, value, receiver);
|
||||
},
|
||||
});
|
||||
const guarded = {
|
||||
nodes: watched(raw.nodes, 'nodes'),
|
||||
relationships: watched(raw.relationships, 'relationships'),
|
||||
};
|
||||
const guard = createOwnerEdgeGuard(guarded);
|
||||
for (const file of input) {
|
||||
guard.open(file.path); // Settles the previous slice before the boundaries advance.
|
||||
nodeStart = raw.nodes.length;
|
||||
relStart = raw.relationships.length;
|
||||
append(raw, file); // Fixture appends bypass the access counters.
|
||||
}
|
||||
guard.settle();
|
||||
const settledReads = { ...reads };
|
||||
guard.settle();
|
||||
assert.deepEqual(reads, settledReads, 'settling twice rescans a file');
|
||||
const n = input.reduce((sum, file) => sum + file.nodes.length, 0);
|
||||
const e = input.reduce((sum, file) => sum + file.relationships.length, 0);
|
||||
assert(reads.nodes <= n, 'more than one node pass');
|
||||
assert(reads.relationships <= 3 * e, 'more than three relationship passes');
|
||||
assert(reads.writes <= e, 'more than one compaction pass');
|
||||
verify(
|
||||
raw,
|
||||
[...prefix.relationships, ...input.flatMap((file) => file.expected)],
|
||||
prefix.nodes.length + n,
|
||||
);
|
||||
return reads;
|
||||
}
|
||||
|
||||
function trial(input: FileFixture[], prefix: Slice, direct: boolean, expected: Rel[]): number {
|
||||
const result: Slice = {
|
||||
nodes: prefix.nodes.slice(),
|
||||
relationships: prefix.relationships.slice(),
|
||||
};
|
||||
const nodeStart = result.nodes.length;
|
||||
const relStart = result.relationships.length;
|
||||
if (direct) append(result, input[0]);
|
||||
globalThis.gc?.();
|
||||
const start = performance.now();
|
||||
if (direct) {
|
||||
dropDanglingInFileOwnerEdges(result, input[0].path, nodeStart, relStart);
|
||||
} else {
|
||||
const guard = createOwnerEdgeGuard(result);
|
||||
for (const file of input) {
|
||||
guard.open(file.path);
|
||||
append(result, file);
|
||||
}
|
||||
guard.settle();
|
||||
}
|
||||
const elapsed = performance.now() - start;
|
||||
verify(
|
||||
result,
|
||||
expected,
|
||||
prefix.nodes.length + input.reduce((sum, file) => sum + file.nodes.length, 0),
|
||||
);
|
||||
return elapsed;
|
||||
}
|
||||
|
||||
const percentile = (values: number[], fraction: number): number => {
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[Math.ceil(fraction * sorted.length) - 1];
|
||||
};
|
||||
|
||||
function measure(shape: Shape, count: number, perFile: number, prefixCount = 0) {
|
||||
const input = files(shape, count, perFile);
|
||||
const prefix = prefixCount
|
||||
? fixture('member-free', prefixCount, 99999999)
|
||||
: { nodes: [], relationships: [] };
|
||||
const nodes = input.reduce((sum, file) => sum + file.nodes.length, 0);
|
||||
const relationships = input.reduce((sum, file) => sum + file.relationships.length, 0);
|
||||
const counts = operations(input, prefix);
|
||||
const direct = prefixCount > 0;
|
||||
const samplesMs: number[] = [];
|
||||
if (!values['operations-only']) {
|
||||
for (let i = 0; i < warmup + samples; i++) {
|
||||
const freshInput = files(shape, count, perFile);
|
||||
const expected = [...prefix.relationships, ...freshInput.flatMap((file) => file.expected)];
|
||||
const elapsed = trial(freshInput, prefix, direct, expected);
|
||||
if (i >= warmup) samplesMs.push(elapsed);
|
||||
}
|
||||
}
|
||||
const medianMs = samplesMs.length ? percentile(samplesMs, 0.5) : null;
|
||||
return {
|
||||
shape,
|
||||
members: count,
|
||||
files: input.length,
|
||||
prefixRows: prefix.nodes.length + prefix.relationships.length,
|
||||
nodes,
|
||||
relationships,
|
||||
indexedReads: counts,
|
||||
samplesMs,
|
||||
medianMs,
|
||||
p95Ms: samplesMs.length ? percentile(samplesMs, 0.95) : null,
|
||||
nsPerRow: medianMs === null ? null : (medianMs * 1e6) / (nodes + relationships),
|
||||
};
|
||||
}
|
||||
|
||||
function slope(points: { x: number; y: number }[]): number | null {
|
||||
if (points.length < 2 || points.some(({ x, y }) => x <= 0 || y <= 0)) return null;
|
||||
const xs = points.map(({ x }) => Math.log(x));
|
||||
const ys = points.map(({ y }) => Math.log(y));
|
||||
const meanX = xs.reduce((a, b) => a + b, 0) / xs.length;
|
||||
const meanY = ys.reduce((a, b) => a + b, 0) / ys.length;
|
||||
return (
|
||||
xs.reduce((sum, x, i) => sum + (x - meanX) * (ys[i] - meanY), 0) /
|
||||
xs.reduce((sum, x) => sum + (x - meanX) ** 2, 0)
|
||||
);
|
||||
}
|
||||
|
||||
const cases: ReturnType<typeof measure>[] = [];
|
||||
const scaling: { shape: Shape; distribution: string; logLogSlope: number | null }[] = [];
|
||||
const shapes: Shape[] = [
|
||||
'member-free',
|
||||
'clean',
|
||||
'dangling',
|
||||
'predefined',
|
||||
'duplicate',
|
||||
'owner-last',
|
||||
];
|
||||
// Alternate ascending/descending sizes to avoid tying every large case to late-run thermal drift.
|
||||
for (const [shapeIndex, shape] of shapes.entries()) {
|
||||
for (const distribution of ['one-file', 'many-files']) {
|
||||
const rows = (shapeIndex % 2 ? [...sizes].reverse() : sizes).map((size) =>
|
||||
measure(shape, size, distribution === 'one-file' ? size : 16),
|
||||
);
|
||||
cases.push(...rows);
|
||||
scaling.push({
|
||||
shape,
|
||||
distribution,
|
||||
logLogSlope: values['operations-only']
|
||||
? null
|
||||
: slope(
|
||||
rows.flatMap((row) =>
|
||||
row.medianMs === null ? [] : [{ x: row.nodes + row.relationships, y: row.medianMs }],
|
||||
),
|
||||
),
|
||||
});
|
||||
process.stderr.write(`Validated ${shape} / ${distribution}\n`);
|
||||
}
|
||||
}
|
||||
const prefixIsolation = sizes.map((size) => measure('dangling', 16, 16, size));
|
||||
process.stdout.write(
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schema: 'gitnexus.owner-edge-guard-scaling/v1',
|
||||
recordedAt: new Date().toISOString(),
|
||||
environment: {
|
||||
node: process.version,
|
||||
v8: process.versions.v8,
|
||||
platform: platform(),
|
||||
arch: arch(),
|
||||
release: release(),
|
||||
cpu: cpus()[0]?.model,
|
||||
explicitGc: typeof globalThis.gc === 'function',
|
||||
processMaxRssKiB: process.resourceUsage().maxRSS,
|
||||
},
|
||||
sourceSha256: createHash('sha256').update(source).digest('hex'),
|
||||
config: { sizes, samples, warmup, operationsOnly: values['operations-only'] },
|
||||
methodology: {
|
||||
timed:
|
||||
'guard lifecycle + append preconstructed rows; prefixIsolation times only the direct guard',
|
||||
excluded:
|
||||
'fresh fixture generation for every sample, prefix copies, assertions, explicit GC, Proxy instrumentation',
|
||||
statistics:
|
||||
'nearest-rank median/p95; least-squares slope of log(median ms) versus log(active rows)',
|
||||
operations:
|
||||
'at most one indexed node read, three indexed relationship reads, one relationship write per active row; no earlier-slice indexed accesses',
|
||||
limits:
|
||||
'Kernel benchmark, not end-to-end indexing throughput. Timings and slopes are descriptive, not asymptotic proof. maxRSS covers all fixtures and validation and is not guard peak memory. Set operations assume average constant-time hashing; string work scales with id lengths.',
|
||||
memory:
|
||||
'Live guard scratch is O(nodes + edges) of the largest active file; output accumulation and delayed garbage collection are excluded from that bound.',
|
||||
},
|
||||
cases,
|
||||
scaling,
|
||||
prefixIsolation,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
109
gitnexus/src/core/ingestion/workers/owner-edge-guard.ts
Normal file
109
gitnexus/src/core/ingestion/workers/owner-edge-guard.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
/**
|
||||
* Per-file guard against member edges whose owner node was never emitted.
|
||||
*
|
||||
* Owner ids are minted from syntax (the binding an object literal is assigned
|
||||
* to, the enclosing container), while node labels come from the query captures,
|
||||
* so the two can disagree (#3747): `const X = z.object({ k: f(() => 0) }).refine(cb)`
|
||||
* is emitted as `Function:<file>:X` but its key's owner is spelled
|
||||
* `Const:<file>:X`, and a TS `declare module 'm' {}` owns its members under a
|
||||
* `Trait` id no TS capture ever emits. LadybugDB's COPY rejects such an edge and
|
||||
* the retry silently drops it.
|
||||
*
|
||||
* Kept free of worker side effects so it can be unit-tested directly.
|
||||
*/
|
||||
import { generateId } from '../../../lib/utils.js';
|
||||
import type { ParseWorkerResult } from './parse-worker.js';
|
||||
|
||||
const isMemberEdge = (type: string): boolean => type === 'HAS_METHOD' || type === 'HAS_PROPERTY';
|
||||
|
||||
/**
|
||||
* Remove `HAS_METHOD` / `HAS_PROPERTY` edges appended for `filePath` (from
|
||||
* `relStart` on) whose owner id names a node in that same file that the file
|
||||
* did not emit (from `nodeStart` on). The orphaned member is re-attached to its
|
||||
* File with `DEFINES` unless it already has one.
|
||||
*
|
||||
* Owners in other files are left alone: they are resolved across the whole
|
||||
* graph, not per file.
|
||||
*/
|
||||
export const dropDanglingInFileOwnerEdges = (
|
||||
result: Pick<ParseWorkerResult, 'nodes' | 'relationships'>,
|
||||
filePath: string,
|
||||
nodeStart: number,
|
||||
relStart: number,
|
||||
): void => {
|
||||
const rels = result.relationships;
|
||||
const inFilePrefix = `:${filePath}:`;
|
||||
let emitted: Set<string> | null = null;
|
||||
let dangling: Set<number> | null = null;
|
||||
|
||||
for (let i = relStart; i < rels.length; i++) {
|
||||
const rel = rels[i];
|
||||
if (!isMemberEdge(rel.type)) continue;
|
||||
const labelEnd = rel.sourceId.indexOf(':');
|
||||
if (labelEnd < 0 || !rel.sourceId.startsWith(inFilePrefix, labelEnd)) continue;
|
||||
if (emitted === null) {
|
||||
emitted = new Set<string>();
|
||||
for (let n = nodeStart; n < result.nodes.length; n++) emitted.add(result.nodes[n].id);
|
||||
}
|
||||
if (!emitted.has(rel.sourceId)) (dangling ??= new Set<number>()).add(i);
|
||||
}
|
||||
if (dangling === null || emitted === null) return;
|
||||
|
||||
const fileId = generateId('File', filePath);
|
||||
const defined = new Set<string>();
|
||||
for (let i = relStart; i < rels.length; i++) {
|
||||
const rel = rels[i];
|
||||
if (rel.type === 'DEFINES' && rel.sourceId === fileId) defined.add(rel.targetId);
|
||||
}
|
||||
|
||||
let write = relStart;
|
||||
for (let i = relStart; i < rels.length; i++) {
|
||||
const rel = rels[i];
|
||||
if (!dangling.has(i)) {
|
||||
rels[write++] = rel;
|
||||
continue;
|
||||
}
|
||||
if (defined.has(rel.targetId) || !emitted.has(rel.targetId)) continue;
|
||||
defined.add(rel.targetId);
|
||||
rels[write++] = {
|
||||
id: generateId('DEFINES', `${fileId}->${rel.targetId}`),
|
||||
sourceId: fileId,
|
||||
targetId: rel.targetId,
|
||||
type: 'DEFINES',
|
||||
confidence: 1.0,
|
||||
reason: '',
|
||||
};
|
||||
}
|
||||
rels.length = write;
|
||||
};
|
||||
|
||||
/** Tracks the file whose slice of a worker result is still being appended. */
|
||||
export interface OwnerEdgeGuard {
|
||||
/** Settle the previous file, then start tracking what `filePath` appends. */
|
||||
open(filePath: string): void;
|
||||
/** Settle the tracked file, if any. Safe to call more than once. */
|
||||
settle(): void;
|
||||
}
|
||||
|
||||
export const createOwnerEdgeGuard = (
|
||||
result: Pick<ParseWorkerResult, 'nodes' | 'relationships'>,
|
||||
): OwnerEdgeGuard => {
|
||||
let current: { path: string; nodeStart: number; relStart: number } | null = null;
|
||||
const settle = (): void => {
|
||||
if (current === null) return;
|
||||
const file = current;
|
||||
current = null;
|
||||
dropDanglingInFileOwnerEdges(result, file.path, file.nodeStart, file.relStart);
|
||||
};
|
||||
return {
|
||||
open(filePath: string): void {
|
||||
settle();
|
||||
current = {
|
||||
path: filePath,
|
||||
nodeStart: result.nodes.length,
|
||||
relStart: result.relationships.length,
|
||||
};
|
||||
},
|
||||
settle,
|
||||
};
|
||||
};
|
||||
|
|
@ -138,6 +138,7 @@ import { buildTypeEnv } from '../type-env.js';
|
|||
import type { ConstructorBinding } from '../type-env.js';
|
||||
import { detectFrameworkFromAST } from '../framework-detection.js';
|
||||
import { generateId } from '../../../lib/utils.js';
|
||||
import { createOwnerEdgeGuard, type OwnerEdgeGuard } from './owner-edge-guard.js';
|
||||
import { defaultExportNameCollides } from '../languages/typescript/cjs-export-assignment.js';
|
||||
import {
|
||||
extractVueScript,
|
||||
|
|
@ -1595,6 +1596,23 @@ const processFileGroup = (
|
|||
queryString: string,
|
||||
result: ParseWorkerResult,
|
||||
onFileProcessed?: () => void,
|
||||
): void => {
|
||||
const ownerEdgeGuard = createOwnerEdgeGuard(result);
|
||||
try {
|
||||
processFileGroupFiles(files, language, queryString, result, ownerEdgeGuard, onFileProcessed);
|
||||
} finally {
|
||||
// Also on a throw: processBatch keeps the partial result of a failed group.
|
||||
ownerEdgeGuard.settle();
|
||||
}
|
||||
};
|
||||
|
||||
const processFileGroupFiles = (
|
||||
files: ParseWorkerInput[],
|
||||
language: SupportedLanguages,
|
||||
queryString: string,
|
||||
result: ParseWorkerResult,
|
||||
ownerEdgeGuard: OwnerEdgeGuard,
|
||||
onFileProcessed?: () => void,
|
||||
): void => {
|
||||
let query: Parser.Query;
|
||||
try {
|
||||
|
|
@ -1615,6 +1633,8 @@ const processFileGroup = (
|
|||
}
|
||||
|
||||
for (const file of files) {
|
||||
// Settles the previous file, so every `continue` below still gets checked.
|
||||
ownerEdgeGuard.open(file.path);
|
||||
// Skip files larger than the max tree-sitter buffer (32 MB)
|
||||
if (getTreeSitterContentByteLength(file.content) > TREE_SITTER_MAX_BUFFER) continue;
|
||||
|
||||
|
|
|
|||
163
gitnexus/test/integration/dangling-owner-edges.test.ts
Normal file
163
gitnexus/test/integration/dangling-owner-edges.test.ts
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
/**
|
||||
* #3747: a member edge must never name an owner node the file did not emit.
|
||||
* Both fixtures are the reporter's repro; before the fix each produced one
|
||||
* HAS_METHOD edge that LadybugDB's COPY rejected with
|
||||
* "Unable to find primary key value".
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import {
|
||||
closeLbug,
|
||||
executeQuery,
|
||||
initLbug,
|
||||
loadGraphToLbug,
|
||||
} from '../../src/core/lbug/lbug-adapter.js';
|
||||
import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js';
|
||||
import { createTempDir } from '../helpers/test-db.js';
|
||||
import { DIST_WORKER_URL, parseFilesWithWorkers } from '../helpers/worker-parse.js';
|
||||
import { createWorkerPool } from '../../src/core/ingestion/workers/worker-pool.js';
|
||||
import type {
|
||||
ParseWorkerInput,
|
||||
ParseWorkerResult,
|
||||
} from '../../src/core/ingestion/workers/parse-worker.js';
|
||||
import { findDanglingEdges } from './resolvers/helpers.js';
|
||||
|
||||
const CHAINED_CONST = `import { z } from 'zod';
|
||||
|
||||
const KEY = 'BRANCH';
|
||||
const Branch = z.string();
|
||||
|
||||
const Context = z
|
||||
.object({
|
||||
REF: z.string(),
|
||||
[KEY]: Branch,
|
||||
REPO: z.string().refine((value) => value.split('/').every((p) => p !== '..')),
|
||||
})
|
||||
.refine((context) => context.REF === \`refs/heads/\${context[KEY]}\`, {
|
||||
message: 'REF must be the branch head',
|
||||
});
|
||||
|
||||
export function run(env) {
|
||||
const context = Context.parse(env);
|
||||
return context;
|
||||
}
|
||||
`;
|
||||
|
||||
const AMBIENT_MODULE = `declare module 'somelib' {
|
||||
export function connect(opts: { user: string }): Promise<void>;
|
||||
const lib: { connect: typeof connect };
|
||||
export default lib;
|
||||
}
|
||||
`;
|
||||
|
||||
const OBJECT_LITERAL_CONTROL = `export const service = {
|
||||
load() { return 1; },
|
||||
save: () => 2,
|
||||
};
|
||||
`;
|
||||
|
||||
describe('#3747 member edges only name emitted owners', () => {
|
||||
it('leaves no dangling member edge and keeps every member reachable from its file', async () => {
|
||||
const { graph } = await parseFilesWithWorkers([
|
||||
{ path: 'ctx.mjs', content: CHAINED_CONST },
|
||||
{ path: 'types/lib.d.ts', content: AMBIENT_MODULE },
|
||||
]);
|
||||
expect(findDanglingEdges({ graph } as never, ['HAS_METHOD', 'HAS_PROPERTY'])).toEqual([]);
|
||||
|
||||
const definedFromFile = (file: string): string[] =>
|
||||
graph.relationships
|
||||
.filter((edge) => edge.type === 'DEFINES' && edge.sourceId === `File:${file}`)
|
||||
.map((edge) => edge.targetId);
|
||||
const repo = graph.nodes.find(
|
||||
(n) => n.properties.filePath === 'ctx.mjs' && n.properties.name === 'REPO',
|
||||
);
|
||||
const connect = graph.nodes.find(
|
||||
(n) => n.properties.filePath === 'types/lib.d.ts' && n.properties.name === 'connect',
|
||||
);
|
||||
expect(repo).toBeDefined();
|
||||
expect(connect).toBeDefined();
|
||||
expect(definedFromFile('ctx.mjs')).toContain(repo?.id);
|
||||
expect(definedFromFile('types/lib.d.ts')).toContain(connect?.id);
|
||||
});
|
||||
|
||||
it('keeps object-literal members owned by their emitted binding', async () => {
|
||||
const { graph } = await parseFilesWithWorkers([
|
||||
{ path: 'service.ts', content: OBJECT_LITERAL_CONTROL },
|
||||
]);
|
||||
const owner = graph.nodes.find(
|
||||
(n) => n.properties.filePath === 'service.ts' && n.properties.name === 'service',
|
||||
);
|
||||
expect(owner).toBeDefined();
|
||||
const members = graph.relationships
|
||||
.filter((edge) => edge.type === 'HAS_METHOD' && edge.sourceId === owner?.id)
|
||||
.map((edge) => graph.getNode(edge.targetId)?.properties.name)
|
||||
.sort();
|
||||
expect(members).toEqual(['load', 'save']);
|
||||
});
|
||||
|
||||
it('persists every relationship through native COPY', async () => {
|
||||
const temp = await createTempDir();
|
||||
try {
|
||||
await fs.mkdir(path.join(temp.dbPath, 'types'), { recursive: true });
|
||||
await fs.writeFile(path.join(temp.dbPath, 'ctx.mjs'), CHAINED_CONST);
|
||||
await fs.writeFile(path.join(temp.dbPath, 'types', 'lib.d.ts'), AMBIENT_MODULE);
|
||||
const result = await runPipelineFromRepo(temp.dbPath, () => {}, { workerPoolSize: 1 });
|
||||
expect(result.usedWorkerPool).toBe(true);
|
||||
expect(findDanglingEdges(result)).toEqual([]);
|
||||
|
||||
await initLbug(path.join(temp.dbPath, 'lbug'), { skipFts: true });
|
||||
await loadGraphToLbug(result.graph, temp.dbPath, path.join(temp.dbPath, 'storage'));
|
||||
const [row] = await executeQuery('MATCH ()-[r:CodeRelation]->() RETURN count(r) AS edges');
|
||||
expect(Number(row.edges)).toBe(result.graph.relationships.length);
|
||||
} finally {
|
||||
await closeLbug();
|
||||
await temp.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it('settles the partial file when a real parser provider throws after emitting members', async () => {
|
||||
const temp = await createTempDir();
|
||||
const bootstrap = path.join(temp.dbPath, 'throwing-provider.mjs');
|
||||
const providersUrl = new URL('../languages/index.js', DIST_WORKER_URL).href;
|
||||
// Inject the failure in the worker isolate, leaving the production loop and
|
||||
// its finally intact. Removing that finally must make this regression fail.
|
||||
await fs.writeFile(
|
||||
bootstrap,
|
||||
`
|
||||
const { getProvider } = await import(${JSON.stringify(providersUrl)});
|
||||
getProvider('javascript').extractToolDefinitions = () => {
|
||||
throw new Error('owner-edge-guard provider fault');
|
||||
};
|
||||
await import(${JSON.stringify(DIST_WORKER_URL.href)});
|
||||
`,
|
||||
);
|
||||
const pool = createWorkerPool(pathToFileURL(bootstrap), 1);
|
||||
try {
|
||||
const [result] = await pool.dispatch<ParseWorkerInput, ParseWorkerResult>([
|
||||
{ path: 'ctx.mjs', content: CHAINED_CONST },
|
||||
{ path: 'after.mjs', content: 'export function shouldNotParse() {}' },
|
||||
]);
|
||||
// The first file is counted before the provider throws; the remaining
|
||||
// file is skipped, proving the injected failure reached processBatch.
|
||||
expect(result.fileCount).toBe(1);
|
||||
expect(result.nodes.some((node) => node.properties.name === 'shouldNotParse')).toBe(false);
|
||||
const member = result.nodes.find((node) => node.properties.name === 'REPO');
|
||||
expect(member).toBeDefined();
|
||||
expect(result.relationships.filter((edge) => edge.targetId === member?.id)).toEqual([
|
||||
{
|
||||
id: `DEFINES:File:ctx.mjs->${member?.id}`,
|
||||
sourceId: 'File:ctx.mjs',
|
||||
targetId: member?.id,
|
||||
type: 'DEFINES',
|
||||
confidence: 1,
|
||||
reason: '',
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
await pool.terminate();
|
||||
await temp.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
97
gitnexus/test/unit/owner-edge-guard-scaling.test.ts
Normal file
97
gitnexus/test/unit/owner-edge-guard-scaling.test.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import { createOwnerEdgeGuard } from '../../src/core/ingestion/workers/owner-edge-guard.js';
|
||||
import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js';
|
||||
|
||||
type Slice = Pick<ParseWorkerResult, 'nodes' | 'relationships'>;
|
||||
type Rel = Slice['relationships'][number];
|
||||
|
||||
const memberNode = (path: string, member: number): Slice['nodes'][number] => ({
|
||||
id: `Method:${path}:m${member}`,
|
||||
label: 'Method',
|
||||
properties: {
|
||||
name: `m${member}`,
|
||||
filePath: path,
|
||||
startLine: 0,
|
||||
endLine: 0,
|
||||
language: SupportedLanguages.TypeScript,
|
||||
isExported: false,
|
||||
},
|
||||
});
|
||||
|
||||
const memberEdge = (type: Rel['type'], sourceId: string, targetId: string): Rel => ({
|
||||
id: `${type}:${sourceId}->${targetId}`,
|
||||
sourceId,
|
||||
targetId,
|
||||
type,
|
||||
confidence: 1,
|
||||
reason: '',
|
||||
});
|
||||
|
||||
describe('owner-edge guard scaling', () => {
|
||||
it.each([8, 32, 128])('processes %i files without rescanning completed files', (fileCount) => {
|
||||
const prefixNode = memberNode('previous.ts', 0);
|
||||
const prefixEdge = memberEdge('DEFINES', 'File:previous.ts', prefixNode.id);
|
||||
const raw: Slice = { nodes: [prefixNode], relationships: [prefixEdge] };
|
||||
let nodeStart = 1;
|
||||
let relStart = 1;
|
||||
let nodeReads = 0;
|
||||
let relReads = 0;
|
||||
let relWrites = 0;
|
||||
const guarded: Slice = {
|
||||
nodes: new Proxy(raw.nodes, {
|
||||
get(target, property, receiver) {
|
||||
if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) {
|
||||
expect(Number(property)).toBeGreaterThanOrEqual(nodeStart);
|
||||
nodeReads++;
|
||||
}
|
||||
return Reflect.get(target, property, receiver);
|
||||
},
|
||||
}),
|
||||
relationships: new Proxy(raw.relationships, {
|
||||
get(target, property, receiver) {
|
||||
if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) {
|
||||
expect(Number(property)).toBeGreaterThanOrEqual(relStart);
|
||||
relReads++;
|
||||
}
|
||||
return Reflect.get(target, property, receiver);
|
||||
},
|
||||
set(target, property, value, receiver) {
|
||||
if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) {
|
||||
expect(Number(property)).toBeGreaterThanOrEqual(relStart);
|
||||
relWrites++;
|
||||
}
|
||||
return Reflect.set(target, property, value, receiver);
|
||||
},
|
||||
}),
|
||||
};
|
||||
const guard = createOwnerEdgeGuard(guarded);
|
||||
for (let file = 0; file < fileCount; file++) {
|
||||
const path = `file-${file}.ts`;
|
||||
guard.open(path);
|
||||
nodeStart = raw.nodes.length;
|
||||
relStart = raw.relationships.length;
|
||||
for (let member = 0; member < 8; member++) {
|
||||
const node = memberNode(path, member);
|
||||
raw.nodes.push(node);
|
||||
if (member % 2 === 0)
|
||||
raw.relationships.push(memberEdge('DEFINES', `File:${path}`, node.id));
|
||||
raw.relationships.push(memberEdge('HAS_METHOD', `Class:${path}:Missing`, node.id));
|
||||
raw.relationships.push(memberEdge('HAS_PROPERTY', `Class:${path}:Missing`, node.id));
|
||||
}
|
||||
}
|
||||
guard.settle();
|
||||
// Bound work by current-file input, including duplicate and already-defined members.
|
||||
expect(nodeReads).toBeGreaterThan(0);
|
||||
expect(nodeReads).toBeLessThanOrEqual(fileCount * 8);
|
||||
expect(relReads).toBeGreaterThan(0);
|
||||
expect(relReads).toBeLessThanOrEqual(fileCount * 20 * 3);
|
||||
expect(relWrites).toBeLessThanOrEqual(fileCount * 20);
|
||||
expect(raw.relationships).toHaveLength(1 + fileCount * 8);
|
||||
expect(raw.relationships[0]).toBe(prefixEdge);
|
||||
expect(raw.relationships.every((rel) => rel.type === 'DEFINES')).toBe(true);
|
||||
expect(new Set(raw.relationships.map((rel) => rel.targetId)).size).toBe(
|
||||
raw.relationships.length,
|
||||
);
|
||||
});
|
||||
});
|
||||
148
gitnexus/test/unit/owner-edge-guard.test.ts
Normal file
148
gitnexus/test/unit/owner-edge-guard.test.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
createOwnerEdgeGuard,
|
||||
dropDanglingInFileOwnerEdges,
|
||||
} from '../../src/core/ingestion/workers/owner-edge-guard.js';
|
||||
import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js';
|
||||
|
||||
type Slice = Pick<ParseWorkerResult, 'nodes' | 'relationships'>;
|
||||
type Rel = Slice['relationships'][number];
|
||||
|
||||
const node = (id: string): Slice['nodes'][number] =>
|
||||
({ id, label: id.slice(0, id.indexOf(':')), properties: {} }) as Slice['nodes'][number];
|
||||
|
||||
const rel = (type: Rel['type'], sourceId: string, targetId: string): Rel => ({
|
||||
id: `${type}:${sourceId}->${targetId}`,
|
||||
sourceId,
|
||||
targetId,
|
||||
type,
|
||||
confidence: 1,
|
||||
reason: '',
|
||||
});
|
||||
|
||||
const summarize = (slice: Slice): string[] =>
|
||||
slice.relationships.map((r) => `${r.type} ${r.sourceId} -> ${r.targetId}`);
|
||||
|
||||
describe('dropDanglingInFileOwnerEdges', () => {
|
||||
it('re-attaches a member whose in-file owner was never emitted to its File', () => {
|
||||
const slice: Slice = {
|
||||
nodes: [node('Function:types/lib.d.ts:m.connect')],
|
||||
relationships: [
|
||||
rel('HAS_METHOD', "Trait:types/lib.d.ts:'m'", 'Function:types/lib.d.ts:m.connect'),
|
||||
],
|
||||
};
|
||||
dropDanglingInFileOwnerEdges(slice, 'types/lib.d.ts', 0, 0);
|
||||
expect(summarize(slice)).toEqual([
|
||||
'DEFINES File:types/lib.d.ts -> Function:types/lib.d.ts:m.connect',
|
||||
]);
|
||||
});
|
||||
|
||||
it('drops the edge without duplicating an existing DEFINES', () => {
|
||||
const slice: Slice = {
|
||||
nodes: [node('Function:a.mjs:Context'), node('Function:a.mjs:Context.REPO')],
|
||||
relationships: [
|
||||
rel('DEFINES', 'File:a.mjs', 'Function:a.mjs:Context.REPO'),
|
||||
rel('HAS_METHOD', 'Const:a.mjs:Context', 'Function:a.mjs:Context.REPO'),
|
||||
rel('HAS_PROPERTY', 'Const:a.mjs:Context', 'Property:a.mjs:Context.REF'),
|
||||
],
|
||||
};
|
||||
slice.nodes.push(node('Property:a.mjs:Context.REF'));
|
||||
dropDanglingInFileOwnerEdges(slice, 'a.mjs', 0, 0);
|
||||
expect(summarize(slice)).toEqual([
|
||||
'DEFINES File:a.mjs -> Function:a.mjs:Context.REPO',
|
||||
'DEFINES File:a.mjs -> Property:a.mjs:Context.REF',
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps edges whose owner the file emitted, owners in other files, and non-member edges', () => {
|
||||
const slice: Slice = {
|
||||
nodes: [node('Const:a.ts:svc'), node('Method:a.ts:load'), node('Method:a.ts:Server.run')],
|
||||
relationships: [
|
||||
rel('HAS_METHOD', 'Const:a.ts:svc', 'Method:a.ts:load'),
|
||||
rel('HAS_METHOD', 'Struct:b.ts:Server', 'Method:a.ts:Server.run'),
|
||||
rel('DECLARES', 'Const:a.ts:gone', 'Method:a.ts:load'),
|
||||
],
|
||||
};
|
||||
const before = summarize(slice);
|
||||
dropDanglingInFileOwnerEdges(slice, 'a.ts', 0, 0);
|
||||
expect(summarize(slice)).toEqual(before);
|
||||
});
|
||||
|
||||
it('does not treat a path that merely shares a prefix as the same file', () => {
|
||||
const slice: Slice = {
|
||||
nodes: [node('Method:a.ts:run')],
|
||||
relationships: [rel('HAS_METHOD', 'Class:a.ts.bak:Foo', 'Method:a.ts:run')],
|
||||
};
|
||||
const before = summarize(slice);
|
||||
dropDanglingInFileOwnerEdges(slice, 'a.ts', 0, 0);
|
||||
expect(summarize(slice)).toEqual(before);
|
||||
});
|
||||
|
||||
it('only inspects the current file slice', () => {
|
||||
const slice: Slice = {
|
||||
nodes: [node('Method:prev.ts:m'), node('Method:cur.ts:m')],
|
||||
relationships: [
|
||||
rel('HAS_METHOD', 'Const:cur.ts:Owner', 'Method:prev.ts:m'),
|
||||
rel('HAS_METHOD', 'Const:cur.ts:Owner', 'Method:cur.ts:m'),
|
||||
],
|
||||
};
|
||||
dropDanglingInFileOwnerEdges(slice, 'cur.ts', 1, 1);
|
||||
expect(summarize(slice)).toEqual([
|
||||
'HAS_METHOD Const:cur.ts:Owner -> Method:prev.ts:m',
|
||||
'DEFINES File:cur.ts -> Method:cur.ts:m',
|
||||
]);
|
||||
});
|
||||
|
||||
it('treats an owner emitted earlier in the same file as present', () => {
|
||||
const slice: Slice = {
|
||||
nodes: [node('Class:cur.ts:Foo'), node('Method:cur.ts:Foo.m')],
|
||||
relationships: [rel('HAS_METHOD', 'Class:cur.ts:Foo', 'Method:cur.ts:Foo.m')],
|
||||
};
|
||||
dropDanglingInFileOwnerEdges(slice, 'cur.ts', 0, 0);
|
||||
expect(summarize(slice)).toEqual(['HAS_METHOD Class:cur.ts:Foo -> Method:cur.ts:Foo.m']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createOwnerEdgeGuard', () => {
|
||||
it('settles the previous file when the next one opens', () => {
|
||||
const slice: Slice = { nodes: [], relationships: [] };
|
||||
const guard = createOwnerEdgeGuard(slice);
|
||||
guard.open('a.ts');
|
||||
slice.nodes.push(node('Method:a.ts:m'));
|
||||
slice.relationships.push(rel('HAS_METHOD', 'Const:a.ts:Gone', 'Method:a.ts:m'));
|
||||
guard.open('b.ts');
|
||||
expect(summarize(slice)).toEqual(['DEFINES File:a.ts -> Method:a.ts:m']);
|
||||
});
|
||||
|
||||
it('settles a file whose processing threw', () => {
|
||||
const slice: Slice = { nodes: [], relationships: [] };
|
||||
const guard = createOwnerEdgeGuard(slice);
|
||||
expect(() => {
|
||||
try {
|
||||
guard.open('a.ts');
|
||||
slice.nodes.push(node('Function:a.ts:X.k'));
|
||||
slice.relationships.push(rel('HAS_METHOD', 'Const:a.ts:X', 'Function:a.ts:X.k'));
|
||||
throw new Error('extractor failed after member edges were appended');
|
||||
} finally {
|
||||
guard.settle();
|
||||
}
|
||||
}).toThrow('extractor failed');
|
||||
expect(summarize(slice)).toEqual(['DEFINES File:a.ts -> Function:a.ts:X.k']);
|
||||
});
|
||||
|
||||
it('settles each file once', () => {
|
||||
const slice: Slice = { nodes: [], relationships: [] };
|
||||
const guard = createOwnerEdgeGuard(slice);
|
||||
guard.open('a.ts');
|
||||
slice.nodes.push(node('Method:a.ts:m'));
|
||||
slice.relationships.push(rel('HAS_METHOD', 'Const:a.ts:Gone', 'Method:a.ts:m'));
|
||||
guard.settle();
|
||||
slice.relationships.push(rel('HAS_METHOD', 'Const:a.ts:Late', 'Method:a.ts:m'));
|
||||
guard.settle();
|
||||
expect(summarize(slice)).toEqual([
|
||||
'DEFINES File:a.ts -> Method:a.ts:m',
|
||||
'HAS_METHOD Const:a.ts:Late -> Method:a.ts:m',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
@ -72,6 +72,7 @@ export default defineConfig({
|
|||
test: {
|
||||
name: 'lbug-db',
|
||||
include: [
|
||||
'test/integration/dangling-owner-edges.test.ts',
|
||||
'test/integration/class-expression-owners.test.ts',
|
||||
'test/integration/rename-provider-*.test.ts',
|
||||
'test/integration/pdg-explicit-owner.test.ts',
|
||||
|
|
@ -183,6 +184,7 @@ export default defineConfig({
|
|||
sequence: { groupOrder: 3 },
|
||||
include: ['test/**/*.test.ts'],
|
||||
exclude: [
|
||||
'test/integration/dangling-owner-edges.test.ts',
|
||||
'test/integration/class-expression-owners.test.ts',
|
||||
'test/integration/rename-provider-*.test.ts',
|
||||
'test/integration/pdg-explicit-owner.test.ts',
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue