mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-04 02:31:36 +00:00
fix(csharp): eliminate O(S·D) BindingRef OOM in namespace siblings
Types declared in the C# global (default) namespace are visible from every file, so the previous per-scope augmentation materialized O(scopes × defs) BindingRefs — on large Unity solutions (tens of thousands of global types) this caused severe slowness and OOM. Route global-namespace types through a single workspace-level binding channel (workspaceFqnBindings, consulted by lookupBindingsAt) for O(D) memory. Also fix quadratic costs in the non-global path: append defs in place instead of copying (was O(D²) per bucket), pre-index the first scope per file (was O(S²·D)), and seed de-dup sets instead of repeated .some scans. Add csharp-pipeline-benchmark.test.ts (mirrors the PHP benchmark) with spread and concentrated-global-namespace scenarios to track elapsedMs, peakHeapMB, nodeCount, and edgeCount. Post-fix runs show linear scaling and stable heap. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
2f15c1ece1
commit
9bda164ca7
2 changed files with 323 additions and 9 deletions
|
|
@ -350,34 +350,98 @@ export function populateCsharpNamespaceSiblings(
|
|||
}
|
||||
}
|
||||
|
||||
for (const [, bucket] of buckets) {
|
||||
// De-dup by (nodeId, filePath) across multiple declarations (e.g.
|
||||
// partial classes declaring the same name in two files — we take
|
||||
// both and leave de-dup to downstream consumers of bindings).
|
||||
// Workspace-level binding channel for global-namespace types (see the
|
||||
// global fast-path below). `lookupBindingsAt` consults this as a third
|
||||
// source after finalized + per-scope augmented bindings.
|
||||
const fqnMap = indexes.workspaceFqnBindings as Map<string, BindingRef[]>;
|
||||
|
||||
for (const [nsName, bucket] of buckets) {
|
||||
// Group sibling defs by simple name. Append in place — the previous
|
||||
// `[...prev, def]` copy made this O(D²) per bucket, which on the
|
||||
// global (`''`) namespace bucket of a large Unity solution (tens of
|
||||
// thousands of type defs) was a primary slowness/OOM source. We keep
|
||||
// every declaration (e.g. partial classes across files) and leave
|
||||
// de-dup to downstream consumers.
|
||||
const defsByName = new Map<string, SymbolDefinition[]>();
|
||||
for (const def of bucket.classDefs) {
|
||||
// Simple name = last segment of qualifiedName (e.g. `App.User` → `User`).
|
||||
const q = def.qualifiedName ?? '';
|
||||
const key = q.includes('.') ? q.slice(q.lastIndexOf('.') + 1) : q;
|
||||
if (key === '') continue;
|
||||
const arr = [...(defsByName.get(key) ?? [])];
|
||||
let arr = defsByName.get(key);
|
||||
if (arr === undefined) {
|
||||
arr = [];
|
||||
defsByName.set(key, arr);
|
||||
}
|
||||
arr.push(def);
|
||||
defsByName.set(key, arr);
|
||||
}
|
||||
|
||||
// Global-namespace fast path (Unity OOM guard). Types declared in the
|
||||
// default (global) namespace are visible from EVERY file in C# — the
|
||||
// global namespace is always implicitly in scope — so one workspace-
|
||||
// level entry per simple name is both semantically correct and O(D)
|
||||
// instead of the O(S·D) per-scope augmentation that materialized
|
||||
// billions of BindingRefs on large Unity solutions (tens of thousands
|
||||
// of global types × tens of thousands of scopes). `walkScopeChain`
|
||||
// checks local `scope.bindings` first, so local declarations still
|
||||
// shadow these workspace entries; a file resolving its own global type
|
||||
// hits the local binding before this map. Dedup by `def.nodeId` keeps
|
||||
// partial-class / duplicate declarations from double-emitting.
|
||||
if (nsName === '') {
|
||||
for (const [name, defs] of defsByName) {
|
||||
let arr = fqnMap.get(name);
|
||||
let seen: Set<string> | null = null;
|
||||
for (const def of defs) {
|
||||
if (arr === undefined) {
|
||||
arr = [];
|
||||
fqnMap.set(name, arr);
|
||||
}
|
||||
if (seen === null) {
|
||||
seen = new Set<string>();
|
||||
for (const b of arr) seen.add(b.def.nodeId);
|
||||
}
|
||||
if (seen.has(def.nodeId)) continue;
|
||||
seen.add(def.nodeId);
|
||||
arr.push({ def, origin: 'namespace' });
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Pre-index the first scope per file once (O(S)) instead of an
|
||||
// O(S) `.find` re-run for every (scope, name) pair, which made the
|
||||
// injection loop O(S²·D) and was the dominant cost on large buckets.
|
||||
// Multiple scopes share a filePath (Module + Namespace); the local
|
||||
// shadow check only needs that file's lexical `Scope.bindings`, which
|
||||
// is identical regardless of which of those scopes we read.
|
||||
const firstScopeByFile = new Map<string, Scope>();
|
||||
for (const s of bucket.scopes) {
|
||||
if (!firstScopeByFile.has(s.filePath)) firstScopeByFile.set(s.filePath, s.scope);
|
||||
}
|
||||
|
||||
for (const { scopeId, filePath } of bucket.scopes) {
|
||||
const localScope = firstScopeByFile.get(filePath);
|
||||
for (const [name, defs] of defsByName) {
|
||||
// Skip names already present locally — `origin: 'local'` in
|
||||
// scope.bindings would naturally shadow the cross-file
|
||||
// namespace entry, but we also keep this index lean.
|
||||
const local = bucket.scopes.find((s) => s.filePath === filePath)?.scope.bindings.get(name);
|
||||
const local = localScope?.bindings.get(name);
|
||||
if (local !== undefined && local.some((b) => b.origin === 'local')) continue;
|
||||
|
||||
let bucketArr: BindingRef[] | null = null;
|
||||
let seen: Set<string> | null = null;
|
||||
for (const def of defs) {
|
||||
if (def.filePath === filePath) continue; // don't self-reference
|
||||
if (bucketArr === null) bucketArr = getAugmentationBucket(augmentations, scopeId, name);
|
||||
if (bucketArr.some((b) => b.def.nodeId === def.nodeId)) continue;
|
||||
if (bucketArr === null) {
|
||||
bucketArr = getAugmentationBucket(augmentations, scopeId, name);
|
||||
// Seed the de-dup set from any entries an earlier pass
|
||||
// (using-static / cross-namespace imports) already added,
|
||||
// replacing the per-def O(A) `.some` scan.
|
||||
seen = new Set<string>();
|
||||
for (const b of bucketArr) seen.add(b.def.nodeId);
|
||||
}
|
||||
if (seen!.has(def.nodeId)) continue;
|
||||
seen!.add(def.nodeId);
|
||||
bucketArr.push({ def, origin: 'namespace' });
|
||||
}
|
||||
}
|
||||
|
|
|
|||
250
gitnexus/test/integration/csharp-pipeline-benchmark.test.ts
Normal file
250
gitnexus/test/integration/csharp-pipeline-benchmark.test.ts
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
/**
|
||||
* C# ingestion pipeline benchmark.
|
||||
*
|
||||
* Generates synthetic C# codebases at increasing scales and measures
|
||||
* wall-clock time and peak heap through the full pipeline — parsing,
|
||||
* scope extraction, C# namespace-siblings (same-namespace cross-file
|
||||
* visibility, using-static, cross-namespace imports), and call
|
||||
* resolution.
|
||||
*
|
||||
* Mirrors test/integration/php-pipeline-benchmark.test.ts. Two shapes:
|
||||
* 1. "spread" — files distributed across many namespaces (the common
|
||||
* case; each namespace bucket stays small).
|
||||
* 2. "concentrated" — every file in the SAME (or global/no) namespace,
|
||||
* so a single namespace bucket holds all type defs. This is the
|
||||
* shape that drove the Unity-solution OOM: `populateCsharpNamespaceSiblings`
|
||||
* materialises O(scopes × defs) BindingRefs into that one bucket.
|
||||
* The concentrated test is the regression guard for that path.
|
||||
*
|
||||
* Run: GITNEXUS_BENCH=1 npx vitest run test/integration/csharp-pipeline-benchmark.test.ts
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js';
|
||||
|
||||
const BENCH_ENABLED = process.env.GITNEXUS_BENCH === '1';
|
||||
|
||||
interface BenchResult {
|
||||
fileCount: number;
|
||||
classCount: number;
|
||||
namespaceCount: number;
|
||||
elapsedMs: number;
|
||||
peakHeapMB: number;
|
||||
nodeCount: number;
|
||||
edgeCount: number;
|
||||
}
|
||||
|
||||
type FixtureShape = 'spread' | 'concentrated';
|
||||
|
||||
function generateCsharpFixture(
|
||||
fileCount: number,
|
||||
namespacesPerLevel: number,
|
||||
shape: FixtureShape,
|
||||
): { dir: string; classCount: number; namespaceCount: number } {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), `csharp-bench-${shape}-${fileCount}-`));
|
||||
|
||||
// "spread": square grid of namespaces. "concentrated": a single
|
||||
// global (no-namespace) bucket so every type lands in the `''` bucket
|
||||
// — the OOM-prone path.
|
||||
const namespaces: string[] = [];
|
||||
if (shape === 'spread') {
|
||||
for (let i = 0; i < namespacesPerLevel; i++) {
|
||||
for (let j = 0; j < namespacesPerLevel; j++) {
|
||||
namespaces.push(`App.Module${i}.Sub${j}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
namespaces.push(''); // global / no namespace declaration
|
||||
}
|
||||
|
||||
const classCount = fileCount;
|
||||
const namespaceCount = namespaces.length;
|
||||
|
||||
for (let f = 0; f < fileCount; f++) {
|
||||
const ns = namespaces[f % namespaces.length]!;
|
||||
const className = `Class${f}`;
|
||||
// Concentrated files share a flat directory; spread files mirror the
|
||||
// namespace as a directory tree (matches typical C# project layout).
|
||||
const targetDir = ns === '' ? dir : path.join(dir, ns.replace(/\./g, '/'));
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
|
||||
const siblingIdx = (f + 1) % fileCount;
|
||||
const siblingClass = `Class${siblingIdx}`;
|
||||
|
||||
const crossNsIdx = (f + Math.floor(fileCount / 3)) % fileCount;
|
||||
const crossNs = namespaces[crossNsIdx % namespaces.length]!;
|
||||
const crossClass = `Class${crossNsIdx}`;
|
||||
const usesCross = ns !== '' && ns !== crossNs;
|
||||
|
||||
const body = [
|
||||
ns !== '' ? `namespace ${ns};` : '',
|
||||
usesCross ? `using ${crossNs};` : '',
|
||||
'',
|
||||
`public class ${className}`,
|
||||
'{',
|
||||
' private int id;',
|
||||
' private string name;',
|
||||
'',
|
||||
' public int GetId()',
|
||||
' {',
|
||||
' return this.id;',
|
||||
' }',
|
||||
'',
|
||||
` public ${siblingClass} Process()`,
|
||||
' {',
|
||||
` var sibling = new ${siblingClass}();`,
|
||||
' return sibling;',
|
||||
' }',
|
||||
usesCross
|
||||
? [
|
||||
'',
|
||||
` public ${crossClass} CrossCall()`,
|
||||
' {',
|
||||
` var cross = new ${crossClass}();`,
|
||||
' cross.GetId();',
|
||||
' return cross;',
|
||||
' }',
|
||||
].join('\n')
|
||||
: '',
|
||||
'}',
|
||||
'',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
|
||||
fs.writeFileSync(path.join(targetDir, `${className}.cs`), body);
|
||||
}
|
||||
|
||||
// Minimal SDK-style csproj so the C# project-loading phase engages
|
||||
// (matches the real-world Unity/.NET solution path).
|
||||
const csproj = [
|
||||
'<Project Sdk="Microsoft.NET.Sdk">',
|
||||
' <PropertyGroup>',
|
||||
' <TargetFramework>net8.0</TargetFramework>',
|
||||
' <Nullable>enable</Nullable>',
|
||||
' </PropertyGroup>',
|
||||
'</Project>',
|
||||
'',
|
||||
].join('\n');
|
||||
fs.writeFileSync(path.join(dir, 'Bench.csproj'), csproj);
|
||||
|
||||
return { dir, classCount, namespaceCount };
|
||||
}
|
||||
|
||||
async function runBenchmark(
|
||||
fileCount: number,
|
||||
nsLevels: number,
|
||||
shape: FixtureShape,
|
||||
budgetMs: number,
|
||||
): Promise<BenchResult> {
|
||||
const { dir, classCount, namespaceCount } = generateCsharpFixture(fileCount, nsLevels, shape);
|
||||
|
||||
let peakHeapMB = 0;
|
||||
const heapSampler = setInterval(() => {
|
||||
const heap = process.memoryUsage().heapUsed / 1024 / 1024;
|
||||
if (heap > peakHeapMB) peakHeapMB = heap;
|
||||
}, 50);
|
||||
|
||||
try {
|
||||
const start = Date.now();
|
||||
const result = await Promise.race([
|
||||
runPipelineFromRepo(dir, () => {}, { skipGraphPhases: true }),
|
||||
new Promise<never>((_, reject) =>
|
||||
setTimeout(
|
||||
() => reject(new Error(`Pipeline exceeded ${budgetMs}ms at ${fileCount} files (${shape})`)),
|
||||
budgetMs,
|
||||
),
|
||||
),
|
||||
]);
|
||||
const elapsedMs = Date.now() - start;
|
||||
|
||||
return {
|
||||
fileCount,
|
||||
classCount,
|
||||
namespaceCount,
|
||||
elapsedMs,
|
||||
peakHeapMB: Math.round(peakHeapMB),
|
||||
nodeCount: result.graph.nodeCount,
|
||||
edgeCount: result.graph.relationshipCount,
|
||||
};
|
||||
} finally {
|
||||
clearInterval(heapSampler);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function printResults(label: string, results: BenchResult[]) {
|
||||
console.log(`\n${label}`);
|
||||
console.log('┌──────────┬─────────┬──────────┬───────────┬──────────┬───────┬───────┐');
|
||||
console.log('│ Files │ Classes │ NS Count │ Time (ms) │ Heap MB │ Nodes │ Edges │');
|
||||
console.log('├──────────┼─────────┼──────────┼───────────┼──────────┼───────┼───────┤');
|
||||
for (const r of results) {
|
||||
console.log(
|
||||
`│ ${String(r.fileCount).padStart(8)} │ ${String(r.classCount).padStart(7)} │ ${String(r.namespaceCount).padStart(8)} │ ${String(r.elapsedMs).padStart(9)} │ ${String(r.peakHeapMB).padStart(8)} │ ${String(r.nodeCount).padStart(5)} │ ${String(r.edgeCount).padStart(5)} │`,
|
||||
);
|
||||
}
|
||||
console.log('└──────────┴─────────┴──────────┴───────────┴──────────┴───────┴───────┘');
|
||||
|
||||
if (results.length >= 2) {
|
||||
console.log('\nScaling ratios (time_ratio / file_ratio):');
|
||||
for (let i = 1; i < results.length; i++) {
|
||||
const fileRatio = results[i].fileCount / results[i - 1].fileCount;
|
||||
const timeRatio = results[i].elapsedMs / results[i - 1].elapsedMs;
|
||||
const scaling = timeRatio / fileRatio;
|
||||
console.log(
|
||||
` ${results[i - 1].fileCount} → ${results[i].fileCount}: ${scaling.toFixed(2)}x (${scaling < 1.5 ? 'linear' : scaling < 3 ? 'superlinear' : 'WARNING: quadratic'})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(!BENCH_ENABLED)('C# pipeline benchmark', () => {
|
||||
it('scales with file count — namespaces spread across the solution', async () => {
|
||||
const scales = [100, 250, 500];
|
||||
const results: BenchResult[] = [];
|
||||
|
||||
for (const fileCount of scales) {
|
||||
const nsLevels = Math.max(2, Math.ceil(Math.sqrt(fileCount / 4)));
|
||||
const result = await runBenchmark(fileCount, nsLevels, 'spread', 180_000);
|
||||
results.push(result);
|
||||
console.log(
|
||||
` ${fileCount} files: ${result.elapsedMs}ms, ${result.peakHeapMB}MB heap, ${result.nodeCount} nodes, ${result.edgeCount} edges`,
|
||||
);
|
||||
}
|
||||
|
||||
printResults('C# Pipeline — Namespaces Spread', results);
|
||||
|
||||
for (let i = 1; i < results.length; i++) {
|
||||
const fileRatio = results[i].fileCount / results[i - 1].fileCount;
|
||||
const timeRatio = results[i].elapsedMs / results[i - 1].elapsedMs;
|
||||
expect(timeRatio / fileRatio).toBeLessThan(3);
|
||||
}
|
||||
}, 600_000);
|
||||
|
||||
it('scales with file count — all types in one (global) namespace bucket', async () => {
|
||||
// Regression guard for the Unity-solution OOM: a single namespace
|
||||
// bucket holds every type def, so naive per-scope binding
|
||||
// materialisation is O(files²). Time must stay sub-quadratic and the
|
||||
// run must not OOM.
|
||||
const scales = [100, 250, 500];
|
||||
const results: BenchResult[] = [];
|
||||
|
||||
for (const fileCount of scales) {
|
||||
const result = await runBenchmark(fileCount, 1, 'concentrated', 180_000);
|
||||
results.push(result);
|
||||
console.log(
|
||||
` ${fileCount} files: ${result.elapsedMs}ms, ${result.peakHeapMB}MB heap, ${result.nodeCount} nodes, ${result.edgeCount} edges`,
|
||||
);
|
||||
}
|
||||
|
||||
printResults('C# Pipeline — Concentrated Global Namespace', results);
|
||||
|
||||
for (let i = 1; i < results.length; i++) {
|
||||
const fileRatio = results[i].fileCount / results[i - 1].fileCount;
|
||||
const timeRatio = results[i].elapsedMs / results[i - 1].elapsedMs;
|
||||
expect(timeRatio / fileRatio).toBeLessThan(3);
|
||||
}
|
||||
}, 600_000);
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue