GitNexus/gitnexus/test/unit/scope-resolution/python/python-fixtures.test.ts
vijaygali aa7c273093
fix(ingestion): index Python repos with empty __init__.py and >32 KB files (#1163)
* fix(ingestion): index Python repos with empty __init__.py and >32 KB files

Two defensive fixes that let `gitnexus analyze` complete on Python
codebases that previously failed.

scope-extractor: synthesize an empty Module scope when the provider
emits zero captures. Previously threw "no Module scope found", which
fired for any 0-byte `__init__.py` package marker if the bridge's
empty-source guard was bypassed.

python/captures: wrap the parser.parse() and getPythonScopeQuery()
.matches() calls in try/catch. node-tree-sitter throws "Invalid
argument" for sources that overrun internal buffers (observed at the
~32 KB threshold on Windows). Degrade gracefully with a clear
"skipping scope extraction for this file" warning instead of the
opaque "Invalid argument" surfacing through the bridge.

Verified by indexing whittlem/pycryptobot (which has 7 empty
__init__.py and 11 Python files between 34 KB and 158 KB):
2,367 nodes / 4,973 edges, no segfault, queries resolve symbols
inside the 158 KB controllers/PyCryptoBot.py.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fix(ingestion): harden Python scope extraction fallbacks

Keep failed Python scope extraction on the bridge skip path and build synthetic module scopes before extractor indexes are derived.

Made-with: Cursor

---------

Co-authored-by: Vijay Gali <vgali@vexcelco.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: Gergo Magyar <gergomagyar@icloud.com>
2026-04-30 19:24:04 +01:00

406 lines
16 KiB
TypeScript

/**
* End-to-end fixture tests for the Python scope-resolution migration
* (RFC #909 Ring 3, RFC §5.1 — first-rollout language).
*
* Each fixture:
* 1. Drives `extractPythonScopeCaptures` on a real Python source string.
* 2. Threads the captures through the central `ScopeExtractor` (via
* `extractParsedFile`) — exactly the path `parse-worker.ts`
* executes at ingest time.
* 3. Asserts on the resulting `ParsedFile` (scopes / declarations /
* imports / type bindings / reference sites).
*
* Coverage matrix (≥30 cases, per Ring 3 deliverables):
*
* * Module / function / class scope construction
* * No-block-scope semantics (if / for / while / with / try)
* * Class- and function-local declarations + variables
* * Imports: plain, aliased, multi-target, from, from-as, multi-from,
* wildcard, dotted-relative
* * Function-local imports
* * Receiver type binding: `self` for instance methods, `cls` for
* classmethods; no binding for `@staticmethod`; no binding for free
* functions
* * Parameter type annotations (typed_parameter / typed_default_parameter
* / forward-ref strings)
* * Call references: free vs member, with explicit-receiver capture
* * `global` / `nonlocal` no-op behaviour (documented gap)
*/
import { describe, it, expect } from 'vitest';
import type { ParsedFile } from 'gitnexus-shared';
import { extractParsedFile } from '../../../../src/core/ingestion/scope-extractor-bridge.js';
import { pythonProvider } from '../../../../src/core/ingestion/languages/python.js';
// ─── Test helper ───────────────────────────────────────────────────────────
function parse(src: string, filePath = 'test.py'): ParsedFile {
const result = extractParsedFile(pythonProvider, src, filePath);
if (result === undefined) {
throw new Error(
`extractParsedFile returned undefined for:\n${src}\n— check warnings or capture shape`,
);
}
return result;
}
function scopesByKind(file: ParsedFile, kind: string) {
return file.scopes.filter((s) => s.kind === kind);
}
function findDef(file: ParsedFile, name: string) {
return file.localDefs.find((d) => d.qualifiedName === name);
}
// ─── Pass 1: scope tree ────────────────────────────────────────────────────
describe('Python scopes — module / class / function', () => {
it('case 01: minimal module produces a single Module scope', () => {
const f = parse('pass\n');
expect(f.scopes).toHaveLength(1);
expect(f.scopes[0]!.kind).toBe('Module');
});
it('case 01a: empty and whitespace-only files are skipped without warnings', () => {
for (const src of ['', ' \n\t']) {
const warnings: string[] = [];
const parsed = extractParsedFile(pythonProvider, src, 'pkg/__init__.py', (msg) => {
warnings.push(msg);
});
expect(parsed).toBeUndefined();
expect(warnings).toEqual([]);
}
});
it('case 01b: large cache-miss files use the adaptive tree-sitter buffer', () => {
const padding = 'x'.repeat(600 * 1024);
const f = parse(`# ${padding}\ndef after_padding():\n return 1\n`);
expect(scopesByKind(f, 'Module')).toHaveLength(1);
expect(findDef(f, 'after_padding')?.type).toBe('Function');
});
it('case 01c: UTF-8-heavy cache-miss files use byte-sized parser buffers', () => {
const padding = '漢'.repeat(190_000);
const f = parse(`# ${padding}\ndef after_padding():\n return 1\n`);
expect(scopesByKind(f, 'Module')).toHaveLength(1);
expect(findDef(f, 'after_padding')?.type).toBe('Function');
});
it('case 01d: scope query failures are skipped through the bridge with context', () => {
const warnings: string[] = [];
const parsed = extractParsedFile(
pythonProvider,
'def broken():\n return 1\n',
'broken.py',
(msg) => warnings.push(msg),
{ rootNode: undefined },
);
expect(parsed).toBeUndefined();
expect(warnings.join('\n')).toMatch(/tree-sitter scope query failed for broken\.py/);
});
it('case 02: module-level assignment produces a Variable declaration in Module scope', () => {
const f = parse('x = 1\n');
expect(scopesByKind(f, 'Module')).toHaveLength(1);
expect(findDef(f, 'x')?.type).toBe('Variable');
});
it('case 03: top-level def produces a Function scope under Module', () => {
const f = parse('def foo():\n pass\n');
const fn = scopesByKind(f, 'Function')[0]!;
const mod = scopesByKind(f, 'Module')[0]!;
expect(fn.parent).toBe(mod.id);
expect(findDef(f, 'foo')?.type).toBe('Function');
});
it('case 04: top-level class produces a Class scope under Module', () => {
const f = parse('class A:\n pass\n');
const cls = scopesByKind(f, 'Class')[0]!;
const mod = scopesByKind(f, 'Module')[0]!;
expect(cls.parent).toBe(mod.id);
expect(findDef(f, 'A')?.type).toBe('Class');
});
it('case 05: method nests Function under Class under Module', () => {
const f = parse('class A:\n def m(self):\n pass\n');
const mod = scopesByKind(f, 'Module')[0]!;
const cls = scopesByKind(f, 'Class')[0]!;
const fn = scopesByKind(f, 'Function')[0]!;
expect(cls.parent).toBe(mod.id);
expect(fn.parent).toBe(cls.id);
expect(findDef(f, 'm')?.type).toBe('Method');
});
it('case 06: nested function remains Function when declared inside a method body', () => {
const f = parse('class A:\n def m(self):\n def inner():\n pass\n');
const fns = scopesByKind(f, 'Function');
expect(fns).toHaveLength(2);
const outer = fns.find((s) => s.range.startLine === 2)!;
const inner = fns.find((s) => s.range.startLine === 3)!;
expect(inner.parent).toBe(outer.id);
expect(findDef(f, 'm')?.type).toBe('Method');
expect(findDef(f, 'inner')?.type).toBe('Function');
});
});
// ─── Pass 1: no block scope ────────────────────────────────────────────────
describe('Python scopes — no block scope (PEP language reference)', () => {
it('case 07: `if` body does NOT create a scope; declarations land in enclosing fn', () => {
const f = parse('def f():\n if True:\n x = 1\n');
expect(scopesByKind(f, 'Block')).toHaveLength(0);
const fn = scopesByKind(f, 'Function')[0]!;
expect(fn.bindings.has('x')).toBe(true);
});
it('case 08: `for` target binds in enclosing function scope, not in for body', () => {
const f = parse('def f():\n for i in range(10):\n pass\n');
expect(scopesByKind(f, 'Block')).toHaveLength(0);
const fn = scopesByKind(f, 'Function')[0]!;
expect(fn.bindings.has('i')).toBe(true);
});
it('case 09: `while`/`try`/`with` bodies do not produce Block scopes', () => {
const f = parse(
`def f():
while True:
a = 1
try:
b = 2
except Exception:
c = 3
with open('x') as fh:
d = 4
`,
);
expect(scopesByKind(f, 'Block')).toHaveLength(0);
const fn = scopesByKind(f, 'Function')[0]!;
for (const name of ['a', 'b', 'c', 'd']) expect(fn.bindings.has(name)).toBe(true);
});
});
// ─── Pass 3: imports ──────────────────────────────────────────────────────
describe('Python imports — interpretImport', () => {
it('case 10: `import numpy` → namespace import', () => {
const f = parse('import numpy\n');
expect(f.parsedImports).toEqual([
{ kind: 'namespace', localName: 'numpy', importedName: 'numpy', targetRaw: 'numpy' },
]);
});
it('case 11: `import numpy as np` → namespace import with rename', () => {
const f = parse('import numpy as np\n');
expect(f.parsedImports).toEqual([
{ kind: 'namespace', localName: 'np', importedName: 'numpy', targetRaw: 'numpy' },
]);
});
it('case 12: `import a.b.c` exposes the leading segment as the local name', () => {
const f = parse('import a.b.c\n');
expect(f.parsedImports).toEqual([
{ kind: 'namespace', localName: 'a', importedName: 'a.b.c', targetRaw: 'a.b.c' },
]);
});
it('case 13: `import a, b as c` decomposes into one ParsedImport per name', () => {
const f = parse('import a, b as c\n');
expect(f.parsedImports).toEqual([
{ kind: 'namespace', localName: 'a', importedName: 'a', targetRaw: 'a' },
{ kind: 'namespace', localName: 'c', importedName: 'b', targetRaw: 'b' },
]);
});
it('case 14: `from m import x` → named import', () => {
const f = parse('from m import x\n');
expect(f.parsedImports).toEqual([
{ kind: 'named', localName: 'x', importedName: 'x', targetRaw: 'm' },
]);
});
it('case 15: `from m import x as y` → alias import', () => {
const f = parse('from m import x as y\n');
expect(f.parsedImports).toEqual([
{ kind: 'alias', localName: 'y', importedName: 'x', alias: 'y', targetRaw: 'm' },
]);
});
it('case 16: `from m import x, y, z` decomposes into three ParsedImports', () => {
const f = parse('from m import x, y, z\n');
expect(f.parsedImports).toEqual([
{ kind: 'named', localName: 'x', importedName: 'x', targetRaw: 'm' },
{ kind: 'named', localName: 'y', importedName: 'y', targetRaw: 'm' },
{ kind: 'named', localName: 'z', importedName: 'z', targetRaw: 'm' },
]);
});
it('case 17: `from m import *` → wildcard', () => {
const f = parse('from m import *\n');
expect(f.parsedImports).toEqual([{ kind: 'wildcard', targetRaw: 'm' }]);
});
it('case 18: PEP-328 dotted relative import `from .pkg import x`', () => {
const f = parse('from .pkg import x\n');
expect(f.parsedImports).toEqual([
{ kind: 'named', localName: 'x', importedName: 'x', targetRaw: '.pkg' },
]);
});
it('case 19: PEP-328 parent-relative import `from ..pkg.sub import x`', () => {
const f = parse('from ..pkg.sub import x\n');
expect(f.parsedImports).toEqual([
{ kind: 'named', localName: 'x', importedName: 'x', targetRaw: '..pkg.sub' },
]);
});
});
// ─── Imports inside functions ─────────────────────────────────────────────
describe('Python imports — function-local', () => {
it('case 20: function-local `from x import Y` is captured (visible to importOwningScope)', () => {
const f = parse('def loader():\n from m import X\n');
// Decomposed at parse time; finalize will route via importOwningScope.
expect(f.parsedImports).toEqual([
{ kind: 'named', localName: 'X', importedName: 'X', targetRaw: 'm' },
]);
});
});
// ─── Pass 4: type bindings ────────────────────────────────────────────────
describe('Python type bindings — parameter annotations + self/cls', () => {
it('case 21: typed parameter `def f(x: User)` binds x → User on function scope', () => {
const f = parse('def f(x: User):\n pass\n');
const fn = scopesByKind(f, 'Function')[0]!;
const tb = fn.typeBindings.get('x');
expect(tb).toBeDefined();
expect(tb!.rawName).toBe('User');
expect(tb!.source).toBe('parameter-annotation');
});
it('case 22: typed default parameter `def f(x: int = 0)` is captured', () => {
const f = parse('def f(x: int = 0):\n pass\n');
const fn = scopesByKind(f, 'Function')[0]!;
expect(fn.typeBindings.get('x')?.rawName).toBe('int');
});
it('case 23: forward-ref string `def f(x: "User")` is unquoted', () => {
const f = parse('def f(x: "User"):\n pass\n');
const fn = scopesByKind(f, 'Function')[0]!;
expect(fn.typeBindings.get('x')?.rawName).toBe('User');
});
it('case 24: instance method gets self → ClassName as `self` source', () => {
const f = parse('class A:\n def m(self):\n pass\n');
const fn = scopesByKind(f, 'Function')[0]!;
const self = fn.typeBindings.get('self');
expect(self).toBeDefined();
expect(self!.rawName).toBe('A');
expect(self!.source).toBe('self');
});
it('case 25: `@classmethod`-decorated method gets cls → ClassName', () => {
const f = parse(
`class A:
@classmethod
def make(cls):
pass
`,
);
const fn = scopesByKind(f, 'Function')[0]!;
expect(fn.typeBindings.get('cls')?.rawName).toBe('A');
expect(fn.typeBindings.has('self')).toBe(false);
});
it('case 26: `@staticmethod`-decorated method gets NO implicit receiver', () => {
const f = parse(
`class A:
@staticmethod
def util(x):
pass
`,
);
const fn = scopesByKind(f, 'Function')[0]!;
expect(fn.typeBindings.has('self')).toBe(false);
expect(fn.typeBindings.has('cls')).toBe(false);
});
it('case 27: free function gets NO `self`/`cls` binding', () => {
const f = parse('def free(x):\n pass\n');
const fn = scopesByKind(f, 'Function')[0]!;
expect(fn.typeBindings.has('self')).toBe(false);
expect(fn.typeBindings.has('cls')).toBe(false);
});
it('case 28: nested function inside method does NOT inherit `self`', () => {
const f = parse(
`class A:
def m(self):
def inner():
pass
`,
);
const inner = scopesByKind(f, 'Function').find((s) => s.range.startLine === 3)!;
expect(inner.typeBindings.has('self')).toBe(false);
});
});
// ─── Pass 5: reference sites ──────────────────────────────────────────────
describe('Python reference sites — calls', () => {
it('case 29: free call `print(x)` records a call reference', () => {
const f = parse('def f():\n print(1)\n');
const calls = f.referenceSites.filter((r) => r.kind === 'call');
expect(calls.some((c) => c.name === 'print' && c.callForm === 'free')).toBe(true);
});
it('case 30: member call `obj.save()` records explicit receiver `obj`', () => {
const f = parse('def f(obj):\n obj.save()\n');
const member = f.referenceSites.find((r) => r.kind === 'call' && r.name === 'save')!;
expect(member.callForm).toBe('member');
expect(member.explicitReceiver).toEqual({ name: 'obj' });
});
it('case 31: chained member call `a.b.c()` captures `c` with receiver `a.b`', () => {
const f = parse('def f(a):\n a.b.c()\n');
const member = f.referenceSites.find((r) => r.kind === 'call' && r.name === 'c')!;
expect(member.callForm).toBe('member');
expect(member.explicitReceiver?.name).toBe('a.b');
});
});
// ─── global / nonlocal — documented under-reporting ───────────────────────
describe('Python `global`/`nonlocal` — documented behavior', () => {
it('case 32: `global x` inside a function does NOT promote the binding to module scope', () => {
// Documented limitation: the assignment lexically lives in `f`, so
// we attach `x` to f's scope. A future Ring may re-bind via
// bindingScopeFor; for Ring 3 this is expected behavior.
const f = parse(
`x = 0
def f():
global x
x = 1
`,
);
const fn = scopesByKind(f, 'Function')[0]!;
const mod = scopesByKind(f, 'Module')[0]!;
expect(mod.bindings.has('x')).toBe(true); // module-level x = 0
expect(fn.bindings.has('x')).toBe(true); // local x = 1 — under-reported as fn-local
});
it('case 33: `nonlocal x` inside a closure does NOT lift binding to enclosing fn', () => {
const f = parse(
`def outer():
x = 0
def inner():
nonlocal x
x = 1
`,
);
const inner = scopesByKind(f, 'Function').find((s) => s.range.startLine === 3)!;
expect(inner.bindings.has('x')).toBe(true); // under-reported
});
});