mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-08-28 05:25:25 +00:00
* fix(swift): preprocess indented conditional directives so class bodies survive parsing * fix(swift): make conditional-directive blanking comment-, string- and brace-aware (#2771) Addresses the review findings on PR #2771. The transform fired unconditionally, which turned valid Swift into parse errors while missing the most common shape it was written for. - The blank/keep decision now consults `blockCommentDepth`, so ` #endif */` — the result of commenting out a conditional block — keeps its comment terminator. Previously `hasError` went raw=false -> preprocessed=true and the rest of the file was swallowed. - The decision keys on the scanner's brace depth instead of indentation. A column-0 `#if` inside a class body is blanked (6 of 7 body shapes previously still lost the enclosing declaration) and an indented file-scope directive is not — matching what the doc comment already claimed. Bare-CR line endings, NBSP/ideographic indentation and a leading BOM are recognized too. - A group is blanked only when every branch is brace-balanced. An `#if`/`#else` that splits a declaration header leaves one unmatched `{` once both branches survive, which collapsed five top-level nodes into one and gave unrelated types fabricated `NetworkClient.` qualified names. Such a group now degrades to the pre-fix behavior. - Multiline strings honour `\"""` escapes, and a plain `"""` closes even when a `#` follows it, so the scanner no longer wedges in string state and silently stops blanking for the rest of the file. - The pound run is counted once per position and skipped. It was quadratic: 10.6s for one 64k-`#` line, well inside the 512 KB walker limit. - Extended regex literals (`#/.../#`) no longer open a phantom block comment. - Directive-free files return early, matching `stripUeMacros`. Worker parity: `emitSwiftScopeCaptures` and `emitCppScopeCaptures` re-apply their provider's `preprocessSource` on the parse-cache-miss path — Dart already did this — and the embedding parse in `ensureAndParse` applies the hook as well. Before this the worker and the scope-capture/embedding halves analyzed different programs, turning a consistent degradation into cold-run/warm-run non-determinism. A new parity test pins the equivalence for every provider that defines the hook. SCHEMA_BUMP 37 -> 38: this changes parse semantics, the chunk key hashes raw on-disk bytes, and `preprocessSource` runs after the key is computed — so a same-package-version warm cache would replay pre-fix Swift results verbatim, including across `--force`. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * refactor(ingestion): apply preprocessSource once in the scope bridge (#2771) Follow-up cleanup on the review fixes. The previous commit re-applied each provider's `preprocessSource` inside `emitSwiftScopeCaptures` and `emitCppScopeCaptures`, mirroring what Dart already did — three copies of the same rule, and a contract that asked every future emitter to remember it. `extractParsedFile` is the single funnel every `emitScopeCaptures` caller passes through (parse worker, scope-resolution run, Vue script extraction), and it already receives the provider. Applying the hook there on the cache-miss path covers all three languages and every future one, names no language in shared code, and drops Dart's unconditional transform on the cache-hit path. Verified the three emitters use `sourceText` for nothing but the parse, so the substitution is output-identical — which the parity test asserts directly. Also from the cleanup pass: - the parity test derives its language list from the provider registry, so a new provider adopting the hook fails until it adds a fixture - `ensureAndParse` resolves the provider from the language it already computed, instead of a second extension table (`getProviderForFile`) - the preprocessor returns `sourceText` unchanged when no group was blanked, which is the common case for files whose only directives are top-level - `split(/(\r\n|\n|\r)/)` replaces the hand-rolled line splitter, and the per-group brace bookkeeping is two scalars instead of an array - the hint regex is derived from the line regex so the two cannot drift - unit assertions compare the WHOLE preprocessed file against the expected blanking, replacing per-line spot checks; the pipeline tests share one `runFixture` helper and `getNodesForFile` in the resolver test helpers - `LanguageProvider.preprocessSource` documents the real call sites and says plainly that the set is not closed — `populateRangeBindings` still hands language helpers raw text Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * chore(autofix): apply prettier + eslint fixes via /autofix command --------- Co-authored-by: Gergő Magyar <gergomagyar@icloud.com> Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
932 lines
38 KiB
TypeScript
932 lines
38 KiB
TypeScript
import { describe, it, expect, beforeAll } from 'vitest';
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
import {
|
|
loadParser,
|
|
loadLanguage,
|
|
isLanguageAvailable,
|
|
} from '../../src/core/tree-sitter/parser-loader.js';
|
|
import { SupportedLanguages, getLanguageFromFilename } from 'gitnexus-shared';
|
|
import { getProvider } from '../../src/core/ingestion/languages/index.js';
|
|
import Parser from 'tree-sitter';
|
|
|
|
const fixturesDir = path.resolve(__dirname, '..', 'fixtures', 'sample-code');
|
|
|
|
function readFixture(filename: string): string {
|
|
return fs.readFileSync(path.join(fixturesDir, filename), 'utf-8');
|
|
}
|
|
|
|
function parseAndQuery(parser: Parser, content: string, queryStr: string) {
|
|
const tree = parser.parse(content);
|
|
const lang = parser.getLanguage();
|
|
const query = new Parser.Query(lang, queryStr);
|
|
const matches = query.matches(tree.rootNode);
|
|
return { tree, matches };
|
|
}
|
|
|
|
function extractDefinitions(matches: any[]) {
|
|
const defs: { type: string; name: string }[] = [];
|
|
for (const match of matches) {
|
|
for (const capture of match.captures) {
|
|
if (
|
|
capture.name === 'name' &&
|
|
match.captures.some((c: any) => c.name.startsWith('definition.'))
|
|
) {
|
|
const defType = match.captures.find((c: any) => c.name.startsWith('definition.'))!.name;
|
|
defs.push({ type: defType, name: capture.node.text });
|
|
}
|
|
}
|
|
}
|
|
return defs;
|
|
}
|
|
|
|
function extractCapturedCallNames(matches: any[]) {
|
|
const names: string[] = [];
|
|
for (const match of matches) {
|
|
if (!match.captures.some((c: any) => c.name === 'call')) continue;
|
|
const nameCapture = match.captures.find((c: any) => c.name === 'call.name');
|
|
if (nameCapture) names.push(nameCapture.node.text);
|
|
}
|
|
return names;
|
|
}
|
|
|
|
describe('Tree-sitter multi-language parsing', () => {
|
|
let parser: Parser;
|
|
|
|
beforeAll(async () => {
|
|
parser = await loadParser();
|
|
});
|
|
|
|
describe('TypeScript', () => {
|
|
it('parses functions, classes, interfaces, methods, and arrow functions', async () => {
|
|
await loadLanguage(SupportedLanguages.TypeScript, 'simple.ts');
|
|
const content = readFixture('simple.ts');
|
|
const provider = getProvider(SupportedLanguages.TypeScript);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.class');
|
|
expect(defTypes).toContain('definition.function');
|
|
});
|
|
});
|
|
|
|
describe('TSX', () => {
|
|
it('parses JSX components with tsx grammar', async () => {
|
|
await loadLanguage(SupportedLanguages.TypeScript, 'simple.tsx');
|
|
const content = readFixture('simple.tsx');
|
|
const provider = getProvider(SupportedLanguages.TypeScript);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(defs.length).toBeGreaterThan(0);
|
|
// Should detect Counter class and Button/useCounter functions
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('Counter');
|
|
});
|
|
});
|
|
|
|
describe('JavaScript', () => {
|
|
it('parses class and function declarations', async () => {
|
|
await loadLanguage(SupportedLanguages.JavaScript);
|
|
const content = readFixture('simple.js');
|
|
const provider = getProvider(SupportedLanguages.JavaScript);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(defs.length).toBeGreaterThan(0);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('EventEmitter');
|
|
expect(names).toContain('createLogger');
|
|
});
|
|
});
|
|
|
|
describe('Python', () => {
|
|
it('parses class and function definitions', async () => {
|
|
await loadLanguage(SupportedLanguages.Python);
|
|
const content = readFixture('simple.py');
|
|
const provider = getProvider(SupportedLanguages.Python);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.class');
|
|
expect(defTypes).toContain('definition.function');
|
|
});
|
|
});
|
|
|
|
describe('Java', () => {
|
|
it('parses class, method, and constructor declarations', async () => {
|
|
await loadLanguage(SupportedLanguages.Java);
|
|
const content = readFixture('simple.java');
|
|
const provider = getProvider(SupportedLanguages.Java);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(defs.length).toBeGreaterThan(0);
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.class');
|
|
expect(defTypes).toContain('definition.method');
|
|
});
|
|
});
|
|
|
|
describe('Go', () => {
|
|
it('parses function and type declarations', async () => {
|
|
await loadLanguage(SupportedLanguages.Go);
|
|
const content = readFixture('simple.go');
|
|
const provider = getProvider(SupportedLanguages.Go);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(defs.length).toBeGreaterThan(0);
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.function');
|
|
});
|
|
|
|
it('captures every name in multi-name const, var, and field declarations', async () => {
|
|
await loadLanguage(SupportedLanguages.Go);
|
|
const provider = getProvider(SupportedLanguages.Go);
|
|
const code = `
|
|
package main
|
|
const X, Y,Z,A,B = 1, 2,3,4,5
|
|
var a, b int
|
|
var (
|
|
c, d string
|
|
)
|
|
const (
|
|
C, D = 3, 4
|
|
)
|
|
type Point struct { X, y int }
|
|
`;
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
const constNames = defs
|
|
.filter((def) => def.type === 'definition.const')
|
|
.map((def) => def.name);
|
|
expect(constNames.sort()).toEqual(['A', 'B', 'C', 'D', 'X', 'Y', 'Z']);
|
|
|
|
const variableNames = defs
|
|
.filter((def) => def.type === 'definition.variable')
|
|
.map((def) => def.name);
|
|
expect(variableNames.sort()).toEqual(['a', 'b', 'c', 'd']);
|
|
|
|
const propertyNames = defs
|
|
.filter((def) => def.type === 'definition.property')
|
|
.map((def) => def.name);
|
|
expect(propertyNames.sort()).toEqual(['X', 'y']);
|
|
});
|
|
});
|
|
|
|
describe('C', () => {
|
|
it('parses function definitions and structs', async () => {
|
|
await loadLanguage(SupportedLanguages.C);
|
|
const content = readFixture('simple.c');
|
|
const provider = getProvider(SupportedLanguages.C);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(defs.length).toBeGreaterThan(0);
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.function');
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('add');
|
|
expect(names).toContain('internal_helper');
|
|
expect(names).toContain('print_message');
|
|
});
|
|
|
|
it('captures pointer-returning function definitions', async () => {
|
|
await loadLanguage(SupportedLanguages.C);
|
|
const code = `int* get_ptr() { return 0; }\nchar** get_strs() { return 0; }`;
|
|
const provider = getProvider(SupportedLanguages.C);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('get_ptr');
|
|
expect(names).toContain('get_strs');
|
|
});
|
|
|
|
it('captures macros and typedefs', async () => {
|
|
await loadLanguage(SupportedLanguages.C);
|
|
const code = `#define MAX_SIZE 100\ntypedef unsigned int uint;\nstruct Point { int x; int y; };`;
|
|
const provider = getProvider(SupportedLanguages.C);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('MAX_SIZE');
|
|
expect(names).toContain('uint');
|
|
expect(names).toContain('Point');
|
|
});
|
|
|
|
it('captures C typedef anonymous structs, enums, and enumerators', async () => {
|
|
await loadLanguage(SupportedLanguages.C);
|
|
const code = `
|
|
typedef struct { int x; int y; } Point;
|
|
typedef enum { RED, GREEN, BLUE } Color;
|
|
`;
|
|
const provider = getProvider(SupportedLanguages.C);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(defs.some((d) => d.type === 'definition.struct' && d.name === 'Point')).toBe(true);
|
|
expect(defs.some((d) => d.type === 'definition.enum' && d.name === 'Color')).toBe(true);
|
|
expect(names).toEqual(expect.arrayContaining(['RED', 'GREEN', 'BLUE']));
|
|
});
|
|
});
|
|
|
|
describe('C++', () => {
|
|
it('parses class, function, and namespace declarations', async () => {
|
|
await loadLanguage(SupportedLanguages.CPlusPlus);
|
|
const content = readFixture('simple.cpp');
|
|
const provider = getProvider(SupportedLanguages.CPlusPlus);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(defs.length).toBeGreaterThan(0);
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.class');
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('UserManager');
|
|
expect(names).toContain('helperFunction');
|
|
});
|
|
|
|
it('captures pointer-returning methods and functions', async () => {
|
|
await loadLanguage(SupportedLanguages.CPlusPlus);
|
|
const code = `int* Factory::create() { return nullptr; }\nchar** getNames() { return 0; }`;
|
|
const provider = getProvider(SupportedLanguages.CPlusPlus);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('create');
|
|
expect(names).toContain('getNames');
|
|
});
|
|
|
|
it('captures reference-returning functions', async () => {
|
|
await loadLanguage(SupportedLanguages.CPlusPlus);
|
|
const code = `int& Container::at(int i) { static int x; return x; }`;
|
|
const provider = getProvider(SupportedLanguages.CPlusPlus);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('at');
|
|
});
|
|
|
|
it('captures destructor definitions', async () => {
|
|
await loadLanguage(SupportedLanguages.CPlusPlus);
|
|
const code = `MyClass::~MyClass() { cleanup(); }`;
|
|
const provider = getProvider(SupportedLanguages.CPlusPlus);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('~MyClass');
|
|
});
|
|
|
|
it('captures template declarations', async () => {
|
|
await loadLanguage(SupportedLanguages.CPlusPlus);
|
|
const code = `template<typename T> class Container { T value; };`;
|
|
const provider = getProvider(SupportedLanguages.CPlusPlus);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('Container');
|
|
});
|
|
|
|
it('captures namespace definitions', async () => {
|
|
await loadLanguage(SupportedLanguages.CPlusPlus);
|
|
const code = `namespace utils { void helper() {} }`;
|
|
const provider = getProvider(SupportedLanguages.CPlusPlus);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('utils');
|
|
expect(names).toContain('helper');
|
|
});
|
|
|
|
it('treats CUDA .cu and .cuh files as C++ for definition extraction', async () => {
|
|
expect(getLanguageFromFilename('src/kernels/force.cu')).toBe(SupportedLanguages.CPlusPlus);
|
|
expect(getLanguageFromFilename('src/force/nep.cuh')).toBe(SupportedLanguages.CPlusPlus);
|
|
|
|
await loadLanguage(SupportedLanguages.CPlusPlus, 'src/kernels/force.cu');
|
|
const code = `class Force { public: void apply(); };\nvoid launchKernel() {}`;
|
|
const provider = getProvider(SupportedLanguages.CPlusPlus);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
|
|
expect(defs.some((d) => d.type === 'definition.class' && d.name === 'Force')).toBe(true);
|
|
expect(names).toContain('launchKernel');
|
|
});
|
|
|
|
it('characterizes CUDA syntax when routed through the C++ parser', async () => {
|
|
await loadLanguage(SupportedLanguages.CPlusPlus, 'src/kernels/force.cu');
|
|
const code = `
|
|
__global__ void axpy(float *x) { x[0] = 1.0f; }
|
|
void host() {
|
|
axpy<<<1, 32>>>(nullptr);
|
|
cudaDeviceSynchronize();
|
|
}
|
|
`;
|
|
const provider = getProvider(SupportedLanguages.CPlusPlus);
|
|
const { tree, matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const callNames = extractCapturedCallNames(matches);
|
|
const ordinaryCalls = extractCapturedCallNames(
|
|
parseAndQuery(
|
|
parser,
|
|
'void host() { cudaDeviceSynchronize(); }',
|
|
provider.treeSitterQueries,
|
|
).matches,
|
|
);
|
|
|
|
expect(tree.rootNode.hasError).toBe(true);
|
|
expect(defs.some((d) => d.name === 'axpy')).toBe(true);
|
|
expect(ordinaryCalls).toContain('cudaDeviceSynchronize');
|
|
expect(callNames).not.toContain('axpy');
|
|
});
|
|
|
|
it('captures C++ typedef anonymous structs, enums, and enumerators', async () => {
|
|
await loadLanguage(SupportedLanguages.CPlusPlus);
|
|
const code = `
|
|
typedef struct { int x; int y; } Point;
|
|
typedef enum { Red, Green, Blue } Color;
|
|
`;
|
|
const provider = getProvider(SupportedLanguages.CPlusPlus);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(defs.some((d) => d.type === 'definition.struct' && d.name === 'Point')).toBe(true);
|
|
expect(defs.some((d) => d.type === 'definition.enum' && d.name === 'Color')).toBe(true);
|
|
expect(names).toEqual(expect.arrayContaining(['Red', 'Green', 'Blue']));
|
|
});
|
|
});
|
|
|
|
describe('C#', () => {
|
|
it('parses class, method, and namespace declarations', async () => {
|
|
await loadLanguage(SupportedLanguages.CSharp);
|
|
const content = readFixture('simple.cs');
|
|
const provider = getProvider(SupportedLanguages.CSharp);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(defs.length).toBeGreaterThan(0);
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.class');
|
|
expect(defTypes).toContain('definition.method');
|
|
expect(defTypes).toContain('definition.namespace');
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('Calculator');
|
|
expect(names).toContain('Add');
|
|
});
|
|
|
|
it('captures interfaces, enums, records, structs', async () => {
|
|
await loadLanguage(SupportedLanguages.CSharp);
|
|
const content = readFixture('simple.cs');
|
|
const provider = getProvider(SupportedLanguages.CSharp);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('ICalculator');
|
|
expect(names).toContain('Operation');
|
|
expect(names).toContain('CalculationResult');
|
|
expect(names).toContain('Point');
|
|
});
|
|
|
|
it('captures file-scoped namespace declarations', async () => {
|
|
await loadLanguage(SupportedLanguages.CSharp);
|
|
const code = `namespace MyApp;\npublic class Program { }`;
|
|
const provider = getProvider(SupportedLanguages.CSharp);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('MyApp');
|
|
expect(names).toContain('Program');
|
|
});
|
|
|
|
it('captures constructors and properties', async () => {
|
|
await loadLanguage(SupportedLanguages.CSharp);
|
|
const content = readFixture('simple.cs');
|
|
const provider = getProvider(SupportedLanguages.CSharp);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.constructor');
|
|
expect(defTypes).toContain('definition.property');
|
|
});
|
|
});
|
|
|
|
describe('Rust', () => {
|
|
it('parses fn, struct, impl, trait, and enum', async () => {
|
|
await loadLanguage(SupportedLanguages.Rust);
|
|
const content = readFixture('simple.rs');
|
|
const provider = getProvider(SupportedLanguages.Rust);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(defs.length).toBeGreaterThan(0);
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.function');
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('public_function');
|
|
expect(names).toContain('private_function');
|
|
expect(names).toContain('Config');
|
|
});
|
|
|
|
it('captures impl blocks and methods', async () => {
|
|
await loadLanguage(SupportedLanguages.Rust);
|
|
const content = readFixture('simple.rs');
|
|
const provider = getProvider(SupportedLanguages.Rust);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.impl');
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('new');
|
|
});
|
|
|
|
it('captures generic impl blocks', async () => {
|
|
await loadLanguage(SupportedLanguages.Rust);
|
|
const code = `struct Vec<T> { data: Vec<T> }\nimpl<T> Vec<T> { fn len(&self) -> usize { 0 } }`;
|
|
const provider = getProvider(SupportedLanguages.Rust);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('Vec');
|
|
});
|
|
|
|
it('captures modules, consts, and statics', async () => {
|
|
await loadLanguage(SupportedLanguages.Rust);
|
|
const code = `mod utils { pub fn helper() {} }\npub const MAX: usize = 100;\nstatic INSTANCE: i32 = 0;`;
|
|
const provider = getProvider(SupportedLanguages.Rust);
|
|
const { matches } = parseAndQuery(parser, code, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
const names = defs.map((d) => d.name);
|
|
expect(names).toContain('utils');
|
|
expect(names).toContain('MAX');
|
|
expect(names).toContain('INSTANCE');
|
|
});
|
|
});
|
|
|
|
describe('PHP', () => {
|
|
it('parses class, function, and method declarations', async () => {
|
|
await loadLanguage(SupportedLanguages.PHP);
|
|
const content = readFixture('simple.php');
|
|
const provider = getProvider(SupportedLanguages.PHP);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(defs.length).toBeGreaterThan(0);
|
|
const defTypes = defs.map((d) => d.type);
|
|
expect(defTypes).toContain('definition.class');
|
|
});
|
|
});
|
|
|
|
describe('Swift', () => {
|
|
it('parses class, struct, protocol, and function if tree-sitter-swift is available', async () => {
|
|
try {
|
|
await loadLanguage(SupportedLanguages.Swift);
|
|
} catch {
|
|
// tree-sitter-swift not installed — skip
|
|
return;
|
|
}
|
|
|
|
const content = readFixture('simple.swift');
|
|
const provider = getProvider(SupportedLanguages.Swift);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(defs.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
// Grammar-load failures are expected on the platform-sensitive matrix, so
|
|
// skip rather than hard-fail — same guard the sibling tests apply inline.
|
|
describe.skipIf(!isLanguageAvailable(SupportedLanguages.Swift))(
|
|
'conditional-compilation directives',
|
|
() => {
|
|
const provider = getProvider(SupportedLanguages.Swift);
|
|
const preprocess = (content: string): string =>
|
|
provider.preprocessSource?.(content, 'Fixture.swift') ?? content;
|
|
|
|
beforeAll(async () => {});
|
|
|
|
it('captures a class whose body contains indented conditional directives after preprocessing', () => {
|
|
const content = [
|
|
'class Outer {',
|
|
' enum A { case x }',
|
|
' #if os(iOS)',
|
|
' enum B { case y }',
|
|
' #endif',
|
|
'}',
|
|
].join('\n');
|
|
const { tree, matches } = parseAndQuery(
|
|
parser,
|
|
preprocess(content),
|
|
provider.treeSitterQueries,
|
|
);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(tree.rootNode.hasError).toBe(false);
|
|
expect(defs).toEqual([
|
|
{ type: 'definition.class', name: 'Outer' },
|
|
{ type: 'definition.enum', name: 'A' },
|
|
{ type: 'definition.property', name: 'x' },
|
|
{ type: 'definition.enum', name: 'B' },
|
|
{ type: 'definition.property', name: 'y' },
|
|
]);
|
|
});
|
|
|
|
it('captures a class whose body contains column-zero conditional directives', () => {
|
|
const content = [
|
|
'class Outer {',
|
|
' enum A { case x }',
|
|
'#if os(iOS)',
|
|
' enum B { case y }',
|
|
'#endif',
|
|
'}',
|
|
].join('\n');
|
|
const { tree, matches } = parseAndQuery(
|
|
parser,
|
|
preprocess(content),
|
|
provider.treeSitterQueries,
|
|
);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(tree.rootNode.hasError).toBe(false);
|
|
expect(defs).toEqual([
|
|
{ type: 'definition.class', name: 'Outer' },
|
|
{ type: 'definition.enum', name: 'A' },
|
|
{ type: 'definition.property', name: 'x' },
|
|
{ type: 'definition.enum', name: 'B' },
|
|
{ type: 'definition.property', name: 'y' },
|
|
]);
|
|
});
|
|
|
|
it('leaves top-level conditional directives intact while capturing their declarations', () => {
|
|
const content = [
|
|
'#if os(iOS)',
|
|
'struct PlatformValue {',
|
|
' let value: Int = 1',
|
|
'}',
|
|
'#else',
|
|
'struct PlatformValue {',
|
|
' let value: Int = 2',
|
|
'}',
|
|
'#endif',
|
|
].join('\n');
|
|
const parseContent = preprocess(content);
|
|
const { tree, matches } = parseAndQuery(parser, parseContent, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
expect(tree.rootNode.hasError).toBe(false);
|
|
expect(parseContent).toBe(content);
|
|
expect(defs).toEqual([
|
|
{ type: 'definition.struct', name: 'PlatformValue' },
|
|
{ type: 'definition.property', name: 'value' },
|
|
{ type: 'definition.struct', name: 'PlatformValue' },
|
|
{ type: 'definition.property', name: 'value' },
|
|
]);
|
|
});
|
|
|
|
it('keeps source that comments out a conditional block parseable', () => {
|
|
const content = [
|
|
'class Foo {',
|
|
' /* temporarily disabled:',
|
|
' #if DEBUG',
|
|
' func f() {}',
|
|
' #endif */',
|
|
' func g() {}',
|
|
'}',
|
|
].join('\n');
|
|
const parseContent = preprocess(content);
|
|
const { tree, matches } = parseAndQuery(parser, parseContent, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
// Erasing the comment terminator would swallow `g()` and the rest.
|
|
expect(parseContent).toBe(content);
|
|
expect(tree.rootNode.hasError).toBe(false);
|
|
expect(defs).toEqual([
|
|
{ type: 'definition.class', name: 'Foo' },
|
|
{ type: 'definition.function', name: 'g' },
|
|
]);
|
|
});
|
|
|
|
it('does not re-parent later declarations when branches split a declaration header', () => {
|
|
const content = [
|
|
'class NetworkClient {',
|
|
' #if swift(>=5.5)',
|
|
' func fetch() async {',
|
|
' #else',
|
|
' func fetch() {',
|
|
' #endif',
|
|
' perform()',
|
|
' }',
|
|
'}',
|
|
'struct SessionStore {}',
|
|
'enum Unrelated { case a }',
|
|
].join('\n');
|
|
const parseContent = preprocess(content);
|
|
const { tree } = parseAndQuery(parser, parseContent, provider.treeSitterQueries);
|
|
const topLevelTypes = tree.rootNode.namedChildren.map((child) => child.type);
|
|
|
|
// Blanking both markers would leave `NetworkClient` unterminated and
|
|
// collapse every later top-level declaration into it (5 nodes -> 1).
|
|
// `struct`/`enum` both surface as `class_declaration` in this grammar.
|
|
expect(parseContent).toBe(content);
|
|
expect(topLevelTypes).toEqual([
|
|
'class_declaration',
|
|
'directive',
|
|
'function_declaration',
|
|
'class_declaration',
|
|
'class_declaration',
|
|
]);
|
|
});
|
|
|
|
it('keeps a declaration from every branch once the directives are blanked', () => {
|
|
const content = [
|
|
'class Themed {',
|
|
' #if os(iOS)',
|
|
' func accent(alpha: Int) {}',
|
|
' #else',
|
|
' func accent() {}',
|
|
' #endif',
|
|
'}',
|
|
].join('\n');
|
|
const { tree, matches } = parseAndQuery(
|
|
parser,
|
|
preprocess(content),
|
|
provider.treeSitterQueries,
|
|
);
|
|
const defs = extractDefinitions(matches);
|
|
|
|
// Branch selection is not modelled: mutually exclusive declarations
|
|
// both reach the graph. Pinned so changing it shows up as a diff.
|
|
expect(tree.rootNode.hasError).toBe(false);
|
|
expect(defs).toEqual([
|
|
{ type: 'definition.class', name: 'Themed' },
|
|
{ type: 'definition.function', name: 'accent' },
|
|
{ type: 'definition.function', name: 'accent' },
|
|
]);
|
|
});
|
|
},
|
|
);
|
|
|
|
it('gracefully handles missing tree-sitter-swift', async () => {
|
|
// If Swift is NOT available, loadLanguage should throw
|
|
// If it IS available, this test just passes
|
|
try {
|
|
await loadLanguage(SupportedLanguages.Swift);
|
|
} catch (e: any) {
|
|
expect(e.message).toContain('Unsupported language');
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('unhappy path', () => {
|
|
it('returns null/undefined for unsupported file extensions', () => {
|
|
expect(getLanguageFromFilename('archive.xyz')).toBeNull();
|
|
expect(getLanguageFromFilename('data.unknown')).toBeNull();
|
|
});
|
|
|
|
it('handles empty string file path', () => {
|
|
expect(getLanguageFromFilename('')).toBeNull();
|
|
});
|
|
|
|
it('returns null/undefined for binary file extensions', () => {
|
|
expect(getLanguageFromFilename('program.exe')).toBeNull();
|
|
expect(getLanguageFromFilename('library.dll')).toBeNull();
|
|
expect(getLanguageFromFilename('object.so')).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe('Dart', () => {
|
|
const dartQueries = () => getProvider(SupportedLanguages.Dart).treeSitterQueries;
|
|
|
|
function loadDartOrSkip() {
|
|
return loadLanguage(SupportedLanguages.Dart).catch(() => null);
|
|
}
|
|
|
|
// ── Definition extraction ──────────────────────────────────────────
|
|
|
|
it('parses classes, functions, mixins, enums, and type aliases', async () => {
|
|
if (!(await loadDartOrSkip())) return;
|
|
const { matches } = parseAndQuery(parser, readFixture('simple.dart'), dartQueries());
|
|
const defs = extractDefinitions(matches);
|
|
const defTypes = defs.map((d) => d.type);
|
|
const defNames = defs.map((d) => d.name);
|
|
|
|
expect(defTypes).toContain('definition.class');
|
|
expect(defTypes).toContain('definition.function');
|
|
expect(defTypes).toContain('definition.method');
|
|
expect(defTypes).toContain('definition.enum');
|
|
expect(defTypes).toContain('definition.trait'); // mixin
|
|
expect(defTypes).toContain('definition.type'); // typedef
|
|
expect(defTypes).toContain('definition.constructor');
|
|
expect(defTypes).toContain('definition.property'); // getter/setter
|
|
|
|
expect(defNames).toContain('Animal');
|
|
expect(defNames).toContain('Dog');
|
|
expect(defNames).toContain('greet');
|
|
expect(defNames).toContain('Swimming');
|
|
expect(defNames).toContain('Status');
|
|
expect(defNames).toContain('main');
|
|
expect(defNames).toContain('StringExtension');
|
|
});
|
|
|
|
it('captures factory constructor variant name (not class name)', async () => {
|
|
if (!(await loadDartOrSkip())) return;
|
|
const { matches } = parseAndQuery(parser, readFixture('simple.dart'), dartQueries());
|
|
const defs = extractDefinitions(matches);
|
|
const constructors = defs.filter((d) => d.type === 'definition.constructor');
|
|
expect(constructors.length).toBeGreaterThan(0);
|
|
// factory Dog.unknown() should capture 'unknown', not 'Dog'
|
|
const constructorNames = constructors.map((c) => c.name);
|
|
expect(constructorNames).toContain('unknown');
|
|
expect(constructorNames).not.toContain('Dog');
|
|
});
|
|
|
|
it('captures getter and setter as definition.property', async () => {
|
|
if (!(await loadDartOrSkip())) return;
|
|
const { matches } = parseAndQuery(parser, readFixture('simple.dart'), dartQueries());
|
|
const defs = extractDefinitions(matches);
|
|
const props = defs.filter((d) => d.type === 'definition.property');
|
|
expect(props.map((p) => p.name)).toContain('info'); // getter
|
|
expect(props.map((p) => p.name)).toContain('nickname'); // setter
|
|
});
|
|
|
|
it('captures typedef names without duplicates', async () => {
|
|
if (!(await loadDartOrSkip())) return;
|
|
const { matches } = parseAndQuery(parser, readFixture('simple.dart'), dartQueries());
|
|
const defs = extractDefinitions(matches);
|
|
const typedefs = defs.filter((d) => d.type === 'definition.type');
|
|
const typedefNames = typedefs.map((d) => d.name);
|
|
expect(typedefNames).toContain('JsonMap');
|
|
expect(typedefNames).toContain('Callback');
|
|
// Should not capture RHS types as names
|
|
expect(typedefNames).not.toContain('Map');
|
|
expect(typedefNames).not.toContain('Function');
|
|
});
|
|
|
|
// ── Export detection (private underscore convention) ────────────────
|
|
|
|
it('filters private symbols via underscore convention', async () => {
|
|
if (!(await loadDartOrSkip())) return;
|
|
const { matches } = parseAndQuery(parser, readFixture('simple.dart'), dartQueries());
|
|
const defs = extractDefinitions(matches);
|
|
const { dartExportChecker } = await import('../../src/core/ingestion/export-detection.js');
|
|
const publicNames = defs
|
|
.filter((d) => dartExportChecker(null as any, d.name))
|
|
.map((d) => d.name);
|
|
const privateNames = defs
|
|
.filter((d) => !dartExportChecker(null as any, d.name))
|
|
.map((d) => d.name);
|
|
|
|
expect(publicNames).toContain('Animal');
|
|
expect(publicNames).toContain('greet');
|
|
expect(privateNames).toContain('_privateHelper');
|
|
});
|
|
|
|
// ── Import extraction ──────────────────────────────────────────────
|
|
|
|
it('extracts imports', async () => {
|
|
if (!(await loadDartOrSkip())) return;
|
|
const { matches } = parseAndQuery(parser, readFixture('simple.dart'), dartQueries());
|
|
|
|
const imports: string[] = [];
|
|
for (const match of matches) {
|
|
for (const capture of match.captures) {
|
|
if (capture.name === 'import.source') imports.push(capture.node.text);
|
|
}
|
|
}
|
|
expect(imports.length).toBe(3);
|
|
});
|
|
|
|
it('extracts re-exports', async () => {
|
|
if (!(await loadDartOrSkip())) return;
|
|
const { matches } = parseAndQuery(parser, readFixture('dart-advanced.dart'), dartQueries());
|
|
|
|
const imports: string[] = [];
|
|
for (const match of matches) {
|
|
for (const capture of match.captures) {
|
|
if (capture.name === 'import.source') imports.push(capture.node.text);
|
|
}
|
|
}
|
|
// 2 imports + 1 re-export = 3 import.source captures
|
|
expect(imports.length).toBe(3);
|
|
});
|
|
|
|
// ── Call extraction ────────────────────────────────────────────────
|
|
|
|
it('extracts calls in expression statements', async () => {
|
|
if (!(await loadDartOrSkip())) return;
|
|
const { matches } = parseAndQuery(parser, readFixture('dart-advanced.dart'), dartQueries());
|
|
|
|
const callNames: string[] = [];
|
|
for (const match of matches) {
|
|
for (const capture of match.captures) {
|
|
if (capture.name === 'call.name') callNames.push(capture.node.text);
|
|
}
|
|
}
|
|
expect(callNames).toContain('fetchUsers');
|
|
expect(callNames).toContain('processData');
|
|
});
|
|
|
|
it('extracts calls in return statements', async () => {
|
|
if (!(await loadDartOrSkip())) return;
|
|
const { matches } = parseAndQuery(parser, readFixture('dart-advanced.dart'), dartQueries());
|
|
|
|
const callNames: string[] = [];
|
|
for (const match of matches) {
|
|
for (const capture of match.captures) {
|
|
if (capture.name === 'call.name') callNames.push(capture.node.text);
|
|
}
|
|
}
|
|
expect(callNames).toContain('formatOutput');
|
|
});
|
|
|
|
it('extracts calls in variable assignments', async () => {
|
|
if (!(await loadDartOrSkip())) return;
|
|
const { matches } = parseAndQuery(parser, readFixture('dart-advanced.dart'), dartQueries());
|
|
|
|
const callNames: string[] = [];
|
|
for (const match of matches) {
|
|
for (const capture of match.captures) {
|
|
if (capture.name === 'call.name') callNames.push(capture.node.text);
|
|
}
|
|
}
|
|
expect(callNames).toContain('computeScore');
|
|
expect(callNames).toContain('loadUser');
|
|
});
|
|
|
|
// ── Framework detection (path-based) ───────────────────────────────
|
|
|
|
it('detects Flutter framework from file paths', async () => {
|
|
const { detectFrameworkFromPath } =
|
|
await import('../../src/core/ingestion/framework-detection.js');
|
|
|
|
expect(detectFrameworkFromPath('lib/main.dart')?.framework).toBe('flutter');
|
|
expect(detectFrameworkFromPath('lib/app.dart')?.framework).toBe('flutter');
|
|
expect(detectFrameworkFromPath('lib/screens/home.dart')?.framework).toBe('flutter');
|
|
expect(detectFrameworkFromPath('lib/pages/login.dart')?.framework).toBe('flutter');
|
|
expect(detectFrameworkFromPath('lib/widgets/button.dart')?.framework).toBe('flutter');
|
|
expect(detectFrameworkFromPath('lib/bloc/user_bloc.dart')?.framework).toBe('flutter');
|
|
expect(detectFrameworkFromPath('lib/services/api.dart')?.framework).toBe('flutter');
|
|
expect(detectFrameworkFromPath('lib/routes/app_router.dart')?.framework).toBe('flutter');
|
|
|
|
// main.dart gets highest boost
|
|
expect(detectFrameworkFromPath('lib/main.dart')?.entryPointMultiplier).toBe(3.0);
|
|
// widgets get lowest
|
|
expect(detectFrameworkFromPath('lib/widgets/button.dart')?.entryPointMultiplier).toBe(1.5);
|
|
});
|
|
});
|
|
|
|
describe('cross-language assertions', () => {
|
|
it('all supported languages produce at least one definition from fixtures', async () => {
|
|
const langFixtures: [SupportedLanguages, string, string?][] = [
|
|
[SupportedLanguages.TypeScript, 'simple.ts'],
|
|
[SupportedLanguages.JavaScript, 'simple.js'],
|
|
[SupportedLanguages.Python, 'simple.py'],
|
|
[SupportedLanguages.Java, 'simple.java'],
|
|
[SupportedLanguages.Go, 'simple.go'],
|
|
[SupportedLanguages.C, 'simple.c'],
|
|
[SupportedLanguages.CPlusPlus, 'simple.cpp'],
|
|
[SupportedLanguages.CSharp, 'simple.cs'],
|
|
[SupportedLanguages.Rust, 'simple.rs'],
|
|
[SupportedLanguages.PHP, 'simple.php'],
|
|
// Dart and Swift are excluded — they are optionalDependencies that may not be installed
|
|
];
|
|
|
|
for (const [lang, fixture, filePath] of langFixtures) {
|
|
await loadLanguage(lang, filePath || fixture);
|
|
const content = readFixture(fixture);
|
|
const provider = getProvider(lang);
|
|
const { matches } = parseAndQuery(parser, content, provider.treeSitterQueries);
|
|
const defs = extractDefinitions(matches);
|
|
expect(defs.length, `${lang} (${fixture}) should have definitions`).toBeGreaterThan(0);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('parser edge cases', () => {
|
|
it('loadLanguage throws for unsupported language', async () => {
|
|
await expect(loadLanguage('brainfuck' as any)).rejects.toThrow(/unsupported language/i);
|
|
});
|
|
|
|
it('parsing empty file content produces empty matches', async () => {
|
|
await loadLanguage(SupportedLanguages.TypeScript, 'empty.ts');
|
|
const tree = parser.parse('');
|
|
expect(tree.rootNode).toBeDefined();
|
|
|
|
const lang = parser.getLanguage();
|
|
const tsProvider = getProvider(SupportedLanguages.TypeScript);
|
|
const query = new Parser.Query(lang, tsProvider.treeSitterQueries);
|
|
const matches = query.matches(tree.rootNode);
|
|
expect(matches).toEqual([]);
|
|
});
|
|
|
|
it('parsing malformed code does not crash', async () => {
|
|
await loadLanguage(SupportedLanguages.TypeScript, 'malformed.ts');
|
|
const tree = parser.parse('function {{{ class >>><< if(( end');
|
|
expect(tree.rootNode).toBeDefined();
|
|
expect(tree.rootNode.hasError).toBe(true);
|
|
});
|
|
});
|
|
});
|