mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-11-13 08:51:28 +00:00
fix: skip unavailable native Swift parsers in sequential ingestion (#188)
* fix: skip unavailable native Swift parsers in sequential ingestion * fix: warn when ingestion skips languages in verbose mode * test: cover verbose skip warnings * docs: update analyze flags * docs: clarify verbose default --------- Co-authored-by: Gergo Magyar <gergomagyar@icloud.com>
This commit is contained in:
parent
05f7f3c52c
commit
edc0c05404
9 changed files with 217 additions and 8 deletions
|
|
@ -135,7 +135,8 @@ claude mcp add gitnexus -- npx -y gitnexus@latest mcp
|
|||
gitnexus setup # Configure MCP for your editors (one-time)
|
||||
gitnexus analyze [path] # Index a repository (or update stale index)
|
||||
gitnexus analyze --force # Force full re-index
|
||||
gitnexus analyze --skip-embeddings # Skip embedding generation (faster)
|
||||
gitnexus analyze --embeddings # Enable embedding generation (slower, better search)
|
||||
gitnexus analyze --verbose # Log skipped files when parsers are unavailable
|
||||
gitnexus mcp # Start MCP server (stdio) — serves all indexed repos
|
||||
gitnexus serve # Start local HTTP server (multi-repo) for web UI connection
|
||||
gitnexus list # List all indexed repositories
|
||||
|
|
|
|||
|
|
@ -139,7 +139,8 @@ Your AI agent gets these tools automatically:
|
|||
gitnexus setup # Configure MCP for your editors (one-time)
|
||||
gitnexus analyze [path] # Index a repository (or update stale index)
|
||||
gitnexus analyze --force # Force full re-index
|
||||
gitnexus analyze --skip-embeddings # Skip embedding generation (faster)
|
||||
gitnexus analyze --embeddings # Enable embedding generation (slower, better search)
|
||||
gitnexus analyze --verbose # Log skipped files when parsers are unavailable
|
||||
gitnexus mcp # Start MCP server (stdio) — serves all indexed repos
|
||||
gitnexus serve # Start local HTTP server (multi-repo) for web UI
|
||||
gitnexus list # List all indexed repositories
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ function ensureHeap(): boolean {
|
|||
export interface AnalyzeOptions {
|
||||
force?: boolean;
|
||||
embeddings?: boolean;
|
||||
verbose?: boolean;
|
||||
}
|
||||
|
||||
/** Threshold: auto-skip embeddings for repos with more nodes than this */
|
||||
|
|
@ -72,6 +73,10 @@ export const analyzeCommand = async (
|
|||
) => {
|
||||
if (ensureHeap()) return;
|
||||
|
||||
if (options?.verbose) {
|
||||
process.env.GITNEXUS_VERBOSE = '1';
|
||||
}
|
||||
|
||||
console.log('\n GitNexus Analyzer\n');
|
||||
|
||||
let repoPath: string;
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ program
|
|||
.description('Index a repository (full analysis)')
|
||||
.option('-f, --force', 'Force full re-index even if up to date')
|
||||
.option('--embeddings', 'Enable embedding generation for semantic search (off by default)')
|
||||
.option('-v, --verbose', 'Enable verbose ingestion warnings (default: false)')
|
||||
.action(createLazyAction(() => import('./analyze.js'), 'analyzeCommand'));
|
||||
|
||||
program
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ import { ASTCache } from './ast-cache.js';
|
|||
import { SymbolTable } from './symbol-table.js';
|
||||
import { ImportMap } from './import-processor.js';
|
||||
import Parser from 'tree-sitter';
|
||||
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
|
||||
import { isLanguageAvailable, loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
|
||||
import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
|
||||
import { generateId } from '../../lib/utils.js';
|
||||
import { getLanguageFromFilename, yieldToEventLoop } from './utils.js';
|
||||
import { getLanguageFromFilename, isVerboseIngestionEnabled, yieldToEventLoop } from './utils.js';
|
||||
import type { ExtractedCall, ExtractedRoute } from './workers/parse-worker.js';
|
||||
|
||||
/**
|
||||
|
|
@ -150,6 +150,8 @@ export const processCalls = async (
|
|||
onProgress?: (current: number, total: number) => void
|
||||
) => {
|
||||
const parser = await loadParser();
|
||||
const logSkipped = isVerboseIngestionEnabled();
|
||||
const skippedByLang = logSkipped ? new Map<string, number>() : null;
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
|
|
@ -159,6 +161,12 @@ export const processCalls = async (
|
|||
// 1. Check language support first
|
||||
const language = getLanguageFromFilename(file.path);
|
||||
if (!language) continue;
|
||||
if (!isLanguageAvailable(language)) {
|
||||
if (skippedByLang) {
|
||||
skippedByLang.set(language, (skippedByLang.get(language) ?? 0) + 1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const queryStr = LANGUAGE_QUERIES[language];
|
||||
if (!queryStr) continue;
|
||||
|
|
@ -242,6 +250,14 @@ export const processCalls = async (
|
|||
|
||||
// Tree is now owned by the LRU cache — no manual delete needed
|
||||
}
|
||||
|
||||
if (skippedByLang && skippedByLang.size > 0) {
|
||||
for (const [lang, count] of skippedByLang.entries()) {
|
||||
console.warn(
|
||||
`[ingestion] Skipped ${count} ${lang} file(s) in call processing — ${lang} parser not available.`
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ import { KnowledgeGraph } from '../graph/types.js';
|
|||
import { ASTCache } from './ast-cache.js';
|
||||
import { SymbolTable } from './symbol-table.js';
|
||||
import Parser from 'tree-sitter';
|
||||
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
|
||||
import { isLanguageAvailable, loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
|
||||
import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
|
||||
import { generateId } from '../../lib/utils.js';
|
||||
import { getLanguageFromFilename, yieldToEventLoop } from './utils.js';
|
||||
import { getLanguageFromFilename, isVerboseIngestionEnabled, yieldToEventLoop } from './utils.js';
|
||||
import type { ExtractedHeritage } from './workers/parse-worker.js';
|
||||
|
||||
export const processHeritage = async (
|
||||
|
|
@ -24,6 +24,8 @@ export const processHeritage = async (
|
|||
onProgress?: (current: number, total: number) => void
|
||||
) => {
|
||||
const parser = await loadParser();
|
||||
const logSkipped = isVerboseIngestionEnabled();
|
||||
const skippedByLang = logSkipped ? new Map<string, number>() : null;
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
|
|
@ -33,6 +35,12 @@ export const processHeritage = async (
|
|||
// 1. Check language support
|
||||
const language = getLanguageFromFilename(file.path);
|
||||
if (!language) continue;
|
||||
if (!isLanguageAvailable(language)) {
|
||||
if (skippedByLang) {
|
||||
skippedByLang.set(language, (skippedByLang.get(language) ?? 0) + 1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const queryStr = LANGUAGE_QUERIES[language];
|
||||
if (!queryStr) continue;
|
||||
|
|
@ -159,6 +167,14 @@ export const processHeritage = async (
|
|||
|
||||
// Tree is now owned by the LRU cache — no manual delete needed
|
||||
}
|
||||
|
||||
if (skippedByLang && skippedByLang.size > 0) {
|
||||
for (const [lang, count] of skippedByLang.entries()) {
|
||||
console.warn(
|
||||
`[ingestion] Skipped ${count} ${lang} file(s) in heritage processing — ${lang} parser not available.`
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ import path from 'path';
|
|||
import { KnowledgeGraph } from '../graph/types.js';
|
||||
import { ASTCache } from './ast-cache.js';
|
||||
import Parser from 'tree-sitter';
|
||||
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
|
||||
import { isLanguageAvailable, loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
|
||||
import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
|
||||
import { generateId } from '../../lib/utils.js';
|
||||
import { getLanguageFromFilename, yieldToEventLoop } from './utils.js';
|
||||
import { getLanguageFromFilename, isVerboseIngestionEnabled, yieldToEventLoop } from './utils.js';
|
||||
import { SupportedLanguages } from '../../config/supported-languages.js';
|
||||
import type { ExtractedImport } from './workers/parse-worker.js';
|
||||
|
||||
|
|
@ -733,6 +733,8 @@ export const processImports = async (
|
|||
const allFileList = allPaths ?? files.map(f => f.path);
|
||||
const allFilePaths = new Set(allFileList);
|
||||
const parser = await loadParser();
|
||||
const logSkipped = isVerboseIngestionEnabled();
|
||||
const skippedByLang = logSkipped ? new Map<string, number>() : null;
|
||||
const resolveCache = new Map<string, string | null>();
|
||||
// Pre-compute normalized file list once (forward slashes)
|
||||
const normalizedFileList = allFileList.map(p => p.replace(/\\/g, '/'));
|
||||
|
|
@ -781,6 +783,12 @@ export const processImports = async (
|
|||
// 1. Check language support first
|
||||
const language = getLanguageFromFilename(file.path);
|
||||
if (!language) continue;
|
||||
if (!isLanguageAvailable(language)) {
|
||||
if (skippedByLang) {
|
||||
skippedByLang.set(language, (skippedByLang.get(language) ?? 0) + 1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const queryStr = LANGUAGE_QUERIES[language];
|
||||
if (!queryStr) continue;
|
||||
|
|
@ -939,6 +947,14 @@ export const processImports = async (
|
|||
// Tree is now owned by the LRU cache — no manual delete needed
|
||||
}
|
||||
|
||||
if (skippedByLang && skippedByLang.size > 0) {
|
||||
for (const [lang, count] of skippedByLang.entries()) {
|
||||
console.warn(
|
||||
`[ingestion] Skipped ${count} ${lang} file(s) in import processing — ${lang} parser not available.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (isDev) {
|
||||
console.log(`📊 Import processing complete: ${totalImportsResolved}/${totalImportsFound} imports resolved to graph edges`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -60,3 +60,9 @@ export const getLanguageFromFilename = (filename: string): SupportedLanguages |
|
|||
return null;
|
||||
};
|
||||
|
||||
export const isVerboseIngestionEnabled = (): boolean => {
|
||||
const raw = process.env.GITNEXUS_VERBOSE;
|
||||
if (!raw) return false;
|
||||
const value = raw.toLowerCase();
|
||||
return value === '1' || value === 'true' || value === 'yes';
|
||||
};
|
||||
|
|
|
|||
147
gitnexus/test/unit/sequential-language-availability.test.ts
Normal file
147
gitnexus/test/unit/sequential-language-availability.test.ts
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
import { describe, expect, it, vi, beforeEach } from 'vitest';
|
||||
|
||||
vi.mock('../../src/core/tree-sitter/parser-loader.js', () => ({
|
||||
loadParser: vi.fn(async () => ({
|
||||
parse: vi.fn(),
|
||||
getLanguage: vi.fn(),
|
||||
})),
|
||||
loadLanguage: vi.fn(async () => undefined),
|
||||
isLanguageAvailable: vi.fn(() => true),
|
||||
}));
|
||||
|
||||
import { createKnowledgeGraph } from '../../src/core/graph/graph.js';
|
||||
import { createASTCache } from '../../src/core/ingestion/ast-cache.js';
|
||||
import { createImportMap, processImports } from '../../src/core/ingestion/import-processor.js';
|
||||
import { processCalls } from '../../src/core/ingestion/call-processor.js';
|
||||
import { processHeritage } from '../../src/core/ingestion/heritage-processor.js';
|
||||
import { createSymbolTable } from '../../src/core/ingestion/symbol-table.js';
|
||||
import * as parserLoader from '../../src/core/tree-sitter/parser-loader.js';
|
||||
|
||||
|
||||
describe('sequential native parser availability', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('skips Swift files in processImports when the native parser is unavailable', async () => {
|
||||
vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false);
|
||||
|
||||
await expect(processImports(
|
||||
createKnowledgeGraph(),
|
||||
[{ path: 'App.swift', content: 'import Foundation' }],
|
||||
createASTCache(),
|
||||
createImportMap(),
|
||||
undefined,
|
||||
'/tmp/repo',
|
||||
['App.swift'],
|
||||
)).resolves.toBeUndefined();
|
||||
|
||||
expect(parserLoader.loadLanguage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('warns when processImports skips files in verbose mode', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
const previous = process.env.GITNEXUS_VERBOSE;
|
||||
process.env.GITNEXUS_VERBOSE = '1';
|
||||
vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false);
|
||||
|
||||
await processImports(
|
||||
createKnowledgeGraph(),
|
||||
[{ path: 'App.swift', content: 'import Foundation' }],
|
||||
createASTCache(),
|
||||
createImportMap(),
|
||||
undefined,
|
||||
'/tmp/repo',
|
||||
['App.swift'],
|
||||
);
|
||||
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
'[ingestion] Skipped 1 swift file(s) in import processing — swift parser not available.'
|
||||
);
|
||||
|
||||
warnSpy.mockRestore();
|
||||
if (previous === undefined) {
|
||||
delete process.env.GITNEXUS_VERBOSE;
|
||||
} else {
|
||||
process.env.GITNEXUS_VERBOSE = previous;
|
||||
}
|
||||
});
|
||||
|
||||
it('skips Swift files in processCalls when the native parser is unavailable', async () => {
|
||||
vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false);
|
||||
|
||||
await expect(processCalls(
|
||||
createKnowledgeGraph(),
|
||||
[{ path: 'App.swift', content: 'func demo() {}' }],
|
||||
createASTCache(),
|
||||
createSymbolTable(),
|
||||
createImportMap(),
|
||||
)).resolves.toBeUndefined();
|
||||
|
||||
expect(parserLoader.loadLanguage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('warns when processCalls skips files in verbose mode', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
const previous = process.env.GITNEXUS_VERBOSE;
|
||||
process.env.GITNEXUS_VERBOSE = '1';
|
||||
vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false);
|
||||
|
||||
await processCalls(
|
||||
createKnowledgeGraph(),
|
||||
[{ path: 'App.swift', content: 'func demo() {}' }],
|
||||
createASTCache(),
|
||||
createSymbolTable(),
|
||||
createImportMap(),
|
||||
);
|
||||
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
'[ingestion] Skipped 1 swift file(s) in call processing — swift parser not available.'
|
||||
);
|
||||
|
||||
warnSpy.mockRestore();
|
||||
if (previous === undefined) {
|
||||
delete process.env.GITNEXUS_VERBOSE;
|
||||
} else {
|
||||
process.env.GITNEXUS_VERBOSE = previous;
|
||||
}
|
||||
});
|
||||
|
||||
it('skips Swift files in processHeritage when the native parser is unavailable', async () => {
|
||||
vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false);
|
||||
|
||||
await expect(processHeritage(
|
||||
createKnowledgeGraph(),
|
||||
[{ path: 'App.swift', content: 'class AppViewController: UIViewController {}' }],
|
||||
createASTCache(),
|
||||
createSymbolTable(),
|
||||
)).resolves.toBeUndefined();
|
||||
|
||||
expect(parserLoader.loadLanguage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('warns when processHeritage skips files in verbose mode', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
const previous = process.env.GITNEXUS_VERBOSE;
|
||||
process.env.GITNEXUS_VERBOSE = '1';
|
||||
vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false);
|
||||
|
||||
await processHeritage(
|
||||
createKnowledgeGraph(),
|
||||
[{ path: 'App.swift', content: 'class AppViewController: UIViewController {}' }],
|
||||
createASTCache(),
|
||||
createSymbolTable(),
|
||||
);
|
||||
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
'[ingestion] Skipped 1 swift file(s) in heritage processing — swift parser not available.'
|
||||
);
|
||||
|
||||
warnSpy.mockRestore();
|
||||
if (previous === undefined) {
|
||||
delete process.env.GITNEXUS_VERBOSE;
|
||||
} else {
|
||||
process.env.GITNEXUS_VERBOSE = previous;
|
||||
}
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue