From edc0c05404679f4db098daec1eecf36a73a841b3 Mon Sep 17 00:00:00 2001 From: Ryanba <92616678+Gujiassh@users.noreply.github.com> Date: Tue, 10 Mar 2026 22:26:36 +0800 Subject: [PATCH] fix: skip unavailable native Swift parsers in sequential ingestion (#188) * fix: skip unavailable native Swift parsers in sequential ingestion * fix: warn when ingestion skips languages in verbose mode * test: cover verbose skip warnings * docs: update analyze flags * docs: clarify verbose default --------- Co-authored-by: Gergo Magyar --- README.md | 3 +- gitnexus/README.md | 3 +- gitnexus/src/cli/analyze.ts | 5 + gitnexus/src/cli/index.ts | 1 + gitnexus/src/core/ingestion/call-processor.ts | 20 ++- .../src/core/ingestion/heritage-processor.ts | 20 ++- .../src/core/ingestion/import-processor.ts | 20 ++- gitnexus/src/core/ingestion/utils.ts | 6 + .../sequential-language-availability.test.ts | 147 ++++++++++++++++++ 9 files changed, 217 insertions(+), 8 deletions(-) create mode 100644 gitnexus/test/unit/sequential-language-availability.test.ts diff --git a/README.md b/README.md index 77d8e5967..a42ba718d 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,8 @@ claude mcp add gitnexus -- npx -y gitnexus@latest mcp gitnexus setup # Configure MCP for your editors (one-time) gitnexus analyze [path] # Index a repository (or update stale index) gitnexus analyze --force # Force full re-index -gitnexus analyze --skip-embeddings # Skip embedding generation (faster) +gitnexus analyze --embeddings # Enable embedding generation (slower, better search) +gitnexus analyze --verbose # Log skipped files when parsers are unavailable gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI connection gitnexus list # List all indexed repositories diff --git a/gitnexus/README.md b/gitnexus/README.md index d66f94fe2..7b6613979 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -139,7 +139,8 @@ Your AI agent gets these tools automatically: gitnexus setup # Configure MCP for your editors (one-time) gitnexus analyze [path] # Index a repository (or update stale index) gitnexus analyze --force # Force full re-index -gitnexus analyze --skip-embeddings # Skip embedding generation (faster) +gitnexus analyze --embeddings # Enable embedding generation (slower, better search) +gitnexus analyze --verbose # Log skipped files when parsers are unavailable gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI gitnexus list # List all indexed repositories diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index c6a7c3b07..4b1d4ff06 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -45,6 +45,7 @@ function ensureHeap(): boolean { export interface AnalyzeOptions { force?: boolean; embeddings?: boolean; + verbose?: boolean; } /** Threshold: auto-skip embeddings for repos with more nodes than this */ @@ -72,6 +73,10 @@ export const analyzeCommand = async ( ) => { if (ensureHeap()) return; + if (options?.verbose) { + process.env.GITNEXUS_VERBOSE = '1'; + } + console.log('\n GitNexus Analyzer\n'); let repoPath: string; diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 7ca2ce430..5968ae97e 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -26,6 +26,7 @@ program .description('Index a repository (full analysis)') .option('-f, --force', 'Force full re-index even if up to date') .option('--embeddings', 'Enable embedding generation for semantic search (off by default)') + .option('-v, --verbose', 'Enable verbose ingestion warnings (default: false)') .action(createLazyAction(() => import('./analyze.js'), 'analyzeCommand')); program diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 9b6dfc3f4..36a53e4ae 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -3,10 +3,10 @@ import { ASTCache } from './ast-cache.js'; import { SymbolTable } from './symbol-table.js'; import { ImportMap } from './import-processor.js'; import Parser from 'tree-sitter'; -import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; +import { isLanguageAvailable, loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; import { LANGUAGE_QUERIES } from './tree-sitter-queries.js'; import { generateId } from '../../lib/utils.js'; -import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; +import { getLanguageFromFilename, isVerboseIngestionEnabled, yieldToEventLoop } from './utils.js'; import type { ExtractedCall, ExtractedRoute } from './workers/parse-worker.js'; /** @@ -150,6 +150,8 @@ export const processCalls = async ( onProgress?: (current: number, total: number) => void ) => { const parser = await loadParser(); + const logSkipped = isVerboseIngestionEnabled(); + const skippedByLang = logSkipped ? new Map() : null; for (let i = 0; i < files.length; i++) { const file = files[i]; @@ -159,6 +161,12 @@ export const processCalls = async ( // 1. Check language support first const language = getLanguageFromFilename(file.path); if (!language) continue; + if (!isLanguageAvailable(language)) { + if (skippedByLang) { + skippedByLang.set(language, (skippedByLang.get(language) ?? 0) + 1); + } + continue; + } const queryStr = LANGUAGE_QUERIES[language]; if (!queryStr) continue; @@ -242,6 +250,14 @@ export const processCalls = async ( // Tree is now owned by the LRU cache — no manual delete needed } + + if (skippedByLang && skippedByLang.size > 0) { + for (const [lang, count] of skippedByLang.entries()) { + console.warn( + `[ingestion] Skipped ${count} ${lang} file(s) in call processing — ${lang} parser not available.` + ); + } + } }; /** diff --git a/gitnexus/src/core/ingestion/heritage-processor.ts b/gitnexus/src/core/ingestion/heritage-processor.ts index dbb7bac8c..6113f04c6 100644 --- a/gitnexus/src/core/ingestion/heritage-processor.ts +++ b/gitnexus/src/core/ingestion/heritage-processor.ts @@ -10,10 +10,10 @@ import { KnowledgeGraph } from '../graph/types.js'; import { ASTCache } from './ast-cache.js'; import { SymbolTable } from './symbol-table.js'; import Parser from 'tree-sitter'; -import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; +import { isLanguageAvailable, loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; import { LANGUAGE_QUERIES } from './tree-sitter-queries.js'; import { generateId } from '../../lib/utils.js'; -import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; +import { getLanguageFromFilename, isVerboseIngestionEnabled, yieldToEventLoop } from './utils.js'; import type { ExtractedHeritage } from './workers/parse-worker.js'; export const processHeritage = async ( @@ -24,6 +24,8 @@ export const processHeritage = async ( onProgress?: (current: number, total: number) => void ) => { const parser = await loadParser(); + const logSkipped = isVerboseIngestionEnabled(); + const skippedByLang = logSkipped ? new Map() : null; for (let i = 0; i < files.length; i++) { const file = files[i]; @@ -33,6 +35,12 @@ export const processHeritage = async ( // 1. Check language support const language = getLanguageFromFilename(file.path); if (!language) continue; + if (!isLanguageAvailable(language)) { + if (skippedByLang) { + skippedByLang.set(language, (skippedByLang.get(language) ?? 0) + 1); + } + continue; + } const queryStr = LANGUAGE_QUERIES[language]; if (!queryStr) continue; @@ -159,6 +167,14 @@ export const processHeritage = async ( // Tree is now owned by the LRU cache — no manual delete needed } + + if (skippedByLang && skippedByLang.size > 0) { + for (const [lang, count] of skippedByLang.entries()) { + console.warn( + `[ingestion] Skipped ${count} ${lang} file(s) in heritage processing — ${lang} parser not available.` + ); + } + } }; /** diff --git a/gitnexus/src/core/ingestion/import-processor.ts b/gitnexus/src/core/ingestion/import-processor.ts index 990f968af..7aaf4a199 100644 --- a/gitnexus/src/core/ingestion/import-processor.ts +++ b/gitnexus/src/core/ingestion/import-processor.ts @@ -3,10 +3,10 @@ import path from 'path'; import { KnowledgeGraph } from '../graph/types.js'; import { ASTCache } from './ast-cache.js'; import Parser from 'tree-sitter'; -import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; +import { isLanguageAvailable, loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; import { LANGUAGE_QUERIES } from './tree-sitter-queries.js'; import { generateId } from '../../lib/utils.js'; -import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; +import { getLanguageFromFilename, isVerboseIngestionEnabled, yieldToEventLoop } from './utils.js'; import { SupportedLanguages } from '../../config/supported-languages.js'; import type { ExtractedImport } from './workers/parse-worker.js'; @@ -733,6 +733,8 @@ export const processImports = async ( const allFileList = allPaths ?? files.map(f => f.path); const allFilePaths = new Set(allFileList); const parser = await loadParser(); + const logSkipped = isVerboseIngestionEnabled(); + const skippedByLang = logSkipped ? new Map() : null; const resolveCache = new Map(); // Pre-compute normalized file list once (forward slashes) const normalizedFileList = allFileList.map(p => p.replace(/\\/g, '/')); @@ -781,6 +783,12 @@ export const processImports = async ( // 1. Check language support first const language = getLanguageFromFilename(file.path); if (!language) continue; + if (!isLanguageAvailable(language)) { + if (skippedByLang) { + skippedByLang.set(language, (skippedByLang.get(language) ?? 0) + 1); + } + continue; + } const queryStr = LANGUAGE_QUERIES[language]; if (!queryStr) continue; @@ -939,6 +947,14 @@ export const processImports = async ( // Tree is now owned by the LRU cache — no manual delete needed } + if (skippedByLang && skippedByLang.size > 0) { + for (const [lang, count] of skippedByLang.entries()) { + console.warn( + `[ingestion] Skipped ${count} ${lang} file(s) in import processing — ${lang} parser not available.` + ); + } + } + if (isDev) { console.log(`📊 Import processing complete: ${totalImportsResolved}/${totalImportsFound} imports resolved to graph edges`); } diff --git a/gitnexus/src/core/ingestion/utils.ts b/gitnexus/src/core/ingestion/utils.ts index 47fa8cbe4..7abe07626 100644 --- a/gitnexus/src/core/ingestion/utils.ts +++ b/gitnexus/src/core/ingestion/utils.ts @@ -60,3 +60,9 @@ export const getLanguageFromFilename = (filename: string): SupportedLanguages | return null; }; +export const isVerboseIngestionEnabled = (): boolean => { + const raw = process.env.GITNEXUS_VERBOSE; + if (!raw) return false; + const value = raw.toLowerCase(); + return value === '1' || value === 'true' || value === 'yes'; +}; diff --git a/gitnexus/test/unit/sequential-language-availability.test.ts b/gitnexus/test/unit/sequential-language-availability.test.ts new file mode 100644 index 000000000..4a88a1a6a --- /dev/null +++ b/gitnexus/test/unit/sequential-language-availability.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest'; + +vi.mock('../../src/core/tree-sitter/parser-loader.js', () => ({ + loadParser: vi.fn(async () => ({ + parse: vi.fn(), + getLanguage: vi.fn(), + })), + loadLanguage: vi.fn(async () => undefined), + isLanguageAvailable: vi.fn(() => true), +})); + +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { createASTCache } from '../../src/core/ingestion/ast-cache.js'; +import { createImportMap, processImports } from '../../src/core/ingestion/import-processor.js'; +import { processCalls } from '../../src/core/ingestion/call-processor.js'; +import { processHeritage } from '../../src/core/ingestion/heritage-processor.js'; +import { createSymbolTable } from '../../src/core/ingestion/symbol-table.js'; +import * as parserLoader from '../../src/core/tree-sitter/parser-loader.js'; + + +describe('sequential native parser availability', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('skips Swift files in processImports when the native parser is unavailable', async () => { + vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false); + + await expect(processImports( + createKnowledgeGraph(), + [{ path: 'App.swift', content: 'import Foundation' }], + createASTCache(), + createImportMap(), + undefined, + '/tmp/repo', + ['App.swift'], + )).resolves.toBeUndefined(); + + expect(parserLoader.loadLanguage).not.toHaveBeenCalled(); + }); + + it('warns when processImports skips files in verbose mode', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const previous = process.env.GITNEXUS_VERBOSE; + process.env.GITNEXUS_VERBOSE = '1'; + vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false); + + await processImports( + createKnowledgeGraph(), + [{ path: 'App.swift', content: 'import Foundation' }], + createASTCache(), + createImportMap(), + undefined, + '/tmp/repo', + ['App.swift'], + ); + + expect(warnSpy).toHaveBeenCalledWith( + '[ingestion] Skipped 1 swift file(s) in import processing — swift parser not available.' + ); + + warnSpy.mockRestore(); + if (previous === undefined) { + delete process.env.GITNEXUS_VERBOSE; + } else { + process.env.GITNEXUS_VERBOSE = previous; + } + }); + + it('skips Swift files in processCalls when the native parser is unavailable', async () => { + vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false); + + await expect(processCalls( + createKnowledgeGraph(), + [{ path: 'App.swift', content: 'func demo() {}' }], + createASTCache(), + createSymbolTable(), + createImportMap(), + )).resolves.toBeUndefined(); + + expect(parserLoader.loadLanguage).not.toHaveBeenCalled(); + }); + + it('warns when processCalls skips files in verbose mode', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const previous = process.env.GITNEXUS_VERBOSE; + process.env.GITNEXUS_VERBOSE = '1'; + vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false); + + await processCalls( + createKnowledgeGraph(), + [{ path: 'App.swift', content: 'func demo() {}' }], + createASTCache(), + createSymbolTable(), + createImportMap(), + ); + + expect(warnSpy).toHaveBeenCalledWith( + '[ingestion] Skipped 1 swift file(s) in call processing — swift parser not available.' + ); + + warnSpy.mockRestore(); + if (previous === undefined) { + delete process.env.GITNEXUS_VERBOSE; + } else { + process.env.GITNEXUS_VERBOSE = previous; + } + }); + + it('skips Swift files in processHeritage when the native parser is unavailable', async () => { + vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false); + + await expect(processHeritage( + createKnowledgeGraph(), + [{ path: 'App.swift', content: 'class AppViewController: UIViewController {}' }], + createASTCache(), + createSymbolTable(), + )).resolves.toBeUndefined(); + + expect(parserLoader.loadLanguage).not.toHaveBeenCalled(); + }); + + it('warns when processHeritage skips files in verbose mode', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const previous = process.env.GITNEXUS_VERBOSE; + process.env.GITNEXUS_VERBOSE = '1'; + vi.mocked(parserLoader.isLanguageAvailable).mockReturnValue(false); + + await processHeritage( + createKnowledgeGraph(), + [{ path: 'App.swift', content: 'class AppViewController: UIViewController {}' }], + createASTCache(), + createSymbolTable(), + ); + + expect(warnSpy).toHaveBeenCalledWith( + '[ingestion] Skipped 1 swift file(s) in heritage processing — swift parser not available.' + ); + + warnSpy.mockRestore(); + if (previous === undefined) { + delete process.env.GITNEXUS_VERBOSE; + } else { + process.env.GITNEXUS_VERBOSE = previous; + } + }); +});