From 42473d67df882b8cccfcfe0300b2bbb00e36c81a Mon Sep 17 00:00:00 2001 From: abhigyanpatwari Date: Tue, 17 Feb 2026 16:31:05 +0530 Subject: [PATCH] fixed MCP concurrency issue with connection pool on kuzu --- gitnexus/package.json | 2 +- gitnexus/src/cli/index.ts | 6 +- gitnexus/src/cli/wiki.ts | 250 ++++++++++++++++--- gitnexus/src/core/wiki/generator.ts | 298 ++++++++++++++++------- gitnexus/src/core/wiki/html-viewer.ts | 331 ++++++++++++++++++++++++++ gitnexus/src/core/wiki/llm-client.ts | 92 +++++-- gitnexus/src/mcp/core/kuzu-adapter.ts | 134 +++++++++-- 7 files changed, 954 insertions(+), 159 deletions(-) create mode 100644 gitnexus/src/core/wiki/html-viewer.ts diff --git a/gitnexus/package.json b/gitnexus/package.json index ef7a2f4394..2a40f17f3b 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -1,6 +1,6 @@ { "name": "gitnexus", - "version": "1.1.9", + "version": "1.2.6", "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.", "author": "Abhigyan Patwari", "license": "PolyForm-Noncommercial-1.0.0", diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index de326fb1cb..b7be04a9f2 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -16,7 +16,7 @@ const program = new Command(); program .name('gitnexus') .description('GitNexus local CLI and MCP server') - .version('1.1.9'); + .version('1.2.0'); program .command('setup') @@ -62,9 +62,11 @@ program .command('wiki [path]') .description('Generate repository wiki from knowledge graph') .option('-f, --force', 'Force full regeneration even if up to date') - .option('--model ', 'LLM model name (default: gpt-4o-mini)') + .option('--model ', 'LLM model name (default: minimax/minimax-m2.5)') .option('--base-url ', 'LLM API base URL (default: OpenAI)') .option('--api-key ', 'LLM API key (saved to ~/.gitnexus/config.json)') + .option('--concurrency ', 'Parallel LLM calls (default: 3)', '3') + .option('--gist', 'Publish wiki as a public GitHub Gist after generation') .action(wikiCommand); program diff --git a/gitnexus/src/cli/wiki.ts b/gitnexus/src/cli/wiki.ts index 6589cb8b77..ac626fe809 100644 --- a/gitnexus/src/cli/wiki.ts +++ b/gitnexus/src/cli/wiki.ts @@ -7,6 +7,7 @@ import path from 'path'; import readline from 'readline'; +import { execSync } from 'child_process'; import cliProgress from 'cli-progress'; import { getGitRoot, isGitRepo } from '../storage/git.js'; import { getStoragePaths, loadMeta, loadCLIConfig, saveCLIConfig } from '../storage/repo-manager.js'; @@ -18,6 +19,8 @@ export interface WikiCommandOptions { model?: string; baseUrl?: string; apiKey?: string; + concurrency?: string; + gist?: boolean; } /** @@ -109,52 +112,103 @@ export const wikiCommand = async ( } // ── Resolve LLM config (with interactive fallback) ───────────────── - // If --api-key was passed via CLI, save it immediately - if (options?.apiKey) { + // Save any CLI overrides immediately + if (options?.apiKey || options?.model || options?.baseUrl) { const existing = await loadCLIConfig(); - await saveCLIConfig({ ...existing, apiKey: options.apiKey }); - console.log(' API key saved to ~/.gitnexus/config.json\n'); + const updates: Record = {}; + if (options.apiKey) updates.apiKey = options.apiKey; + if (options.model) updates.model = options.model; + if (options.baseUrl) updates.baseUrl = options.baseUrl; + await saveCLIConfig({ ...existing, ...updates }); + console.log(' Config saved to ~/.gitnexus/config.json\n'); } + const savedConfig = await loadCLIConfig(); + const hasSavedConfig = !!(savedConfig.apiKey && savedConfig.baseUrl); + const hasCLIOverrides = !!(options?.apiKey || options?.model || options?.baseUrl); + let llmConfig = await resolveLLMConfig({ model: options?.model, baseUrl: options?.baseUrl, apiKey: options?.apiKey, }); - if (!llmConfig.apiKey) { + // Run interactive setup if no saved config and no CLI flags provided + // (even if env vars exist — let user explicitly choose their provider) + if (!hasSavedConfig && !hasCLIOverrides) { if (!process.stdin.isTTY) { - console.log(' Error: No LLM API key found.'); - console.log(' Set OPENAI_API_KEY or GITNEXUS_API_KEY environment variable,'); - console.log(' or pass --api-key .\n'); - process.exitCode = 1; - return; - } - - console.log(' No API key configured.\n'); - console.log(' The wiki command requires an LLM API key (OpenAI-compatible).'); - console.log(' You can also set OPENAI_API_KEY or GITNEXUS_API_KEY env var.\n'); - - const key = await prompt(' Enter your API key: ', true); - if (!key) { - console.log('\n No key provided. Aborting.\n'); - process.exitCode = 1; - return; - } - - const save = await prompt(' Save key to ~/.gitnexus/config.json for future use? (Y/n): '); - if (!save || save.toLowerCase() === 'y' || save.toLowerCase() === 'yes') { - const existing = await loadCLIConfig(); - await saveCLIConfig({ ...existing, apiKey: key }); - console.log(' Key saved.\n'); + if (!llmConfig.apiKey) { + console.log(' Error: No LLM API key found.'); + console.log(' Set OPENAI_API_KEY or GITNEXUS_API_KEY environment variable,'); + console.log(' or pass --api-key .\n'); + process.exitCode = 1; + return; + } + // Non-interactive with env var — just use it } else { - console.log(' Key will be used for this session only.\n'); - } + console.log(' No LLM configured. Let\'s set it up.\n'); + console.log(' Supports OpenAI, OpenRouter, or any OpenAI-compatible API.\n'); - llmConfig = { ...llmConfig, apiKey: key }; + // Provider selection + console.log(' [1] OpenAI (api.openai.com)'); + console.log(' [2] OpenRouter (openrouter.ai)'); + console.log(' [3] Custom endpoint\n'); + + const choice = await prompt(' Select provider (1/2/3): '); + + let baseUrl: string; + let defaultModel: string; + + if (choice === '2') { + baseUrl = 'https://openrouter.ai/api/v1'; + defaultModel = 'minimax/minimax-m2.5'; + } else if (choice === '3') { + baseUrl = await prompt(' Base URL (e.g. http://localhost:11434/v1): '); + if (!baseUrl) { + console.log('\n No URL provided. Aborting.\n'); + process.exitCode = 1; + return; + } + defaultModel = 'gpt-4o-mini'; + } else { + baseUrl = 'https://api.openai.com/v1'; + defaultModel = 'gpt-4o-mini'; + } + + // Model + const modelInput = await prompt(` Model (default: ${defaultModel}): `); + const model = modelInput || defaultModel; + + // API key — pre-fill hint if env var exists + const envKey = process.env.GITNEXUS_API_KEY || process.env.OPENAI_API_KEY || ''; + let key: string; + if (envKey) { + const masked = envKey.slice(0, 6) + '...' + envKey.slice(-4); + const useEnv = await prompt(` Use existing env key (${masked})? (Y/n): `); + if (!useEnv || useEnv.toLowerCase() === 'y' || useEnv.toLowerCase() === 'yes') { + key = envKey; + } else { + key = await prompt(' API key: ', true); + } + } else { + key = await prompt(' API key: ', true); + } + + if (!key) { + console.log('\n No key provided. Aborting.\n'); + process.exitCode = 1; + return; + } + + // Save + await saveCLIConfig({ apiKey: key, baseUrl, model }); + console.log(' Config saved to ~/.gitnexus/config.json\n'); + + llmConfig = { ...llmConfig, apiKey: key, baseUrl, model }; + } } - // ── Setup progress bar ────────────────────────────────────────────── + // ── Setup progress bar with elapsed timer ────────────────────────── const bar = new cliProgress.SingleBar({ format: ' {bar} {percentage}% | {phase}', barCompleteChar: '\u2588', @@ -169,12 +223,25 @@ export const wikiCommand = async ( bar.start(100, 0, { phase: 'Initializing...' }); const t0 = Date.now(); + let lastPhase = ''; + let phaseStart = t0; + + // Tick elapsed time every second while stuck on the same phase + const elapsedTimer = setInterval(() => { + if (lastPhase) { + const elapsed = Math.round((Date.now() - phaseStart) / 1000); + if (elapsed >= 3) { + bar.update({ phase: `${lastPhase} (${elapsed}s)` }); + } + } + }, 1000); // ── Run generator ─────────────────────────────────────────────────── const wikiOptions: WikiOptions = { force: options?.force, model: options?.model, baseUrl: options?.baseUrl, + concurrency: options?.concurrency ? parseInt(options.concurrency, 10) : undefined, }; const generator = new WikiGenerator( @@ -184,28 +251,39 @@ export const wikiCommand = async ( llmConfig, wikiOptions, (phase, percent, detail) => { - bar.update(percent, { phase: detail || phase }); + const label = detail || phase; + if (label !== lastPhase) { + lastPhase = label; + phaseStart = Date.now(); + } + bar.update(percent, { phase: label }); }, ); try { const result = await generator.run(); + clearInterval(elapsedTimer); bar.update(100, { phase: 'Done' }); bar.stop(); const elapsed = ((Date.now() - t0) / 1000).toFixed(1); + const wikiDir = path.join(storagePath, 'wiki'); + const viewerPath = path.join(wikiDir, 'index.html'); + if (result.mode === 'up-to-date' && !options?.force) { console.log('\n Wiki is already up to date.'); - console.log(` ${path.join(storagePath, 'wiki')}\n`); + console.log(` Viewer: ${viewerPath}\n`); + await maybePublishGist(viewerPath, options?.gist); return; } console.log(`\n Wiki generated successfully (${elapsed}s)\n`); console.log(` Mode: ${result.mode}`); console.log(` Pages: ${result.pagesGenerated}`); - console.log(` Output: ${path.join(storagePath, 'wiki')}`); + console.log(` Output: ${wikiDir}`); + console.log(` Viewer: ${viewerPath}`); if (result.failedModules && result.failedModules.length > 0) { console.log(`\n Failed modules (${result.failedModules.length}):`); @@ -216,13 +294,32 @@ export const wikiCommand = async ( } console.log(''); + + await maybePublishGist(viewerPath, options?.gist); } catch (err: any) { + clearInterval(elapsedTimer); bar.stop(); if (err.message?.includes('No source files')) { console.log(`\n ${err.message}\n`); } else if (err.message?.includes('API key') || err.message?.includes('API error')) { console.log(`\n LLM Error: ${err.message}\n`); + + // Offer to reconfigure on auth-related failures + const isAuthError = err.message?.includes('401') || err.message?.includes('403') + || err.message?.includes('502') || err.message?.includes('authenticate') + || err.message?.includes('Unauthorized'); + if (isAuthError && process.stdin.isTTY) { + const answer = await new Promise((resolve) => { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + rl.question(' Reconfigure LLM settings? (Y/n): ', (ans) => { rl.close(); resolve(ans.trim().toLowerCase()); }); + }); + if (!answer || answer === 'y' || answer === 'yes') { + // Clear saved config so next run triggers interactive setup + await saveCLIConfig({}); + console.log(' Config cleared. Run `gitnexus wiki` again to reconfigure.\n'); + } + } } else { console.log(`\n Error: ${err.message}\n`); if (process.env.DEBUG) { @@ -232,3 +329,86 @@ export const wikiCommand = async ( process.exitCode = 1; } }; + +// ─── Gist Publishing ─────────────────────────────────────────────────── + +function hasGhCLI(): boolean { + try { + execSync('gh --version', { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +function publishGist(htmlPath: string): { url: string; rawUrl: string } | null { + try { + const output = execSync( + `gh gist create "${htmlPath}" --desc "Repository Wiki — generated by GitNexus" --public`, + { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }, + ).trim(); + + // gh gist create prints the gist URL as the last line + const lines = output.split('\n'); + const gistUrl = lines.find(l => l.includes('gist.github.com')) || lines[lines.length - 1]; + + if (!gistUrl || !gistUrl.includes('gist.github.com')) return null; + + // Build a raw viewer URL via gist.githack.com + // gist URL format: https://gist.github.com/{user}/{id} + const match = gistUrl.match(/gist\.github\.com\/([^/]+)\/([a-f0-9]+)/); + let rawUrl = gistUrl; + if (match) { + rawUrl = `https://gistcdn.githack.com/${match[1]}/${match[2]}/raw/index.html`; + } + + return { url: gistUrl.trim(), rawUrl }; + } catch { + return null; + } +} + +async function maybePublishGist(htmlPath: string, gistFlag?: boolean): Promise { + if (gistFlag === false) return; + + // Check that the HTML file exists + try { + const fs = await import('fs/promises'); + await fs.access(htmlPath); + } catch { + return; + } + + if (!hasGhCLI()) { + if (gistFlag) { + console.log(' GitHub CLI (gh) is not installed. Cannot publish gist.'); + console.log(' Install it: https://cli.github.com\n'); + } + return; + } + + let shouldPublish = !!gistFlag; + + if (!shouldPublish && process.stdin.isTTY) { + const answer = await new Promise((resolve) => { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + rl.question(' Publish wiki as a GitHub Gist for easy viewing? (Y/n): ', (ans) => { + rl.close(); + resolve(ans.trim().toLowerCase()); + }); + }); + shouldPublish = !answer || answer === 'y' || answer === 'yes'; + } + + if (!shouldPublish) return; + + console.log('\n Publishing to GitHub Gist...'); + const result = publishGist(htmlPath); + + if (result) { + console.log(` Gist: ${result.url}`); + console.log(` Viewer: ${result.rawUrl}\n`); + } else { + console.log(' Failed to publish gist. Make sure `gh auth login` is configured.\n'); + } +} diff --git a/gitnexus/src/core/wiki/generator.ts b/gitnexus/src/core/wiki/generator.ts index 092ce2e531..29a16a5419 100644 --- a/gitnexus/src/core/wiki/generator.ts +++ b/gitnexus/src/core/wiki/generator.ts @@ -27,11 +27,13 @@ import { getInterModuleEdgesForOverview, type FileWithExports, } from './graph-queries.js'; +import { generateHTMLViewer } from './html-viewer.js'; import { callLLM, estimateTokens, type LLMConfig, + type CallLLMOptions, } from './llm-client.js'; import { @@ -60,6 +62,7 @@ export interface WikiOptions { baseUrl?: string; apiKey?: string; maxTokensPerModule?: number; + concurrency?: number; } export interface WikiMeta { @@ -93,6 +96,7 @@ export class WikiGenerator { private kuzuPath: string; private llmConfig: LLMConfig; private maxTokensPerModule: number; + private concurrency: number; private options: WikiOptions; private onProgress: ProgressCallback; private failedModules: string[] = []; @@ -112,7 +116,28 @@ export class WikiGenerator { this.options = options; this.llmConfig = llmConfig; this.maxTokensPerModule = options.maxTokensPerModule ?? DEFAULT_MAX_TOKENS_PER_MODULE; - this.onProgress = onProgress || (() => {}); + this.concurrency = options.concurrency ?? 3; + const progressFn = onProgress || (() => {}); + this.onProgress = (phase, percent, detail) => { + if (percent > 0) this.lastPercent = percent; + progressFn(phase, percent, detail); + }; + } + + private lastPercent = 0; + + /** + * Create streaming options that report LLM progress to the progress bar. + * Uses the last known percent so streaming doesn't reset the bar backwards. + */ + private streamOpts(label: string, fixedPercent?: number): CallLLMOptions { + return { + onChunk: (chars: number) => { + const tokens = Math.round(chars / 4); + const pct = fixedPercent ?? this.lastPercent; + this.onProgress('stream', pct, `${label} (${tokens} tok)`); + }, + }; } /** @@ -127,6 +152,8 @@ export class WikiGenerator { // Up-to-date check (skip if --force) if (!forceMode && existingMeta && existingMeta.fromCommit === currentCommit) { + // Still regenerate the HTML viewer in case it's missing + await this.ensureHTMLViewer(); return { pagesGenerated: 0, mode: 'up-to-date', failedModules: [] }; } @@ -146,14 +173,34 @@ export class WikiGenerator { this.onProgress('init', 2, 'Connecting to knowledge graph...'); await initWikiDb(this.kuzuPath); + let result: { pagesGenerated: number; mode: 'full' | 'incremental' | 'up-to-date'; failedModules: string[] }; try { if (!forceMode && existingMeta && existingMeta.fromCommit) { - return await this.incrementalUpdate(existingMeta, currentCommit); + result = await this.incrementalUpdate(existingMeta, currentCommit); + } else { + result = await this.fullGeneration(currentCommit); } - return await this.fullGeneration(currentCommit); } finally { await closeWikiDb(); } + + // Always generate the HTML viewer after wiki content changes + await this.ensureHTMLViewer(); + + return result; + } + + // ─── HTML Viewer ───────────────────────────────────────────────────── + + private async ensureHTMLViewer(): Promise { + // Only generate if there are markdown pages to bundle + const dirEntries = await fs.readdir(this.wikiDir).catch(() => [] as string[]); + const hasMd = dirEntries.some(f => f.endsWith('.md')); + if (!hasMd) return; + + this.onProgress('html', 98, 'Building HTML viewer...'); + const repoName = path.basename(this.repoPath); + await generateHTMLViewer(this.wikiDir, repoName); } // ─── Full Generation ──────────────────────────────────────────────── @@ -184,17 +231,56 @@ export class WikiGenerator { const moduleTree = await this.buildModuleTree(enrichedFiles); pagesGenerated = 0; - // Phase 2: Generate module pages (bottom-up) + // Phase 2: Generate module pages (parallel with concurrency limit) const totalModules = this.countModules(moduleTree); let modulesProcessed = 0; - for (const node of moduleTree) { - const generated = await this.generateModulePage(node, () => { - modulesProcessed++; - const percent = 30 + Math.round((modulesProcessed / totalModules) * 55); - this.onProgress('modules', percent, `${modulesProcessed}/${totalModules} modules`); - }); - pagesGenerated += generated; + const reportProgress = (moduleName?: string) => { + modulesProcessed++; + const percent = 30 + Math.round((modulesProcessed / totalModules) * 55); + const detail = moduleName + ? `${modulesProcessed}/${totalModules} — ${moduleName}` + : `${modulesProcessed}/${totalModules} modules`; + this.onProgress('modules', percent, detail); + }; + + // Flatten tree into layers: leaves first, then parents + // Leaves can run in parallel; parents must wait for their children + const { leaves, parents } = this.flattenModuleTree(moduleTree); + + // Process all leaf modules in parallel + pagesGenerated += await this.runParallel(leaves, async (node) => { + const pagePath = path.join(this.wikiDir, `${node.slug}.md`); + if (await this.fileExists(pagePath)) { + reportProgress(node.name); + return 0; + } + try { + await this.generateLeafPage(node); + reportProgress(node.name); + return 1; + } catch (err: any) { + this.failedModules.push(node.name); + reportProgress(`Failed: ${node.name}`); + return 0; + } + }); + + // Process parent modules sequentially (they depend on child docs) + for (const node of parents) { + const pagePath = path.join(this.wikiDir, `${node.slug}.md`); + if (await this.fileExists(pagePath)) { + reportProgress(node.name); + continue; + } + try { + await this.generateParentPage(node); + pagesGenerated++; + reportProgress(node.name); + } catch (err: any) { + this.failedModules.push(node.name); + reportProgress(`Failed: ${node.name}`); + } } // Phase 3: Generate overview @@ -244,7 +330,10 @@ export class WikiGenerator { DIRECTORY_TREE: dirTree, }); - const response = await callLLM(prompt, this.llmConfig, GROUPING_SYSTEM_PROMPT); + const response = await callLLM( + prompt, this.llmConfig, GROUPING_SYSTEM_PROMPT, + this.streamOpts('Grouping files', 15), + ); const grouping = this.parseGroupingResponse(response.content, files); // Convert to tree nodes @@ -366,62 +455,6 @@ export class WikiGenerator { // ─── Phase 2: Generate Module Pages ───────────────────────────────── - /** - * Recursively generate pages for a module tree node. - * Returns count of pages generated. - */ - private async generateModulePage( - node: ModuleTreeNode, - onPageDone: () => void, - ): Promise { - let count = 0; - - // If node has children, generate children first (bottom-up) - if (node.children && node.children.length > 0) { - for (const child of node.children) { - count += await this.generateModulePage(child, onPageDone); - } - - // Then generate parent page from children docs - const pagePath = path.join(this.wikiDir, `${node.slug}.md`); - - // Resumability: skip if page already exists - if (await this.fileExists(pagePath)) { - onPageDone(); - return count; - } - - try { - await this.generateParentPage(node); - count++; - } catch (err: any) { - this.failedModules.push(node.name); - this.onProgress('modules', 0, `Failed: ${node.name} — ${err.message?.slice(0, 80)}`); - } - onPageDone(); - return count; - } - - // Leaf module — generate from source code - const pagePath = path.join(this.wikiDir, `${node.slug}.md`); - - // Resumability: skip if page already exists - if (await this.fileExists(pagePath)) { - onPageDone(); - return count; - } - - try { - await this.generateLeafPage(node); - count++; - } catch (err: any) { - this.failedModules.push(node.name); - this.onProgress('modules', 0, `Failed: ${node.name} — ${err.message?.slice(0, 80)}`); - } - onPageDone(); - return count; - } - /** * Generate a leaf module page from source code + graph data. */ @@ -454,7 +487,10 @@ export class WikiGenerator { PROCESSES: formatProcesses(processes), }); - const response = await callLLM(prompt, this.llmConfig, MODULE_SYSTEM_PROMPT); + const response = await callLLM( + prompt, this.llmConfig, MODULE_SYSTEM_PROMPT, + this.streamOpts(node.name), + ); // Write page with front matter const pageContent = `# ${node.name}\n\n${response.content}`; @@ -494,7 +530,10 @@ export class WikiGenerator { CROSS_PROCESSES: formatProcesses(processes), }); - const response = await callLLM(prompt, this.llmConfig, PARENT_SYSTEM_PROMPT); + const response = await callLLM( + prompt, this.llmConfig, PARENT_SYSTEM_PROMPT, + this.streamOpts(node.name), + ); const pageContent = `# ${node.name}\n\n${response.content}`; await fs.writeFile(path.join(this.wikiDir, `${node.slug}.md`), pageContent, 'utf-8'); @@ -538,7 +577,10 @@ export class WikiGenerator { TOP_PROCESSES: formatProcesses(topProcesses), }); - const response = await callLLM(prompt, this.llmConfig, OVERVIEW_SYSTEM_PROMPT); + const response = await callLLM( + prompt, this.llmConfig, OVERVIEW_SYSTEM_PROMPT, + this.streamOpts('Generating overview', 88), + ); const pageContent = `# ${path.basename(this.repoPath)} — Wiki\n\n${response.content}`; await fs.writeFile(path.join(this.wikiDir, 'overview.md'), pageContent, 'utf-8'); @@ -602,29 +644,42 @@ export class WikiGenerator { affectedModules.add('Other'); } - // Regenerate affected module pages + // Regenerate affected module pages (parallel) let pagesGenerated = 0; const moduleTree = existingMeta.moduleTree; const affectedArray = Array.from(affectedModules); this.onProgress('incremental', 20, `Regenerating ${affectedArray.length} module(s)...`); - for (let i = 0; i < affectedArray.length; i++) { - const modSlug = this.slugify(affectedArray[i]); + const affectedNodes: ModuleTreeNode[] = []; + for (const mod of affectedArray) { + const modSlug = this.slugify(mod); const node = this.findNodeBySlug(moduleTree, modSlug); - if (node) { - // Delete existing page to force re-generation try { await fs.unlink(path.join(this.wikiDir, `${node.slug}.md`)); } catch {} - - await this.generateModulePage(node, () => {}); - pagesGenerated++; + affectedNodes.push(node); } - - const percent = 20 + Math.round(((i + 1) / affectedArray.length) * 60); - this.onProgress('incremental', percent, `${i + 1}/${affectedArray.length} modules`); } + let incProcessed = 0; + pagesGenerated += await this.runParallel(affectedNodes, async (node) => { + try { + if (node.children && node.children.length > 0) { + await this.generateParentPage(node); + } else { + await this.generateLeafPage(node); + } + incProcessed++; + const percent = 20 + Math.round((incProcessed / affectedNodes.length) * 60); + this.onProgress('incremental', percent, `${incProcessed}/${affectedNodes.length} — ${node.name}`); + return 1; + } catch (err: any) { + this.failedModules.push(node.name); + incProcessed++; + return 0; + } + }); + // Regenerate overview if any pages changed if (pagesGenerated > 0) { this.onProgress('incremental', 85, 'Updating overview...'); @@ -764,6 +819,85 @@ export class WikiGenerator { return count; } + /** + * Flatten the module tree into leaf nodes and parent nodes. + * Leaves can be processed in parallel; parents must wait for children. + */ + private flattenModuleTree(tree: ModuleTreeNode[]): { leaves: ModuleTreeNode[]; parents: ModuleTreeNode[] } { + const leaves: ModuleTreeNode[] = []; + const parents: ModuleTreeNode[] = []; + + for (const node of tree) { + if (node.children && node.children.length > 0) { + for (const child of node.children) { + leaves.push(child); + } + parents.push(node); + } else { + leaves.push(node); + } + } + + return { leaves, parents }; + } + + /** + * Run async tasks in parallel with a concurrency limit and adaptive rate limiting. + * If a 429 rate limit is hit, concurrency is temporarily reduced. + */ + private async runParallel( + items: T[], + fn: (item: T) => Promise, + ): Promise { + let total = 0; + let activeConcurrency = this.concurrency; + let running = 0; + let idx = 0; + + return new Promise((resolve, reject) => { + const next = () => { + while (running < activeConcurrency && idx < items.length) { + const item = items[idx++]; + running++; + + fn(item) + .then((count) => { + total += count; + running--; + if (idx >= items.length && running === 0) { + resolve(total); + } else { + next(); + } + }) + .catch((err) => { + running--; + // On rate limit, reduce concurrency temporarily + if (err.message?.includes('429')) { + activeConcurrency = Math.max(1, activeConcurrency - 1); + this.onProgress('modules', this.lastPercent, `Rate limited — concurrency → ${activeConcurrency}`); + // Re-queue the item + idx--; + setTimeout(next, 5000); + } else { + if (idx >= items.length && running === 0) { + resolve(total); + } else { + next(); + } + } + }); + } + }; + + if (items.length === 0) { + resolve(0); + } else { + next(); + } + }); + } + private findNodeBySlug(tree: ModuleTreeNode[], slug: string): ModuleTreeNode | null { for (const node of tree) { if (node.slug === slug) return node; diff --git a/gitnexus/src/core/wiki/html-viewer.ts b/gitnexus/src/core/wiki/html-viewer.ts new file mode 100644 index 0000000000..e0f36a0736 --- /dev/null +++ b/gitnexus/src/core/wiki/html-viewer.ts @@ -0,0 +1,331 @@ +/** + * HTML Viewer Generator for Wiki + * + * Produces a self-contained index.html that embeds all markdown pages, + * module tree, and metadata — viewable offline in any browser. + */ + +import fs from 'fs/promises'; +import path from 'path'; + +interface ModuleTreeNode { + name: string; + slug: string; + files: string[]; + children?: ModuleTreeNode[]; +} + +/** + * Generate the wiki HTML viewer (index.html) from existing markdown pages. + */ +export async function generateHTMLViewer( + wikiDir: string, + projectName: string, +): Promise { + // Load module tree + let moduleTree: ModuleTreeNode[] = []; + try { + const raw = await fs.readFile(path.join(wikiDir, 'module_tree.json'), 'utf-8'); + moduleTree = JSON.parse(raw); + } catch { /* will show empty nav */ } + + // Load meta + let meta: Record | null = null; + try { + const raw = await fs.readFile(path.join(wikiDir, 'meta.json'), 'utf-8'); + meta = JSON.parse(raw); + } catch { /* no meta */ } + + // Read all markdown files into a { slug: content } map + const pages: Record = {}; + const dirEntries = await fs.readdir(wikiDir); + for (const f of dirEntries.filter(f => f.endsWith('.md'))) { + const content = await fs.readFile(path.join(wikiDir, f), 'utf-8'); + pages[f.replace(/\.md$/, '')] = content; + } + + const html = buildHTML(projectName, moduleTree, pages, meta); + const outputPath = path.join(wikiDir, 'index.html'); + await fs.writeFile(outputPath, html, 'utf-8'); + return outputPath; +} + +// ─── HTML Builder ─────────────────────────────────────────────────────── + +function esc(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +function buildHTML( + projectName: string, + moduleTree: ModuleTreeNode[], + pages: Record, + meta: Record | null, +): string { + // Embed data as JSON inside the HTML + const pagesJSON = JSON.stringify(pages); + const treeJSON = JSON.stringify(moduleTree); + const metaJSON = JSON.stringify(meta); + + const parts: string[] = []; + + // ── Head ── + parts.push(''); + parts.push(''); + parts.push(''); + parts.push(''); + parts.push(''); + parts.push('' + esc(projectName) + ' — Wiki'); + parts.push('