mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-29 06:32:28 +00:00
fixed MCP concurrency issue with connection pool on kuzu
This commit is contained in:
abhigyanpatwari
parent
ccf38129e6
commit
42473d67df
7 changed files with 954 additions and 159 deletions
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "gitnexus",
|
||||
"version": "1.1.9",
|
||||
"version": "1.2.6",
|
||||
"description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
|
||||
"author": "Abhigyan Patwari",
|
||||
"license": "PolyForm-Noncommercial-1.0.0",
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ const program = new Command();
|
|||
program
|
||||
.name('gitnexus')
|
||||
.description('GitNexus local CLI and MCP server')
|
||||
.version('1.1.9');
|
||||
.version('1.2.0');
|
||||
|
||||
program
|
||||
.command('setup')
|
||||
|
|
@ -62,9 +62,11 @@ program
|
|||
.command('wiki [path]')
|
||||
.description('Generate repository wiki from knowledge graph')
|
||||
.option('-f, --force', 'Force full regeneration even if up to date')
|
||||
.option('--model <model>', 'LLM model name (default: gpt-4o-mini)')
|
||||
.option('--model <model>', 'LLM model name (default: minimax/minimax-m2.5)')
|
||||
.option('--base-url <url>', 'LLM API base URL (default: OpenAI)')
|
||||
.option('--api-key <key>', 'LLM API key (saved to ~/.gitnexus/config.json)')
|
||||
.option('--concurrency <n>', 'Parallel LLM calls (default: 3)', '3')
|
||||
.option('--gist', 'Publish wiki as a public GitHub Gist after generation')
|
||||
.action(wikiCommand);
|
||||
|
||||
program
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
import path from 'path';
|
||||
import readline from 'readline';
|
||||
import { execSync } from 'child_process';
|
||||
import cliProgress from 'cli-progress';
|
||||
import { getGitRoot, isGitRepo } from '../storage/git.js';
|
||||
import { getStoragePaths, loadMeta, loadCLIConfig, saveCLIConfig } from '../storage/repo-manager.js';
|
||||
|
|
@ -18,6 +19,8 @@ export interface WikiCommandOptions {
|
|||
model?: string;
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
concurrency?: string;
|
||||
gist?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -109,52 +112,103 @@ export const wikiCommand = async (
|
|||
}
|
||||
|
||||
// ── Resolve LLM config (with interactive fallback) ─────────────────
|
||||
// If --api-key was passed via CLI, save it immediately
|
||||
if (options?.apiKey) {
|
||||
// Save any CLI overrides immediately
|
||||
if (options?.apiKey || options?.model || options?.baseUrl) {
|
||||
const existing = await loadCLIConfig();
|
||||
await saveCLIConfig({ ...existing, apiKey: options.apiKey });
|
||||
console.log(' API key saved to ~/.gitnexus/config.json\n');
|
||||
const updates: Record<string, string> = {};
|
||||
if (options.apiKey) updates.apiKey = options.apiKey;
|
||||
if (options.model) updates.model = options.model;
|
||||
if (options.baseUrl) updates.baseUrl = options.baseUrl;
|
||||
await saveCLIConfig({ ...existing, ...updates });
|
||||
console.log(' Config saved to ~/.gitnexus/config.json\n');
|
||||
}
|
||||
|
||||
const savedConfig = await loadCLIConfig();
|
||||
const hasSavedConfig = !!(savedConfig.apiKey && savedConfig.baseUrl);
|
||||
const hasCLIOverrides = !!(options?.apiKey || options?.model || options?.baseUrl);
|
||||
|
||||
let llmConfig = await resolveLLMConfig({
|
||||
model: options?.model,
|
||||
baseUrl: options?.baseUrl,
|
||||
apiKey: options?.apiKey,
|
||||
});
|
||||
|
||||
if (!llmConfig.apiKey) {
|
||||
// Run interactive setup if no saved config and no CLI flags provided
|
||||
// (even if env vars exist — let user explicitly choose their provider)
|
||||
if (!hasSavedConfig && !hasCLIOverrides) {
|
||||
if (!process.stdin.isTTY) {
|
||||
console.log(' Error: No LLM API key found.');
|
||||
console.log(' Set OPENAI_API_KEY or GITNEXUS_API_KEY environment variable,');
|
||||
console.log(' or pass --api-key <key>.\n');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(' No API key configured.\n');
|
||||
console.log(' The wiki command requires an LLM API key (OpenAI-compatible).');
|
||||
console.log(' You can also set OPENAI_API_KEY or GITNEXUS_API_KEY env var.\n');
|
||||
|
||||
const key = await prompt(' Enter your API key: ', true);
|
||||
if (!key) {
|
||||
console.log('\n No key provided. Aborting.\n');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const save = await prompt(' Save key to ~/.gitnexus/config.json for future use? (Y/n): ');
|
||||
if (!save || save.toLowerCase() === 'y' || save.toLowerCase() === 'yes') {
|
||||
const existing = await loadCLIConfig();
|
||||
await saveCLIConfig({ ...existing, apiKey: key });
|
||||
console.log(' Key saved.\n');
|
||||
if (!llmConfig.apiKey) {
|
||||
console.log(' Error: No LLM API key found.');
|
||||
console.log(' Set OPENAI_API_KEY or GITNEXUS_API_KEY environment variable,');
|
||||
console.log(' or pass --api-key <key>.\n');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
// Non-interactive with env var — just use it
|
||||
} else {
|
||||
console.log(' Key will be used for this session only.\n');
|
||||
}
|
||||
console.log(' No LLM configured. Let\'s set it up.\n');
|
||||
console.log(' Supports OpenAI, OpenRouter, or any OpenAI-compatible API.\n');
|
||||
|
||||
llmConfig = { ...llmConfig, apiKey: key };
|
||||
// Provider selection
|
||||
console.log(' [1] OpenAI (api.openai.com)');
|
||||
console.log(' [2] OpenRouter (openrouter.ai)');
|
||||
console.log(' [3] Custom endpoint\n');
|
||||
|
||||
const choice = await prompt(' Select provider (1/2/3): ');
|
||||
|
||||
let baseUrl: string;
|
||||
let defaultModel: string;
|
||||
|
||||
if (choice === '2') {
|
||||
baseUrl = 'https://openrouter.ai/api/v1';
|
||||
defaultModel = 'minimax/minimax-m2.5';
|
||||
} else if (choice === '3') {
|
||||
baseUrl = await prompt(' Base URL (e.g. http://localhost:11434/v1): ');
|
||||
if (!baseUrl) {
|
||||
console.log('\n No URL provided. Aborting.\n');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
defaultModel = 'gpt-4o-mini';
|
||||
} else {
|
||||
baseUrl = 'https://api.openai.com/v1';
|
||||
defaultModel = 'gpt-4o-mini';
|
||||
}
|
||||
|
||||
// Model
|
||||
const modelInput = await prompt(` Model (default: ${defaultModel}): `);
|
||||
const model = modelInput || defaultModel;
|
||||
|
||||
// API key — pre-fill hint if env var exists
|
||||
const envKey = process.env.GITNEXUS_API_KEY || process.env.OPENAI_API_KEY || '';
|
||||
let key: string;
|
||||
if (envKey) {
|
||||
const masked = envKey.slice(0, 6) + '...' + envKey.slice(-4);
|
||||
const useEnv = await prompt(` Use existing env key (${masked})? (Y/n): `);
|
||||
if (!useEnv || useEnv.toLowerCase() === 'y' || useEnv.toLowerCase() === 'yes') {
|
||||
key = envKey;
|
||||
} else {
|
||||
key = await prompt(' API key: ', true);
|
||||
}
|
||||
} else {
|
||||
key = await prompt(' API key: ', true);
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
console.log('\n No key provided. Aborting.\n');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// Save
|
||||
await saveCLIConfig({ apiKey: key, baseUrl, model });
|
||||
console.log(' Config saved to ~/.gitnexus/config.json\n');
|
||||
|
||||
llmConfig = { ...llmConfig, apiKey: key, baseUrl, model };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Setup progress bar ──────────────────────────────────────────────
|
||||
// ── Setup progress bar with elapsed timer ──────────────────────────
|
||||
const bar = new cliProgress.SingleBar({
|
||||
format: ' {bar} {percentage}% | {phase}',
|
||||
barCompleteChar: '\u2588',
|
||||
|
|
@ -169,12 +223,25 @@ export const wikiCommand = async (
|
|||
bar.start(100, 0, { phase: 'Initializing...' });
|
||||
|
||||
const t0 = Date.now();
|
||||
let lastPhase = '';
|
||||
let phaseStart = t0;
|
||||
|
||||
// Tick elapsed time every second while stuck on the same phase
|
||||
const elapsedTimer = setInterval(() => {
|
||||
if (lastPhase) {
|
||||
const elapsed = Math.round((Date.now() - phaseStart) / 1000);
|
||||
if (elapsed >= 3) {
|
||||
bar.update({ phase: `${lastPhase} (${elapsed}s)` });
|
||||
}
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
// ── Run generator ───────────────────────────────────────────────────
|
||||
const wikiOptions: WikiOptions = {
|
||||
force: options?.force,
|
||||
model: options?.model,
|
||||
baseUrl: options?.baseUrl,
|
||||
concurrency: options?.concurrency ? parseInt(options.concurrency, 10) : undefined,
|
||||
};
|
||||
|
||||
const generator = new WikiGenerator(
|
||||
|
|
@ -184,28 +251,39 @@ export const wikiCommand = async (
|
|||
llmConfig,
|
||||
wikiOptions,
|
||||
(phase, percent, detail) => {
|
||||
bar.update(percent, { phase: detail || phase });
|
||||
const label = detail || phase;
|
||||
if (label !== lastPhase) {
|
||||
lastPhase = label;
|
||||
phaseStart = Date.now();
|
||||
}
|
||||
bar.update(percent, { phase: label });
|
||||
},
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await generator.run();
|
||||
|
||||
clearInterval(elapsedTimer);
|
||||
bar.update(100, { phase: 'Done' });
|
||||
bar.stop();
|
||||
|
||||
const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
|
||||
|
||||
const wikiDir = path.join(storagePath, 'wiki');
|
||||
const viewerPath = path.join(wikiDir, 'index.html');
|
||||
|
||||
if (result.mode === 'up-to-date' && !options?.force) {
|
||||
console.log('\n Wiki is already up to date.');
|
||||
console.log(` ${path.join(storagePath, 'wiki')}\n`);
|
||||
console.log(` Viewer: ${viewerPath}\n`);
|
||||
await maybePublishGist(viewerPath, options?.gist);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`\n Wiki generated successfully (${elapsed}s)\n`);
|
||||
console.log(` Mode: ${result.mode}`);
|
||||
console.log(` Pages: ${result.pagesGenerated}`);
|
||||
console.log(` Output: ${path.join(storagePath, 'wiki')}`);
|
||||
console.log(` Output: ${wikiDir}`);
|
||||
console.log(` Viewer: ${viewerPath}`);
|
||||
|
||||
if (result.failedModules && result.failedModules.length > 0) {
|
||||
console.log(`\n Failed modules (${result.failedModules.length}):`);
|
||||
|
|
@ -216,13 +294,32 @@ export const wikiCommand = async (
|
|||
}
|
||||
|
||||
console.log('');
|
||||
|
||||
await maybePublishGist(viewerPath, options?.gist);
|
||||
} catch (err: any) {
|
||||
clearInterval(elapsedTimer);
|
||||
bar.stop();
|
||||
|
||||
if (err.message?.includes('No source files')) {
|
||||
console.log(`\n ${err.message}\n`);
|
||||
} else if (err.message?.includes('API key') || err.message?.includes('API error')) {
|
||||
console.log(`\n LLM Error: ${err.message}\n`);
|
||||
|
||||
// Offer to reconfigure on auth-related failures
|
||||
const isAuthError = err.message?.includes('401') || err.message?.includes('403')
|
||||
|| err.message?.includes('502') || err.message?.includes('authenticate')
|
||||
|| err.message?.includes('Unauthorized');
|
||||
if (isAuthError && process.stdin.isTTY) {
|
||||
const answer = await new Promise<string>((resolve) => {
|
||||
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
||||
rl.question(' Reconfigure LLM settings? (Y/n): ', (ans) => { rl.close(); resolve(ans.trim().toLowerCase()); });
|
||||
});
|
||||
if (!answer || answer === 'y' || answer === 'yes') {
|
||||
// Clear saved config so next run triggers interactive setup
|
||||
await saveCLIConfig({});
|
||||
console.log(' Config cleared. Run `gitnexus wiki` again to reconfigure.\n');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log(`\n Error: ${err.message}\n`);
|
||||
if (process.env.DEBUG) {
|
||||
|
|
@ -232,3 +329,86 @@ export const wikiCommand = async (
|
|||
process.exitCode = 1;
|
||||
}
|
||||
};
|
||||
|
||||
// ─── Gist Publishing ───────────────────────────────────────────────────
|
||||
|
||||
function hasGhCLI(): boolean {
|
||||
try {
|
||||
execSync('gh --version', { stdio: 'ignore' });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function publishGist(htmlPath: string): { url: string; rawUrl: string } | null {
|
||||
try {
|
||||
const output = execSync(
|
||||
`gh gist create "${htmlPath}" --desc "Repository Wiki — generated by GitNexus" --public`,
|
||||
{ encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
).trim();
|
||||
|
||||
// gh gist create prints the gist URL as the last line
|
||||
const lines = output.split('\n');
|
||||
const gistUrl = lines.find(l => l.includes('gist.github.com')) || lines[lines.length - 1];
|
||||
|
||||
if (!gistUrl || !gistUrl.includes('gist.github.com')) return null;
|
||||
|
||||
// Build a raw viewer URL via gist.githack.com
|
||||
// gist URL format: https://gist.github.com/{user}/{id}
|
||||
const match = gistUrl.match(/gist\.github\.com\/([^/]+)\/([a-f0-9]+)/);
|
||||
let rawUrl = gistUrl;
|
||||
if (match) {
|
||||
rawUrl = `https://gistcdn.githack.com/${match[1]}/${match[2]}/raw/index.html`;
|
||||
}
|
||||
|
||||
return { url: gistUrl.trim(), rawUrl };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function maybePublishGist(htmlPath: string, gistFlag?: boolean): Promise<void> {
|
||||
if (gistFlag === false) return;
|
||||
|
||||
// Check that the HTML file exists
|
||||
try {
|
||||
const fs = await import('fs/promises');
|
||||
await fs.access(htmlPath);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!hasGhCLI()) {
|
||||
if (gistFlag) {
|
||||
console.log(' GitHub CLI (gh) is not installed. Cannot publish gist.');
|
||||
console.log(' Install it: https://cli.github.com\n');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let shouldPublish = !!gistFlag;
|
||||
|
||||
if (!shouldPublish && process.stdin.isTTY) {
|
||||
const answer = await new Promise<string>((resolve) => {
|
||||
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
||||
rl.question(' Publish wiki as a GitHub Gist for easy viewing? (Y/n): ', (ans) => {
|
||||
rl.close();
|
||||
resolve(ans.trim().toLowerCase());
|
||||
});
|
||||
});
|
||||
shouldPublish = !answer || answer === 'y' || answer === 'yes';
|
||||
}
|
||||
|
||||
if (!shouldPublish) return;
|
||||
|
||||
console.log('\n Publishing to GitHub Gist...');
|
||||
const result = publishGist(htmlPath);
|
||||
|
||||
if (result) {
|
||||
console.log(` Gist: ${result.url}`);
|
||||
console.log(` Viewer: ${result.rawUrl}\n`);
|
||||
} else {
|
||||
console.log(' Failed to publish gist. Make sure `gh auth login` is configured.\n');
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,11 +27,13 @@ import {
|
|||
getInterModuleEdgesForOverview,
|
||||
type FileWithExports,
|
||||
} from './graph-queries.js';
|
||||
import { generateHTMLViewer } from './html-viewer.js';
|
||||
|
||||
import {
|
||||
callLLM,
|
||||
estimateTokens,
|
||||
type LLMConfig,
|
||||
type CallLLMOptions,
|
||||
} from './llm-client.js';
|
||||
|
||||
import {
|
||||
|
|
@ -60,6 +62,7 @@ export interface WikiOptions {
|
|||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
maxTokensPerModule?: number;
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
export interface WikiMeta {
|
||||
|
|
@ -93,6 +96,7 @@ export class WikiGenerator {
|
|||
private kuzuPath: string;
|
||||
private llmConfig: LLMConfig;
|
||||
private maxTokensPerModule: number;
|
||||
private concurrency: number;
|
||||
private options: WikiOptions;
|
||||
private onProgress: ProgressCallback;
|
||||
private failedModules: string[] = [];
|
||||
|
|
@ -112,7 +116,28 @@ export class WikiGenerator {
|
|||
this.options = options;
|
||||
this.llmConfig = llmConfig;
|
||||
this.maxTokensPerModule = options.maxTokensPerModule ?? DEFAULT_MAX_TOKENS_PER_MODULE;
|
||||
this.onProgress = onProgress || (() => {});
|
||||
this.concurrency = options.concurrency ?? 3;
|
||||
const progressFn = onProgress || (() => {});
|
||||
this.onProgress = (phase, percent, detail) => {
|
||||
if (percent > 0) this.lastPercent = percent;
|
||||
progressFn(phase, percent, detail);
|
||||
};
|
||||
}
|
||||
|
||||
private lastPercent = 0;
|
||||
|
||||
/**
|
||||
* Create streaming options that report LLM progress to the progress bar.
|
||||
* Uses the last known percent so streaming doesn't reset the bar backwards.
|
||||
*/
|
||||
private streamOpts(label: string, fixedPercent?: number): CallLLMOptions {
|
||||
return {
|
||||
onChunk: (chars: number) => {
|
||||
const tokens = Math.round(chars / 4);
|
||||
const pct = fixedPercent ?? this.lastPercent;
|
||||
this.onProgress('stream', pct, `${label} (${tokens} tok)`);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -127,6 +152,8 @@ export class WikiGenerator {
|
|||
|
||||
// Up-to-date check (skip if --force)
|
||||
if (!forceMode && existingMeta && existingMeta.fromCommit === currentCommit) {
|
||||
// Still regenerate the HTML viewer in case it's missing
|
||||
await this.ensureHTMLViewer();
|
||||
return { pagesGenerated: 0, mode: 'up-to-date', failedModules: [] };
|
||||
}
|
||||
|
||||
|
|
@ -146,14 +173,34 @@ export class WikiGenerator {
|
|||
this.onProgress('init', 2, 'Connecting to knowledge graph...');
|
||||
await initWikiDb(this.kuzuPath);
|
||||
|
||||
let result: { pagesGenerated: number; mode: 'full' | 'incremental' | 'up-to-date'; failedModules: string[] };
|
||||
try {
|
||||
if (!forceMode && existingMeta && existingMeta.fromCommit) {
|
||||
return await this.incrementalUpdate(existingMeta, currentCommit);
|
||||
result = await this.incrementalUpdate(existingMeta, currentCommit);
|
||||
} else {
|
||||
result = await this.fullGeneration(currentCommit);
|
||||
}
|
||||
return await this.fullGeneration(currentCommit);
|
||||
} finally {
|
||||
await closeWikiDb();
|
||||
}
|
||||
|
||||
// Always generate the HTML viewer after wiki content changes
|
||||
await this.ensureHTMLViewer();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ─── HTML Viewer ─────────────────────────────────────────────────────
|
||||
|
||||
private async ensureHTMLViewer(): Promise<void> {
|
||||
// Only generate if there are markdown pages to bundle
|
||||
const dirEntries = await fs.readdir(this.wikiDir).catch(() => [] as string[]);
|
||||
const hasMd = dirEntries.some(f => f.endsWith('.md'));
|
||||
if (!hasMd) return;
|
||||
|
||||
this.onProgress('html', 98, 'Building HTML viewer...');
|
||||
const repoName = path.basename(this.repoPath);
|
||||
await generateHTMLViewer(this.wikiDir, repoName);
|
||||
}
|
||||
|
||||
// ─── Full Generation ────────────────────────────────────────────────
|
||||
|
|
@ -184,17 +231,56 @@ export class WikiGenerator {
|
|||
const moduleTree = await this.buildModuleTree(enrichedFiles);
|
||||
pagesGenerated = 0;
|
||||
|
||||
// Phase 2: Generate module pages (bottom-up)
|
||||
// Phase 2: Generate module pages (parallel with concurrency limit)
|
||||
const totalModules = this.countModules(moduleTree);
|
||||
let modulesProcessed = 0;
|
||||
|
||||
for (const node of moduleTree) {
|
||||
const generated = await this.generateModulePage(node, () => {
|
||||
modulesProcessed++;
|
||||
const percent = 30 + Math.round((modulesProcessed / totalModules) * 55);
|
||||
this.onProgress('modules', percent, `${modulesProcessed}/${totalModules} modules`);
|
||||
});
|
||||
pagesGenerated += generated;
|
||||
const reportProgress = (moduleName?: string) => {
|
||||
modulesProcessed++;
|
||||
const percent = 30 + Math.round((modulesProcessed / totalModules) * 55);
|
||||
const detail = moduleName
|
||||
? `${modulesProcessed}/${totalModules} — ${moduleName}`
|
||||
: `${modulesProcessed}/${totalModules} modules`;
|
||||
this.onProgress('modules', percent, detail);
|
||||
};
|
||||
|
||||
// Flatten tree into layers: leaves first, then parents
|
||||
// Leaves can run in parallel; parents must wait for their children
|
||||
const { leaves, parents } = this.flattenModuleTree(moduleTree);
|
||||
|
||||
// Process all leaf modules in parallel
|
||||
pagesGenerated += await this.runParallel(leaves, async (node) => {
|
||||
const pagePath = path.join(this.wikiDir, `${node.slug}.md`);
|
||||
if (await this.fileExists(pagePath)) {
|
||||
reportProgress(node.name);
|
||||
return 0;
|
||||
}
|
||||
try {
|
||||
await this.generateLeafPage(node);
|
||||
reportProgress(node.name);
|
||||
return 1;
|
||||
} catch (err: any) {
|
||||
this.failedModules.push(node.name);
|
||||
reportProgress(`Failed: ${node.name}`);
|
||||
return 0;
|
||||
}
|
||||
});
|
||||
|
||||
// Process parent modules sequentially (they depend on child docs)
|
||||
for (const node of parents) {
|
||||
const pagePath = path.join(this.wikiDir, `${node.slug}.md`);
|
||||
if (await this.fileExists(pagePath)) {
|
||||
reportProgress(node.name);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await this.generateParentPage(node);
|
||||
pagesGenerated++;
|
||||
reportProgress(node.name);
|
||||
} catch (err: any) {
|
||||
this.failedModules.push(node.name);
|
||||
reportProgress(`Failed: ${node.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 3: Generate overview
|
||||
|
|
@ -244,7 +330,10 @@ export class WikiGenerator {
|
|||
DIRECTORY_TREE: dirTree,
|
||||
});
|
||||
|
||||
const response = await callLLM(prompt, this.llmConfig, GROUPING_SYSTEM_PROMPT);
|
||||
const response = await callLLM(
|
||||
prompt, this.llmConfig, GROUPING_SYSTEM_PROMPT,
|
||||
this.streamOpts('Grouping files', 15),
|
||||
);
|
||||
const grouping = this.parseGroupingResponse(response.content, files);
|
||||
|
||||
// Convert to tree nodes
|
||||
|
|
@ -366,62 +455,6 @@ export class WikiGenerator {
|
|||
|
||||
// ─── Phase 2: Generate Module Pages ─────────────────────────────────
|
||||
|
||||
/**
|
||||
* Recursively generate pages for a module tree node.
|
||||
* Returns count of pages generated.
|
||||
*/
|
||||
private async generateModulePage(
|
||||
node: ModuleTreeNode,
|
||||
onPageDone: () => void,
|
||||
): Promise<number> {
|
||||
let count = 0;
|
||||
|
||||
// If node has children, generate children first (bottom-up)
|
||||
if (node.children && node.children.length > 0) {
|
||||
for (const child of node.children) {
|
||||
count += await this.generateModulePage(child, onPageDone);
|
||||
}
|
||||
|
||||
// Then generate parent page from children docs
|
||||
const pagePath = path.join(this.wikiDir, `${node.slug}.md`);
|
||||
|
||||
// Resumability: skip if page already exists
|
||||
if (await this.fileExists(pagePath)) {
|
||||
onPageDone();
|
||||
return count;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.generateParentPage(node);
|
||||
count++;
|
||||
} catch (err: any) {
|
||||
this.failedModules.push(node.name);
|
||||
this.onProgress('modules', 0, `Failed: ${node.name} — ${err.message?.slice(0, 80)}`);
|
||||
}
|
||||
onPageDone();
|
||||
return count;
|
||||
}
|
||||
|
||||
// Leaf module — generate from source code
|
||||
const pagePath = path.join(this.wikiDir, `${node.slug}.md`);
|
||||
|
||||
// Resumability: skip if page already exists
|
||||
if (await this.fileExists(pagePath)) {
|
||||
onPageDone();
|
||||
return count;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.generateLeafPage(node);
|
||||
count++;
|
||||
} catch (err: any) {
|
||||
this.failedModules.push(node.name);
|
||||
this.onProgress('modules', 0, `Failed: ${node.name} — ${err.message?.slice(0, 80)}`);
|
||||
}
|
||||
onPageDone();
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a leaf module page from source code + graph data.
|
||||
*/
|
||||
|
|
@ -454,7 +487,10 @@ export class WikiGenerator {
|
|||
PROCESSES: formatProcesses(processes),
|
||||
});
|
||||
|
||||
const response = await callLLM(prompt, this.llmConfig, MODULE_SYSTEM_PROMPT);
|
||||
const response = await callLLM(
|
||||
prompt, this.llmConfig, MODULE_SYSTEM_PROMPT,
|
||||
this.streamOpts(node.name),
|
||||
);
|
||||
|
||||
// Write page with front matter
|
||||
const pageContent = `# ${node.name}\n\n${response.content}`;
|
||||
|
|
@ -494,7 +530,10 @@ export class WikiGenerator {
|
|||
CROSS_PROCESSES: formatProcesses(processes),
|
||||
});
|
||||
|
||||
const response = await callLLM(prompt, this.llmConfig, PARENT_SYSTEM_PROMPT);
|
||||
const response = await callLLM(
|
||||
prompt, this.llmConfig, PARENT_SYSTEM_PROMPT,
|
||||
this.streamOpts(node.name),
|
||||
);
|
||||
|
||||
const pageContent = `# ${node.name}\n\n${response.content}`;
|
||||
await fs.writeFile(path.join(this.wikiDir, `${node.slug}.md`), pageContent, 'utf-8');
|
||||
|
|
@ -538,7 +577,10 @@ export class WikiGenerator {
|
|||
TOP_PROCESSES: formatProcesses(topProcesses),
|
||||
});
|
||||
|
||||
const response = await callLLM(prompt, this.llmConfig, OVERVIEW_SYSTEM_PROMPT);
|
||||
const response = await callLLM(
|
||||
prompt, this.llmConfig, OVERVIEW_SYSTEM_PROMPT,
|
||||
this.streamOpts('Generating overview', 88),
|
||||
);
|
||||
|
||||
const pageContent = `# ${path.basename(this.repoPath)} — Wiki\n\n${response.content}`;
|
||||
await fs.writeFile(path.join(this.wikiDir, 'overview.md'), pageContent, 'utf-8');
|
||||
|
|
@ -602,29 +644,42 @@ export class WikiGenerator {
|
|||
affectedModules.add('Other');
|
||||
}
|
||||
|
||||
// Regenerate affected module pages
|
||||
// Regenerate affected module pages (parallel)
|
||||
let pagesGenerated = 0;
|
||||
const moduleTree = existingMeta.moduleTree;
|
||||
const affectedArray = Array.from(affectedModules);
|
||||
|
||||
this.onProgress('incremental', 20, `Regenerating ${affectedArray.length} module(s)...`);
|
||||
|
||||
for (let i = 0; i < affectedArray.length; i++) {
|
||||
const modSlug = this.slugify(affectedArray[i]);
|
||||
const affectedNodes: ModuleTreeNode[] = [];
|
||||
for (const mod of affectedArray) {
|
||||
const modSlug = this.slugify(mod);
|
||||
const node = this.findNodeBySlug(moduleTree, modSlug);
|
||||
|
||||
if (node) {
|
||||
// Delete existing page to force re-generation
|
||||
try { await fs.unlink(path.join(this.wikiDir, `${node.slug}.md`)); } catch {}
|
||||
|
||||
await this.generateModulePage(node, () => {});
|
||||
pagesGenerated++;
|
||||
affectedNodes.push(node);
|
||||
}
|
||||
|
||||
const percent = 20 + Math.round(((i + 1) / affectedArray.length) * 60);
|
||||
this.onProgress('incremental', percent, `${i + 1}/${affectedArray.length} modules`);
|
||||
}
|
||||
|
||||
let incProcessed = 0;
|
||||
pagesGenerated += await this.runParallel(affectedNodes, async (node) => {
|
||||
try {
|
||||
if (node.children && node.children.length > 0) {
|
||||
await this.generateParentPage(node);
|
||||
} else {
|
||||
await this.generateLeafPage(node);
|
||||
}
|
||||
incProcessed++;
|
||||
const percent = 20 + Math.round((incProcessed / affectedNodes.length) * 60);
|
||||
this.onProgress('incremental', percent, `${incProcessed}/${affectedNodes.length} — ${node.name}`);
|
||||
return 1;
|
||||
} catch (err: any) {
|
||||
this.failedModules.push(node.name);
|
||||
incProcessed++;
|
||||
return 0;
|
||||
}
|
||||
});
|
||||
|
||||
// Regenerate overview if any pages changed
|
||||
if (pagesGenerated > 0) {
|
||||
this.onProgress('incremental', 85, 'Updating overview...');
|
||||
|
|
@ -764,6 +819,85 @@ export class WikiGenerator {
|
|||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flatten the module tree into leaf nodes and parent nodes.
|
||||
* Leaves can be processed in parallel; parents must wait for children.
|
||||
*/
|
||||
private flattenModuleTree(tree: ModuleTreeNode[]): { leaves: ModuleTreeNode[]; parents: ModuleTreeNode[] } {
|
||||
const leaves: ModuleTreeNode[] = [];
|
||||
const parents: ModuleTreeNode[] = [];
|
||||
|
||||
for (const node of tree) {
|
||||
if (node.children && node.children.length > 0) {
|
||||
for (const child of node.children) {
|
||||
leaves.push(child);
|
||||
}
|
||||
parents.push(node);
|
||||
} else {
|
||||
leaves.push(node);
|
||||
}
|
||||
}
|
||||
|
||||
return { leaves, parents };
|
||||
}
|
||||
|
||||
/**
|
||||
* Run async tasks in parallel with a concurrency limit and adaptive rate limiting.
|
||||
* If a 429 rate limit is hit, concurrency is temporarily reduced.
|
||||
*/
|
||||
private async runParallel<T>(
|
||||
items: T[],
|
||||
fn: (item: T) => Promise<number>,
|
||||
): Promise<number> {
|
||||
let total = 0;
|
||||
let activeConcurrency = this.concurrency;
|
||||
let running = 0;
|
||||
let idx = 0;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const next = () => {
|
||||
while (running < activeConcurrency && idx < items.length) {
|
||||
const item = items[idx++];
|
||||
running++;
|
||||
|
||||
fn(item)
|
||||
.then((count) => {
|
||||
total += count;
|
||||
running--;
|
||||
if (idx >= items.length && running === 0) {
|
||||
resolve(total);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
running--;
|
||||
// On rate limit, reduce concurrency temporarily
|
||||
if (err.message?.includes('429')) {
|
||||
activeConcurrency = Math.max(1, activeConcurrency - 1);
|
||||
this.onProgress('modules', this.lastPercent, `Rate limited — concurrency → ${activeConcurrency}`);
|
||||
// Re-queue the item
|
||||
idx--;
|
||||
setTimeout(next, 5000);
|
||||
} else {
|
||||
if (idx >= items.length && running === 0) {
|
||||
resolve(total);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
if (items.length === 0) {
|
||||
resolve(0);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private findNodeBySlug(tree: ModuleTreeNode[], slug: string): ModuleTreeNode | null {
|
||||
for (const node of tree) {
|
||||
if (node.slug === slug) return node;
|
||||
|
|
|
|||
331
gitnexus/src/core/wiki/html-viewer.ts
Normal file
331
gitnexus/src/core/wiki/html-viewer.ts
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
/**
|
||||
* HTML Viewer Generator for Wiki
|
||||
*
|
||||
* Produces a self-contained index.html that embeds all markdown pages,
|
||||
* module tree, and metadata — viewable offline in any browser.
|
||||
*/
|
||||
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
|
||||
interface ModuleTreeNode {
|
||||
name: string;
|
||||
slug: string;
|
||||
files: string[];
|
||||
children?: ModuleTreeNode[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the wiki HTML viewer (index.html) from existing markdown pages.
|
||||
*/
|
||||
export async function generateHTMLViewer(
|
||||
wikiDir: string,
|
||||
projectName: string,
|
||||
): Promise<string> {
|
||||
// Load module tree
|
||||
let moduleTree: ModuleTreeNode[] = [];
|
||||
try {
|
||||
const raw = await fs.readFile(path.join(wikiDir, 'module_tree.json'), 'utf-8');
|
||||
moduleTree = JSON.parse(raw);
|
||||
} catch { /* will show empty nav */ }
|
||||
|
||||
// Load meta
|
||||
let meta: Record<string, unknown> | null = null;
|
||||
try {
|
||||
const raw = await fs.readFile(path.join(wikiDir, 'meta.json'), 'utf-8');
|
||||
meta = JSON.parse(raw);
|
||||
} catch { /* no meta */ }
|
||||
|
||||
// Read all markdown files into a { slug: content } map
|
||||
const pages: Record<string, string> = {};
|
||||
const dirEntries = await fs.readdir(wikiDir);
|
||||
for (const f of dirEntries.filter(f => f.endsWith('.md'))) {
|
||||
const content = await fs.readFile(path.join(wikiDir, f), 'utf-8');
|
||||
pages[f.replace(/\.md$/, '')] = content;
|
||||
}
|
||||
|
||||
const html = buildHTML(projectName, moduleTree, pages, meta);
|
||||
const outputPath = path.join(wikiDir, 'index.html');
|
||||
await fs.writeFile(outputPath, html, 'utf-8');
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
// ─── HTML Builder ───────────────────────────────────────────────────────
|
||||
|
||||
function esc(text: string): string {
|
||||
return text
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"');
|
||||
}
|
||||
|
||||
function buildHTML(
|
||||
projectName: string,
|
||||
moduleTree: ModuleTreeNode[],
|
||||
pages: Record<string, string>,
|
||||
meta: Record<string, unknown> | null,
|
||||
): string {
|
||||
// Embed data as JSON inside the HTML
|
||||
const pagesJSON = JSON.stringify(pages);
|
||||
const treeJSON = JSON.stringify(moduleTree);
|
||||
const metaJSON = JSON.stringify(meta);
|
||||
|
||||
const parts: string[] = [];
|
||||
|
||||
// ── Head ──
|
||||
parts.push('<!DOCTYPE html>');
|
||||
parts.push('<html lang="en">');
|
||||
parts.push('<head>');
|
||||
parts.push('<meta charset="UTF-8">');
|
||||
parts.push('<meta name="viewport" content="width=device-width, initial-scale=1.0">');
|
||||
parts.push('<title>' + esc(projectName) + ' — Wiki</title>');
|
||||
parts.push('<script src="https://cdn.jsdelivr.net/npm/marked@11.0.0/marked.min.js"><\/script>');
|
||||
parts.push('<script src="https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.min.js"><\/script>');
|
||||
parts.push('<style>');
|
||||
parts.push(CSS);
|
||||
parts.push('</style>');
|
||||
parts.push('</head>');
|
||||
|
||||
// ── Body ──
|
||||
parts.push('<body>');
|
||||
parts.push('<button class="menu-toggle" id="menu-toggle" aria-label="Toggle menu">☰</button>');
|
||||
parts.push('<div class="layout">');
|
||||
|
||||
// Sidebar
|
||||
parts.push('<nav class="sidebar" id="sidebar">');
|
||||
parts.push('<div class="sidebar-header">');
|
||||
parts.push('<div class="sidebar-title">');
|
||||
parts.push(BOOK_SVG);
|
||||
parts.push(esc(projectName));
|
||||
parts.push('</div>');
|
||||
parts.push('<div class="sidebar-meta" id="meta-info"></div>');
|
||||
parts.push('</div>');
|
||||
parts.push('<div id="nav-tree"></div>');
|
||||
parts.push('<div class="sidebar-footer">Generated by GitNexus</div>');
|
||||
parts.push('</nav>');
|
||||
|
||||
// Content
|
||||
parts.push('<main class="content" id="content">');
|
||||
parts.push('<div class="empty-state"><h2>Loading…</h2></div>');
|
||||
parts.push('</main>');
|
||||
parts.push('</div>');
|
||||
|
||||
// ── Script ──
|
||||
parts.push('<script>');
|
||||
parts.push('var PAGES = ' + pagesJSON + ';');
|
||||
parts.push('var TREE = ' + treeJSON + ';');
|
||||
parts.push('var META = ' + metaJSON + ';');
|
||||
parts.push(JS_APP);
|
||||
parts.push('<\/script>');
|
||||
|
||||
parts.push('</body>');
|
||||
parts.push('</html>');
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
// ─── Static Assets ────────────────────────────────────────────────────
|
||||
|
||||
const BOOK_SVG =
|
||||
'<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">' +
|
||||
'<path d="M2 3h6a4 4 0 014 4v14a3 3 0 00-3-3H2z"/>' +
|
||||
'<path d="M22 3h-6a4 4 0 00-4 4v14a3 3 0 013-3h7z"/>' +
|
||||
'</svg>';
|
||||
|
||||
const CSS = `
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
:root{
|
||||
--bg:#ffffff;--sidebar-bg:#f8f9fb;--border:#e5e7eb;
|
||||
--text:#1e293b;--text-muted:#64748b;--primary:#2563eb;
|
||||
--primary-soft:#eff6ff;--hover:#f1f5f9;--code-bg:#f1f5f9;
|
||||
--radius:8px;--shadow:0 1px 3px rgba(0,0,0,.08);
|
||||
}
|
||||
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;
|
||||
line-height:1.65;color:var(--text);background:var(--bg)}
|
||||
|
||||
.layout{display:flex;min-height:100vh}
|
||||
.sidebar{width:280px;background:var(--sidebar-bg);border-right:1px solid var(--border);
|
||||
position:fixed;top:0;left:0;bottom:0;overflow-y:auto;padding:24px 16px;
|
||||
display:flex;flex-direction:column;z-index:10}
|
||||
.content{margin-left:280px;flex:1;padding:48px 64px;max-width:960px}
|
||||
|
||||
.sidebar-header{margin-bottom:20px;padding-bottom:16px;border-bottom:1px solid var(--border)}
|
||||
.sidebar-title{font-size:16px;font-weight:700;color:var(--text);display:flex;align-items:center;gap:8px}
|
||||
.sidebar-title svg{flex-shrink:0}
|
||||
.sidebar-meta{font-size:11px;color:var(--text-muted);margin-top:6px}
|
||||
.nav-section{margin-bottom:2px}
|
||||
.nav-item{display:block;padding:7px 12px;border-radius:var(--radius);cursor:pointer;
|
||||
font-size:13px;color:var(--text);text-decoration:none;transition:all .15s;
|
||||
white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||
.nav-item:hover{background:var(--hover)}
|
||||
.nav-item.active{background:var(--primary-soft);color:var(--primary);font-weight:600}
|
||||
.nav-item.overview{font-weight:600;margin-bottom:4px}
|
||||
.nav-children{padding-left:14px;border-left:1px solid var(--border);margin-left:12px}
|
||||
.nav-group-label{font-size:11px;font-weight:600;color:var(--text-muted);
|
||||
text-transform:uppercase;letter-spacing:.5px;padding:12px 12px 4px;user-select:none}
|
||||
.sidebar-footer{margin-top:auto;padding-top:16px;border-top:1px solid var(--border);
|
||||
font-size:11px;color:var(--text-muted);text-align:center}
|
||||
|
||||
.content h1{font-size:28px;font-weight:700;margin-bottom:8px;line-height:1.3}
|
||||
.content h2{font-size:22px;font-weight:600;margin:32px 0 12px;padding-bottom:6px;border-bottom:1px solid var(--border)}
|
||||
.content h3{font-size:17px;font-weight:600;margin:24px 0 8px}
|
||||
.content h4{font-size:15px;font-weight:600;margin:20px 0 6px}
|
||||
.content p{margin:12px 0}
|
||||
.content ul,.content ol{margin:12px 0 12px 24px}
|
||||
.content li{margin:4px 0}
|
||||
.content a{color:var(--primary);text-decoration:none}
|
||||
.content a:hover{text-decoration:underline}
|
||||
.content blockquote{border-left:3px solid var(--primary);padding:8px 16px;margin:16px 0;
|
||||
background:var(--primary-soft);border-radius:0 var(--radius) var(--radius) 0;
|
||||
color:var(--text-muted);font-size:14px}
|
||||
.content code{font-family:'SF Mono',Consolas,'Courier New',monospace;font-size:13px;
|
||||
background:var(--code-bg);padding:2px 6px;border-radius:4px}
|
||||
.content pre{background:#1e293b;color:#e2e8f0;border-radius:var(--radius);padding:16px;
|
||||
overflow-x:auto;margin:16px 0}
|
||||
.content pre code{background:none;padding:0;font-size:13px;line-height:1.6;color:inherit}
|
||||
.content table{border-collapse:collapse;width:100%;margin:16px 0}
|
||||
.content th,.content td{border:1px solid var(--border);padding:8px 12px;text-align:left;font-size:14px}
|
||||
.content th{background:var(--sidebar-bg);font-weight:600}
|
||||
.content img{max-width:100%;border-radius:var(--radius)}
|
||||
.content hr{border:none;border-top:1px solid var(--border);margin:32px 0}
|
||||
.content .mermaid{margin:20px 0;text-align:center}
|
||||
|
||||
.menu-toggle{display:none;position:fixed;top:12px;left:12px;z-index:20;
|
||||
background:var(--bg);border:1px solid var(--border);border-radius:var(--radius);
|
||||
padding:8px 12px;cursor:pointer;font-size:18px;box-shadow:var(--shadow)}
|
||||
@media(max-width:768px){
|
||||
.sidebar{transform:translateX(-100%);transition:transform .2s}
|
||||
.sidebar.open{transform:translateX(0);box-shadow:2px 0 12px rgba(0,0,0,.1)}
|
||||
.content{margin-left:0;padding:24px 20px;padding-top:56px}
|
||||
.menu-toggle{display:block}
|
||||
}
|
||||
.empty-state{text-align:center;padding:80px 20px;color:var(--text-muted)}
|
||||
.empty-state h2{font-size:20px;margin-bottom:8px;border:none}
|
||||
`;
|
||||
|
||||
// The client-side JS is kept as a plain string to avoid template literal conflicts
|
||||
const JS_APP = `
|
||||
(function() {
|
||||
var activePage = 'overview';
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
mermaid.initialize({ startOnLoad: false, theme: 'neutral', securityLevel: 'loose' });
|
||||
renderMeta();
|
||||
renderNav();
|
||||
document.getElementById('menu-toggle').addEventListener('click', function() {
|
||||
document.getElementById('sidebar').classList.toggle('open');
|
||||
});
|
||||
if (location.hash && location.hash.length > 1) {
|
||||
activePage = decodeURIComponent(location.hash.slice(1));
|
||||
}
|
||||
navigateTo(activePage);
|
||||
});
|
||||
|
||||
function renderMeta() {
|
||||
if (!META) return;
|
||||
var el = document.getElementById('meta-info');
|
||||
var parts = [];
|
||||
if (META.generatedAt) {
|
||||
parts.push(new Date(META.generatedAt).toLocaleDateString());
|
||||
}
|
||||
if (META.model) parts.push(META.model);
|
||||
if (META.fromCommit) parts.push(META.fromCommit.slice(0, 8));
|
||||
el.textContent = parts.join(' \\u00b7 ');
|
||||
}
|
||||
|
||||
function renderNav() {
|
||||
var container = document.getElementById('nav-tree');
|
||||
var html = '<div class="nav-section">';
|
||||
html += '<a class="nav-item overview" data-page="overview" href="#overview">Overview</a>';
|
||||
html += '</div>';
|
||||
if (TREE.length > 0) {
|
||||
html += '<div class="nav-group-label">Modules</div>';
|
||||
html += buildNavTree(TREE);
|
||||
}
|
||||
container.innerHTML = html;
|
||||
container.addEventListener('click', function(e) {
|
||||
var target = e.target;
|
||||
while (target && !target.dataset.page) { target = target.parentElement; }
|
||||
if (target && target.dataset.page) {
|
||||
e.preventDefault();
|
||||
navigateTo(target.dataset.page);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function buildNavTree(nodes) {
|
||||
var html = '';
|
||||
for (var i = 0; i < nodes.length; i++) {
|
||||
var node = nodes[i];
|
||||
html += '<div class="nav-section">';
|
||||
html += '<a class="nav-item" data-page="' + escH(node.slug) + '" href="#' + encodeURIComponent(node.slug) + '">' + escH(node.name) + '</a>';
|
||||
if (node.children && node.children.length > 0) {
|
||||
html += '<div class="nav-children">' + buildNavTree(node.children) + '</div>';
|
||||
}
|
||||
html += '</div>';
|
||||
}
|
||||
return html;
|
||||
}
|
||||
|
||||
function escH(s) {
|
||||
var d = document.createElement('div');
|
||||
d.textContent = s;
|
||||
return d.innerHTML;
|
||||
}
|
||||
|
||||
function navigateTo(page) {
|
||||
activePage = page;
|
||||
location.hash = encodeURIComponent(page);
|
||||
|
||||
var items = document.querySelectorAll('.nav-item');
|
||||
for (var i = 0; i < items.length; i++) {
|
||||
if (items[i].dataset.page === page) {
|
||||
items[i].classList.add('active');
|
||||
} else {
|
||||
items[i].classList.remove('active');
|
||||
}
|
||||
}
|
||||
|
||||
var contentEl = document.getElementById('content');
|
||||
var md = PAGES[page];
|
||||
|
||||
if (!md) {
|
||||
contentEl.innerHTML = '<div class="empty-state"><h2>Page not found</h2><p>' + escH(page) + '.md does not exist.</p></div>';
|
||||
return;
|
||||
}
|
||||
|
||||
contentEl.innerHTML = marked.parse(md);
|
||||
|
||||
// Rewrite .md links to hash navigation
|
||||
var links = contentEl.querySelectorAll('a[href]');
|
||||
for (var i = 0; i < links.length; i++) {
|
||||
var href = links[i].getAttribute('href');
|
||||
if (href && href.endsWith('.md') && href.indexOf('://') === -1) {
|
||||
var slug = href.replace(/\\.md$/, '');
|
||||
links[i].setAttribute('href', '#' + encodeURIComponent(slug));
|
||||
(function(s) {
|
||||
links[i].addEventListener('click', function(e) {
|
||||
e.preventDefault();
|
||||
navigateTo(s);
|
||||
});
|
||||
})(slug);
|
||||
}
|
||||
}
|
||||
|
||||
// Convert mermaid code blocks into mermaid divs
|
||||
var mermaidBlocks = contentEl.querySelectorAll('pre code.language-mermaid');
|
||||
for (var i = 0; i < mermaidBlocks.length; i++) {
|
||||
var pre = mermaidBlocks[i].parentElement;
|
||||
var div = document.createElement('div');
|
||||
div.className = 'mermaid';
|
||||
div.textContent = mermaidBlocks[i].textContent;
|
||||
pre.parentNode.replaceChild(div, pre);
|
||||
}
|
||||
try { mermaid.run({ querySelector: '.mermaid' }); } catch(e) {}
|
||||
|
||||
window.scrollTo(0, 0);
|
||||
document.getElementById('sidebar').classList.remove('open');
|
||||
}
|
||||
})();
|
||||
`;
|
||||
|
|
@ -42,11 +42,11 @@ export async function resolveLLMConfig(overrides?: Partial<LLMConfig>): Promise<
|
|||
baseUrl: overrides?.baseUrl
|
||||
|| process.env.GITNEXUS_LLM_BASE_URL
|
||||
|| savedConfig.baseUrl
|
||||
|| 'https://api.openai.com/v1',
|
||||
|| 'https://openrouter.ai/api/v1',
|
||||
model: overrides?.model
|
||||
|| process.env.GITNEXUS_MODEL
|
||||
|| savedConfig.model
|
||||
|| 'gpt-4o-mini',
|
||||
|| 'minimax/minimax-m2.5',
|
||||
maxTokens: overrides?.maxTokens ?? 16_384,
|
||||
temperature: overrides?.temperature ?? 0,
|
||||
};
|
||||
|
|
@ -59,14 +59,20 @@ export function estimateTokens(text: string): number {
|
|||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
export interface CallLLMOptions {
|
||||
onChunk?: (charsReceived: number) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Call an OpenAI-compatible LLM API.
|
||||
* Retries once on transient failures (5xx, network errors).
|
||||
* Uses streaming when onChunk callback is provided for real-time progress.
|
||||
* Retries up to 3 times on transient failures (429, 5xx, network errors).
|
||||
*/
|
||||
export async function callLLM(
|
||||
prompt: string,
|
||||
config: LLMConfig,
|
||||
systemPrompt?: string,
|
||||
options?: CallLLMOptions,
|
||||
): Promise<LLMResponse> {
|
||||
const messages: Array<{ role: string; content: string }> = [];
|
||||
if (systemPrompt) {
|
||||
|
|
@ -75,17 +81,20 @@ export async function callLLM(
|
|||
messages.push({ role: 'user', content: prompt });
|
||||
|
||||
const url = `${config.baseUrl.replace(/\/+$/, '')}/chat/completions`;
|
||||
const useStream = !!options?.onChunk;
|
||||
|
||||
const body = {
|
||||
const body: Record<string, unknown> = {
|
||||
model: config.model,
|
||||
messages,
|
||||
max_tokens: config.maxTokens,
|
||||
temperature: config.temperature,
|
||||
};
|
||||
if (useStream) body.stream = true;
|
||||
|
||||
const MAX_RETRIES = 3;
|
||||
let lastError: Error | null = null;
|
||||
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
|
|
@ -99,22 +108,29 @@ export async function callLLM(
|
|||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => 'unknown error');
|
||||
|
||||
// Rate limit — wait and retry
|
||||
if (response.status === 429 && attempt === 0) {
|
||||
const retryAfter = parseInt(response.headers.get('retry-after') || '5', 10);
|
||||
await sleep(retryAfter * 1000);
|
||||
// Rate limit — wait with exponential backoff and retry
|
||||
if (response.status === 429 && attempt < MAX_RETRIES - 1) {
|
||||
const retryAfter = parseInt(response.headers.get('retry-after') || '0', 10);
|
||||
const delay = retryAfter > 0 ? retryAfter * 1000 : (2 ** attempt) * 3000;
|
||||
await sleep(delay);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Server error — retry once
|
||||
if (response.status >= 500 && attempt === 0) {
|
||||
await sleep(2000);
|
||||
// Server error — retry with backoff
|
||||
if (response.status >= 500 && attempt < MAX_RETRIES - 1) {
|
||||
await sleep((attempt + 1) * 2000);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`LLM API error (${response.status}): ${errorText.slice(0, 500)}`);
|
||||
}
|
||||
|
||||
// Streaming path
|
||||
if (useStream && response.body) {
|
||||
return await readSSEStream(response.body, options!.onChunk!);
|
||||
}
|
||||
|
||||
// Non-streaming path
|
||||
const json = await response.json() as any;
|
||||
const choice = json.choices?.[0];
|
||||
if (!choice?.message?.content) {
|
||||
|
|
@ -129,9 +145,9 @@ export async function callLLM(
|
|||
} catch (err: any) {
|
||||
lastError = err;
|
||||
|
||||
// Network error — retry once
|
||||
if (attempt === 0 && (err.code === 'ECONNREFUSED' || err.code === 'ETIMEDOUT' || err.message?.includes('fetch'))) {
|
||||
await sleep(3000);
|
||||
// Network error — retry with backoff
|
||||
if (attempt < MAX_RETRIES - 1 && (err.code === 'ECONNREFUSED' || err.code === 'ETIMEDOUT' || err.message?.includes('fetch'))) {
|
||||
await sleep((attempt + 1) * 3000);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -142,6 +158,52 @@ export async function callLLM(
|
|||
throw lastError || new Error('LLM call failed after retries');
|
||||
}
|
||||
|
||||
/**
|
||||
* Read an SSE stream from an OpenAI-compatible streaming response.
|
||||
*/
|
||||
async function readSSEStream(
|
||||
body: ReadableStream<Uint8Array>,
|
||||
onChunk: (charsReceived: number) => void,
|
||||
): Promise<LLMResponse> {
|
||||
const decoder = new TextDecoder();
|
||||
const reader = body.getReader();
|
||||
let content = '';
|
||||
let buffer = '';
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop() || '';
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || !trimmed.startsWith('data: ')) continue;
|
||||
const data = trimmed.slice(6);
|
||||
if (data === '[DONE]') continue;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(data);
|
||||
const delta = parsed.choices?.[0]?.delta?.content;
|
||||
if (delta) {
|
||||
content += delta;
|
||||
onChunk(content.length);
|
||||
}
|
||||
} catch {
|
||||
// Skip malformed SSE chunks
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!content) {
|
||||
throw new Error('LLM returned empty streaming response');
|
||||
}
|
||||
|
||||
return { content };
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,24 +1,44 @@
|
|||
/**
|
||||
* KuzuDB Adapter (Connection Pool)
|
||||
*
|
||||
* Manages a pool of KuzuDB connections keyed by repoId.
|
||||
* Connections are lazily opened on first query and evicted
|
||||
* after idle timeout or when pool exceeds max size (LRU).
|
||||
* Manages a pool of KuzuDB databases keyed by repoId, each with
|
||||
* multiple Connection objects for safe concurrent query execution.
|
||||
*
|
||||
* KuzuDB Connections are NOT thread-safe — a single Connection
|
||||
* segfaults if concurrent .query() calls hit it simultaneously.
|
||||
* This adapter provides a checkout/return connection pool so each
|
||||
* concurrent query gets its own Connection from the same Database.
|
||||
*
|
||||
* @see https://docs.kuzudb.com/concurrency — multiple Connections
|
||||
* from the same Database is the officially supported concurrency pattern.
|
||||
*/
|
||||
|
||||
import fs from 'fs/promises';
|
||||
import kuzu from 'kuzu';
|
||||
|
||||
/** Per-repo pool: one Database, many Connections */
|
||||
interface PoolEntry {
|
||||
db: kuzu.Database;
|
||||
conn: kuzu.Connection;
|
||||
/** Available connections ready for checkout */
|
||||
available: kuzu.Connection[];
|
||||
/** Number of connections currently checked out */
|
||||
checkedOut: number;
|
||||
/** Queued waiters for when all connections are busy */
|
||||
waiters: Array<(conn: kuzu.Connection) => void>;
|
||||
lastUsed: number;
|
||||
dbPath: string;
|
||||
}
|
||||
|
||||
const pool = new Map<string, PoolEntry>();
|
||||
|
||||
/** Max repos in the pool (LRU eviction) */
|
||||
const MAX_POOL_SIZE = 5;
|
||||
/** Idle timeout before closing a repo's connections */
|
||||
const IDLE_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
|
||||
/** Max connections per repo (caps concurrent queries per repo) */
|
||||
const MAX_CONNS_PER_REPO = 8;
|
||||
/** Connections created eagerly on init */
|
||||
const INITIAL_CONNS_PER_REPO = 2;
|
||||
|
||||
let idleTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
|
|
@ -35,14 +55,13 @@ function ensureIdleTimer(): void {
|
|||
}
|
||||
}
|
||||
}, 60_000);
|
||||
// Don't keep the process alive just for this timer
|
||||
if (idleTimer && typeof idleTimer === 'object' && 'unref' in idleTimer) {
|
||||
(idleTimer as NodeJS.Timeout).unref();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict the least-recently-used connection if pool is at capacity
|
||||
* Evict the least-recently-used repo if pool is at capacity
|
||||
*/
|
||||
function evictLRU(): void {
|
||||
if (pool.size < MAX_POOL_SIZE) return;
|
||||
|
|
@ -61,21 +80,37 @@ function evictLRU(): void {
|
|||
}
|
||||
|
||||
/**
|
||||
* Close a single pool entry
|
||||
* Close all connections for a repo and remove it from the pool
|
||||
*/
|
||||
function closeOne(repoId: string): void {
|
||||
const entry = pool.get(repoId);
|
||||
if (!entry) return;
|
||||
try { entry.conn.close(); } catch {}
|
||||
for (const conn of entry.available) {
|
||||
try { conn.close(); } catch {}
|
||||
}
|
||||
try { entry.db.close(); } catch {}
|
||||
pool.delete(repoId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new Connection from a repo's Database.
|
||||
* Silences stdout to prevent native module output from corrupting MCP stdio.
|
||||
*/
|
||||
function createConnection(db: kuzu.Database): kuzu.Connection {
|
||||
const origWrite = process.stdout.write;
|
||||
process.stdout.write = (() => true) as any;
|
||||
try {
|
||||
return new kuzu.Connection(db);
|
||||
} finally {
|
||||
process.stdout.write = origWrite;
|
||||
}
|
||||
}
|
||||
|
||||
const LOCK_RETRY_ATTEMPTS = 3;
|
||||
const LOCK_RETRY_DELAY_MS = 2000;
|
||||
|
||||
/**
|
||||
* Initialize (or reuse) a connection for a specific repo.
|
||||
* Initialize (or reuse) a Database + connection pool for a specific repo.
|
||||
* Retries on lock errors (e.g., when `gitnexus analyze` is running).
|
||||
*/
|
||||
export const initKuzu = async (repoId: string, dbPath: string): Promise<void> => {
|
||||
|
|
@ -99,8 +134,6 @@ export const initKuzu = async (repoId: string, dbPath: string): Promise<void> =>
|
|||
// avoids lock conflicts when `gitnexus analyze` is writing.
|
||||
let lastError: Error | null = null;
|
||||
for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) {
|
||||
// Silence stdout during KuzuDB init — native module may write to stdout
|
||||
// which corrupts the MCP stdio protocol.
|
||||
const origWrite = process.stdout.write;
|
||||
process.stdout.write = (() => true) as any;
|
||||
try {
|
||||
|
|
@ -110,9 +143,15 @@ export const initKuzu = async (repoId: string, dbPath: string): Promise<void> =>
|
|||
false, // enableCompression (default)
|
||||
true, // readOnly
|
||||
);
|
||||
const conn = new kuzu.Connection(db);
|
||||
process.stdout.write = origWrite;
|
||||
pool.set(repoId, { db, conn, lastUsed: Date.now(), dbPath });
|
||||
|
||||
// Pre-create a small pool of connections
|
||||
const available: kuzu.Connection[] = [];
|
||||
for (let i = 0; i < INITIAL_CONNS_PER_REPO; i++) {
|
||||
available.push(createConnection(db));
|
||||
}
|
||||
|
||||
pool.set(repoId, { db, available, checkedOut: 0, waiters: [], lastUsed: Date.now(), dbPath });
|
||||
ensureIdleTimer();
|
||||
return;
|
||||
} catch (err: any) {
|
||||
|
|
@ -121,7 +160,6 @@ export const initKuzu = async (repoId: string, dbPath: string): Promise<void> =>
|
|||
const isLockError = lastError.message.includes('Could not set lock')
|
||||
|| lastError.message.includes('lock');
|
||||
if (!isLockError || attempt === LOCK_RETRY_ATTEMPTS) break;
|
||||
// Wait before retrying — analyze may be mid-rebuild
|
||||
await new Promise(resolve => setTimeout(resolve, LOCK_RETRY_DELAY_MS * attempt));
|
||||
}
|
||||
}
|
||||
|
|
@ -133,7 +171,50 @@ export const initKuzu = async (repoId: string, dbPath: string): Promise<void> =>
|
|||
};
|
||||
|
||||
/**
|
||||
* Execute a query on a specific repo's connection
|
||||
* Checkout a connection from the pool.
|
||||
* Returns an available connection, or creates a new one if under the cap.
|
||||
* If all connections are busy and at cap, queues the caller until one is returned.
|
||||
*/
|
||||
function checkout(entry: PoolEntry): Promise<kuzu.Connection> {
|
||||
// Fast path: grab an available connection
|
||||
if (entry.available.length > 0) {
|
||||
entry.checkedOut++;
|
||||
return Promise.resolve(entry.available.pop()!);
|
||||
}
|
||||
|
||||
// Grow the pool if under the cap
|
||||
const totalConns = entry.available.length + entry.checkedOut;
|
||||
if (totalConns < MAX_CONNS_PER_REPO) {
|
||||
entry.checkedOut++;
|
||||
return Promise.resolve(createConnection(entry.db));
|
||||
}
|
||||
|
||||
// At capacity — queue the caller. checkin() will resolve this when
|
||||
// a connection is returned, handing it directly to the next waiter.
|
||||
return new Promise<kuzu.Connection>(resolve => {
|
||||
entry.waiters.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a connection to the pool after use.
|
||||
* If there are queued waiters, hand the connection directly to the next one
|
||||
* instead of putting it back in the available array (avoids race conditions).
|
||||
*/
|
||||
function checkin(entry: PoolEntry, conn: kuzu.Connection): void {
|
||||
if (entry.waiters.length > 0) {
|
||||
// Hand directly to the next waiter — no intermediate available state
|
||||
const waiter = entry.waiters.shift()!;
|
||||
waiter(conn);
|
||||
} else {
|
||||
entry.checkedOut--;
|
||||
entry.available.push(conn);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a query on a specific repo's connection pool.
|
||||
* Automatically checks out a connection, runs the query, and returns it.
|
||||
*/
|
||||
export const executeQuery = async (repoId: string, cypher: string): Promise<any[]> => {
|
||||
const entry = pool.get(repoId);
|
||||
|
|
@ -142,16 +223,22 @@ export const executeQuery = async (repoId: string, cypher: string): Promise<any[
|
|||
}
|
||||
|
||||
entry.lastUsed = Date.now();
|
||||
const queryResult = await entry.conn.query(cypher);
|
||||
const result = Array.isArray(queryResult) ? queryResult[0] : queryResult;
|
||||
const rows = await result.getAll();
|
||||
return rows;
|
||||
|
||||
const conn = await checkout(entry);
|
||||
try {
|
||||
const queryResult = await conn.query(cypher);
|
||||
const result = Array.isArray(queryResult) ? queryResult[0] : queryResult;
|
||||
const rows = await result.getAll();
|
||||
return rows;
|
||||
} finally {
|
||||
checkin(entry, conn);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Close one or all connections.
|
||||
* If repoId is provided, close only that connection.
|
||||
* If omitted, close all connections in the pool.
|
||||
* Close one or all repo pools.
|
||||
* If repoId is provided, close only that repo's connections.
|
||||
* If omitted, close all repos.
|
||||
*/
|
||||
export const closeKuzu = async (repoId?: string): Promise<void> => {
|
||||
if (repoId) {
|
||||
|
|
@ -159,7 +246,6 @@ export const closeKuzu = async (repoId?: string): Promise<void> => {
|
|||
return;
|
||||
}
|
||||
|
||||
// Close all
|
||||
for (const id of [...pool.keys()]) {
|
||||
closeOne(id);
|
||||
}
|
||||
|
|
@ -171,6 +257,6 @@ export const closeKuzu = async (repoId?: string): Promise<void> => {
|
|||
};
|
||||
|
||||
/**
|
||||
* Check if a specific repo's connection is active
|
||||
* Check if a specific repo's pool is active
|
||||
*/
|
||||
export const isKuzuReady = (repoId: string): boolean => pool.has(repoId);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue