From 1e5a1ad01ead3c7401a1f47ef2c39ba28c64475f Mon Sep 17 00:00:00 2001 From: enih Date: Sat, 31 Oct 2026 21:24:37 +0800 Subject: [PATCH] fix(wiki): configure limits and retry truncated pages (#3775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(wiki): let callers set the completion token limit Reasoning models can spend the hardcoded 16384 cap before a page is finished. Keep that default, and honor a flag, GITNEXUS_MAX_COMPLETION_TOKENS, or maxCompletionTokens in the CLI config. Fixes #3774 Co-authored-by: Cursor * fix(wiki): reject LLM responses cut off at the completion token limit callLLM accepted a response whose finish_reason was "length" as a finished one, streamed or not, so a page cut off mid-sentence or mid-table was written and the run reported success. With a reasoning model the limit can be spent before the answer starts. Both paths now throw when the provider reports finish_reason "length", naming the limit. The generator already treats a thrown error as a failed module: the page is not written, the module is listed under "Failed modules", and the next run regenerates it. --------- Co-authored-by: nullptr Co-authored-by: Cursor Co-authored-by: Samuel Mbabhazi Co-authored-by: Gergo Magyar Co-authored-by: Gergő Magyar --- .claude/skills/gitnexus-cli/SKILL.md | 1 + docs/wiki-generation.md | 17 ++ .../skills/gitnexus-cli/SKILL.md | 1 + gitnexus/skills/gitnexus-cli.md | 1 + gitnexus/src/cli/help-i18n.ts | 1 + gitnexus/src/cli/i18n/en.ts | 2 + gitnexus/src/cli/i18n/zh-CN.ts | 1 + gitnexus/src/cli/index.ts | 4 + gitnexus/src/cli/wiki.ts | 50 +++- gitnexus/src/core/wiki/generator.ts | 38 ++- gitnexus/src/core/wiki/llm-client.ts | 59 ++++- gitnexus/src/storage/repo-manager.ts | 2 + gitnexus/test/unit/wiki-flags.test.ts | 69 ++++++ gitnexus/test/unit/wiki-llm-client.test.ts | 94 +++++++ .../test/unit/wiki-llm-truncation.test.ts | 150 ++++++++++++ .../unit/wiki-truncation-recovery.test.ts | 230 ++++++++++++++++++ 16 files changed, 702 insertions(+), 18 deletions(-) create mode 100644 gitnexus/test/unit/wiki-llm-truncation.test.ts create mode 100644 gitnexus/test/unit/wiki-truncation-recovery.test.ts diff --git a/.claude/skills/gitnexus-cli/SKILL.md b/.claude/skills/gitnexus-cli/SKILL.md index 3a00ab546..53a581bd6 100644 --- a/.claude/skills/gitnexus-cli/SKILL.md +++ b/.claude/skills/gitnexus-cli/SKILL.md @@ -87,6 +87,7 @@ Generates repository documentation from the knowledge graph using an LLM. HTTP p | `--concurrency ` | Parallel LLM calls (default: 3) | | `--timeout ` | LLM request timeout in seconds (default: disabled) | | `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--max-completion-tokens ` | Max completion tokens per LLM request (default: 16384). `GITNEXUS_MAX_COMPLETION_TOKENS` overrides `maxCompletionTokens` in `~/.gitnexus/config.json`; the flag overrides both. | | `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | | `--gist` | Publish wiki as a public GitHub Gist | diff --git a/docs/wiki-generation.md b/docs/wiki-generation.md index 4ddcf1822..a5348e42f 100644 --- a/docs/wiki-generation.md +++ b/docs/wiki-generation.md @@ -20,6 +20,12 @@ gitnexus wiki --force gitnexus wiki --timeout # LLM request timeout in seconds (default: disabled) gitnexus wiki --retries # Max LLM retry attempts per request (default: 3) +# Raise the completion-token limit (default: 16384) +gitnexus wiki --max-completion-tokens 131072 +GITNEXUS_MAX_COMPLETION_TOKENS=131072 gitnexus wiki +# Or set "maxCompletionTokens": 131072 in ~/.gitnexus/config.json +# Precedence: flag, then the environment variable, then the config file. + # Allow a specific LAN/self-hosted HTTP LLM host (HTTPS is preferred for remote endpoints) gitnexus wiki --base-url http://llama-box.local:8080/v1 --allow-insecure-connection llama-box.local # Or set a comma-separated host allowlist: @@ -29,6 +35,17 @@ GITNEXUS_ALLOW_INSECURE_CONNECTION=llama-box.local,192.168.1.23 gitnexus wiki --lang # e.g. english, chinese, spanish, japanese ``` +The completion limit must be a positive integer supported by your model. It applies to +OpenAI-compatible HTTP providers; local agent CLI providers manage their own limits. +For reasoning models, the limit includes reasoning tokens as well as the visible answer. +When a provider reports `finish_reason: "length"`, GitNexus rejects the incomplete response +instead of saving it as a finished page. Failed modules are listed, the command exits with +status 1, and a normal rerun retries them even if the repository commit has not changed. +Increase `--max-completion-tokens` within your model's supported limit before retrying. +Truncated grouping or overview responses also fail the run; batched grouping retains its +directory-based fallback. Incomplete runs are not published with `--gist`. +Use `--force` to replace incomplete pages already saved by an older version. + For safety, `http://` LLM base URLs are allowed by default only for loopback hosts (`localhost`, `127.0.0.1`, `::1`). `--allow-insecure-connection` and `GITNEXUS_ALLOW_INSECURE_CONNECTION` accept exact hostnames or IP addresses only; do not include schemes, ports, paths, credentials, or wildcards. The wiki generator reads the indexed graph structure, groups files into modules via LLM, generates per-module documentation pages, and creates an overview page — all with cross-references to the knowledge graph. diff --git a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md index 3a00ab546..53a581bd6 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-cli/SKILL.md @@ -87,6 +87,7 @@ Generates repository documentation from the knowledge graph using an LLM. HTTP p | `--concurrency ` | Parallel LLM calls (default: 3) | | `--timeout ` | LLM request timeout in seconds (default: disabled) | | `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--max-completion-tokens ` | Max completion tokens per LLM request (default: 16384). `GITNEXUS_MAX_COMPLETION_TOKENS` overrides `maxCompletionTokens` in `~/.gitnexus/config.json`; the flag overrides both. | | `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | | `--gist` | Publish wiki as a public GitHub Gist | diff --git a/gitnexus/skills/gitnexus-cli.md b/gitnexus/skills/gitnexus-cli.md index 3a00ab546..53a581bd6 100644 --- a/gitnexus/skills/gitnexus-cli.md +++ b/gitnexus/skills/gitnexus-cli.md @@ -87,6 +87,7 @@ Generates repository documentation from the knowledge graph using an LLM. HTTP p | `--concurrency ` | Parallel LLM calls (default: 3) | | `--timeout ` | LLM request timeout in seconds (default: disabled) | | `--retries ` | Max LLM retry attempts per request (default: 3) | +| `--max-completion-tokens ` | Max completion tokens per LLM request (default: 16384). `GITNEXUS_MAX_COMPLETION_TOKENS` overrides `maxCompletionTokens` in `~/.gitnexus/config.json`; the flag overrides both. | | `--lang ` | Output language for generated documentation (e.g. english, chinese, spanish, japanese) | | `--gist` | Publish wiki as a public GitHub Gist | diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index d8543a62a..c7d748818 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -114,6 +114,7 @@ const OPTION_DESCRIPTION_KEYS = { 'wiki|--concurrency ': 'help.option.wiki.concurrency', 'wiki|--timeout ': 'help.option.wiki.timeout', 'wiki|--retries ': 'help.option.wiki.retries', + 'wiki|--max-completion-tokens ': 'help.option.wiki.maxCompletionTokens', 'wiki|--allow-insecure-connection ': 'help.option.wiki.allowInsecureConnection', 'wiki|--gist': 'help.option.wiki.gist', 'wiki|-v, --verbose': 'help.option.verbose', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 8cedc843f..5c38de802 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -361,6 +361,8 @@ export const en = { 'help.option.wiki.concurrency': 'Parallel LLM calls (default: 3)', 'help.option.wiki.timeout': 'LLM request timeout in seconds (default: disabled)', 'help.option.wiki.retries': 'Max LLM retry attempts per request (default: 3)', + 'help.option.wiki.maxCompletionTokens': + 'Max completion tokens for each LLM request (default: 16384)', 'help.option.wiki.allowInsecureConnection': 'Allow exact host(s) for http:// LLM base URLs (comma-separated; HTTPS is preferred)', 'help.option.wiki.gist': 'Publish wiki as a public GitHub Gist after generation', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index f333188b7..a1b0fd1aa 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -329,6 +329,7 @@ export const zhCN = { 'help.option.wiki.concurrency': '并行 LLM 调用数（默认：3）', 'help.option.wiki.timeout': 'LLM 请求超时时间（秒，默认：禁用）', 'help.option.wiki.retries': '每个请求的最大 LLM 重试次数（默认：3）', + 'help.option.wiki.maxCompletionTokens': '每次 LLM 请求的最大 completion token 数（默认：16384）', 'help.option.wiki.allowInsecureConnection': '允许 http:// LLM base URL 使用的精确主机（逗号分隔；推荐使用 HTTPS）', 'help.option.wiki.gist': '生成后发布 Wiki 为公开 GitHub Gist', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 37dfe1ddb..baceb40d9 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -427,6 +427,10 @@ program .option('--concurrency ', 'Parallel LLM calls (default: 3)', '3') .option('--timeout ', 'LLM request timeout in seconds (default: disabled)') .option('--retries ', 'Max LLM retry attempts per request (default: 3)') + .option( + '--max-completion-tokens ', + 'Max completion tokens for each LLM request (default: 16384)', + ) .option( '--allow-insecure-connection ', 'Allow exact host(s) for http:// LLM base URLs (comma-separated; HTTPS is preferred)', diff --git a/gitnexus/src/cli/wiki.ts b/gitnexus/src/cli/wiki.ts index 11325a3c3..37badb70c 100644 --- a/gitnexus/src/cli/wiki.ts +++ b/gitnexus/src/cli/wiki.ts @@ -47,6 +47,7 @@ export interface WikiCommandOptions { retries?: string; lang?: string; allowInsecureConnection?: string; + maxCompletionTokens?: string; } function isLocalProvider( @@ -189,6 +190,7 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) let timeoutSeconds: number | undefined; let retries: number | undefined; let allowedInsecureHttpHosts: string[] | undefined; + let maxCompletionTokens: number | undefined; try { timeoutSeconds = options?.timeout === undefined @@ -202,6 +204,12 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) options?.allowInsecureConnection === undefined ? undefined : parseLLMAllowedInsecureHttpHosts(options.allowInsecureConnection); + maxCompletionTokens = + options?.maxCompletionTokens === undefined + ? undefined + : parseIntegerOption(options.maxCompletionTokens, '--max-completion-tokens', { + minimum: 1, + }); } catch (error) { console.log(` Error: ${(error as Error).message}\n`); process.exitCode = 1; @@ -271,15 +279,23 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) options?.reasoningModel !== undefined ); - let llmConfig = await resolveLLMConfig({ - model: options?.model, - baseUrl: options?.baseUrl, - apiKey: options?.apiKey, - provider: options?.provider, - apiVersion: options?.apiVersion, - isReasoningModel: options?.reasoningModel, - allowedInsecureHttpHosts, - }); + let llmConfig; + try { + llmConfig = await resolveLLMConfig({ + model: options?.model, + baseUrl: options?.baseUrl, + apiKey: options?.apiKey, + provider: options?.provider, + apiVersion: options?.apiVersion, + isReasoningModel: options?.reasoningModel, + allowedInsecureHttpHosts, + maxTokens: maxCompletionTokens, + }); + } catch (error) { + console.log(` Error: ${(error as Error).message}\n`); + process.exitCode = 1; + return; + } // Run interactive setup if no saved config and no CLI flags provided // (even if env vars exist — let user explicitly choose their provider) @@ -687,7 +703,10 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) bar.stop(); const totalElapsed = ((Date.now() - t0) / 1000).toFixed(1); - console.log(`\n Wiki generated successfully (${totalElapsed}s)\n`); + const incomplete = !!continueResult.failedModules?.length; + console.log( + `\n ${incomplete ? 'Wiki generation incomplete' : 'Wiki generated successfully'} (${totalElapsed}s)\n`, + ); console.log(` Mode: ${continueResult.mode}`); console.log(` Pages: ${continueResult.pagesGenerated}`); console.log(` Output: ${wikiDir}`); @@ -698,6 +717,9 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) for (const mod of continueResult.failedModules) { console.log(` - ${mod}`); } + console.log(' Re-run to retry failed modules (pages will be regenerated).\n'); + process.exitCode = 1; + return; } console.log(''); @@ -714,7 +736,10 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) return; } - console.log(`\n Wiki generated successfully (${elapsed}s)\n`); + const incomplete = !!result.failedModules?.length; + console.log( + `\n ${incomplete ? 'Wiki generation incomplete' : 'Wiki generated successfully'} (${elapsed}s)\n`, + ); console.log(` Mode: ${result.mode}`); console.log(` Pages: ${result.pagesGenerated}`); console.log(` Output: ${wikiDir}`); @@ -726,6 +751,9 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) console.log(` - ${mod}`); } console.log(' Re-run to retry failed modules (pages will be regenerated).'); + console.log(''); + process.exitCode = 1; + return; } console.log(''); diff --git a/gitnexus/src/core/wiki/generator.ts b/gitnexus/src/core/wiki/generator.ts index 92e656fce..2afea8b6f 100644 --- a/gitnexus/src/core/wiki/generator.ts +++ b/gitnexus/src/core/wiki/generator.ts @@ -85,6 +85,8 @@ export interface WikiMeta { lang: string; moduleFiles: Record; moduleTree: ModuleTreeNode[]; + /** Pages that must be retried even when fromCommit is still current. */ + failedModules?: string[]; } export interface ModuleTreeNode { @@ -260,6 +262,7 @@ export class WikiGenerator { * Main entry point. Runs the full pipeline or incremental update. */ async run(): Promise { + this.failedModules = []; await fs.mkdir(this.wikiDir, { recursive: true }); const existingMeta = await this.loadWikiMeta(); @@ -267,7 +270,12 @@ export class WikiGenerator { const forceMode = this.options.force; // Up-to-date check (skip if --force) - if (!forceMode && existingMeta && existingMeta.fromCommit === currentCommit) { + if ( + !forceMode && + existingMeta && + existingMeta.fromCommit === currentCommit && + !existingMeta.failedModules?.length + ) { const currentLang = this.effectiveLang(); const metaLang = existingMeta.lang ?? ''; if (currentLang !== metaLang) { @@ -284,6 +292,8 @@ export class WikiGenerator { // Force mode: delete snapshot to force full re-grouping if (forceMode) { + // A failure must not leave the previous wiki marked complete at this commit. + await fs.rm(path.join(this.wikiDir, 'meta.json'), { force: true }); try { await fs.unlink(path.join(this.wikiDir, 'first_module_tree.json')); } catch {} @@ -421,6 +431,11 @@ export class WikiGenerator { // Process parent modules sequentially (they depend on child docs) for (const node of parents) { + if (node.children?.some((child) => this.failedModules.includes(child.name))) { + this.failedModules.push(node.name); + reportProgress(`Failed: ${node.name}`); + continue; + } const pagePath = path.join(this.wikiDir, `${node.slug}.md`); if (await this.fileExists(pagePath)) { reportProgress(node.name); @@ -452,6 +467,7 @@ export class WikiGenerator { lang: this.effectiveLang(), moduleFiles, moduleTree, + failedModules: [...this.failedModules], }); this.onProgress('done', 100, 'Wiki generation complete'); @@ -1005,7 +1021,7 @@ export class WikiGenerator { return { ...fullResult, mode: 'incremental' }; } - if (changedFiles.length === 0) { + if (changedFiles.length === 0 && !existingMeta.failedModules?.length) { // No file changes but commit differs (e.g. merge commit) await this.saveWikiMeta({ ...existingMeta, @@ -1019,7 +1035,7 @@ export class WikiGenerator { this.onProgress('incremental', 10, `${changedFiles.length} files changed`); // Determine affected modules - const affectedModules = new Set(); + const affectedModules = new Set(existingMeta.failedModules ?? []); const newFiles: string[] = []; for (const fp of changedFiles) { @@ -1080,7 +1096,7 @@ export class WikiGenerator { } let incProcessed = 0; - pagesGenerated += await this.runParallel(affectedNodes, async (node) => { + const regenerateNode = async (node: ModuleTreeNode): Promise => { try { if (node.children && node.children.length > 0) { await this.generateParentPage(node); @@ -1100,7 +1116,18 @@ export class WikiGenerator { incProcessed++; return 0; } - }); + }; + + const leaves = affectedNodes.filter((node) => !node.children?.length); + const parents = affectedNodes.filter((node) => node.children?.length); + pagesGenerated += await this.runParallel(leaves, regenerateNode); + for (const node of parents) { + if (node.children?.some((child) => this.failedModules.includes(child.name))) { + this.failedModules.push(node.name); + continue; + } + pagesGenerated += await regenerateNode(node); + } // Regenerate overview if any pages changed if (pagesGenerated > 0) { @@ -1117,6 +1144,7 @@ export class WikiGenerator { generatedAt: new Date().toISOString(), model: this.llmConfig.model, lang: this.effectiveLang(), + failedModules: [...this.failedModules], }); this.onProgress('done', 100, 'Incremental update complete'); diff --git a/gitnexus/src/core/wiki/llm-client.ts b/gitnexus/src/core/wiki/llm-client.ts index b8af62099..a635874ea 100644 --- a/gitnexus/src/core/wiki/llm-client.ts +++ b/gitnexus/src/core/wiki/llm-client.ts @@ -148,7 +148,11 @@ export async function resolveLLMConfig(overrides?: Partial): Promise< ? '' : (reuseSavedHttpConfig ? savedConfig.model : undefined) || (savedProvider === 'minimax' ? MINIMAX_MODEL_IDS[0] : '')), - maxTokens: overrides?.maxTokens ?? 16_384, + maxTokens: + readOverrideMaxCompletionTokens(overrides?.maxTokens) ?? + readEnvMaxCompletionTokens(process.env[LLM_MAX_COMPLETION_TOKENS_ENV]) ?? + readConfiguredMaxCompletionTokens(savedConfig.maxCompletionTokens) ?? + DEFAULT_MAX_COMPLETION_TOKENS, temperature: overrides?.temperature ?? 0, provider: savedProvider, apiVersion: @@ -185,6 +189,37 @@ function isTimeoutLikeError(err: unknown): boolean { } export const LLM_ALLOW_INSECURE_CONNECTION_ENV = 'GITNEXUS_ALLOW_INSECURE_CONNECTION'; +export const LLM_MAX_COMPLETION_TOKENS_ENV = 'GITNEXUS_MAX_COMPLETION_TOKENS'; +export const DEFAULT_MAX_COMPLETION_TOKENS = 16_384; + +function readOverrideMaxCompletionTokens(value: number | undefined): number | undefined { + if (value === undefined) return undefined; + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error('--max-completion-tokens must be a positive integer'); + } + return value; +} + +function readEnvMaxCompletionTokens(value: string | undefined): number | undefined { + if (value === undefined || value.trim() === '') return undefined; + const trimmed = value.trim(); + if (!/^[1-9]\d*$/.test(trimmed)) { + throw new Error(`${LLM_MAX_COMPLETION_TOKENS_ENV} must be a positive integer`); + } + const parsed = Number(trimmed); + if (!Number.isSafeInteger(parsed) || String(parsed) !== trimmed) { + throw new Error(`${LLM_MAX_COMPLETION_TOKENS_ENV} is too large`); + } + return parsed; +} + +function readConfiguredMaxCompletionTokens(value: unknown): number | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1) { + throw new Error('maxCompletionTokens in ~/.gitnexus/config.json must be a positive integer'); + } + return value; +} function normalizeAllowedInsecureHttpHost(host: string): string { const trimmed = host.trim().toLowerCase(); @@ -450,12 +485,15 @@ export async function callLLM( // Streaming path if (useStream && response.body) { - return await readSSEStream(response.body, options!.onChunk!); + return await readSSEStream(response.body, options!.onChunk!, config.maxTokens); } // Non-streaming path const json = (await response.json()) as any; const choice = json.choices?.[0]; + if (choice?.finish_reason === 'length') { + throw truncatedResponseError(config.maxTokens); + } if (!choice?.message?.content) { throw new Error('LLM returned empty response'); } @@ -467,18 +505,28 @@ export async function callLLM( }; } +// A response cut off by the completion limit reads like a finished one, and a +// reasoning model can spend the whole limit before the answer starts. +function truncatedResponseError(maxTokens: number): Error { + return new Error( + `LLM response stopped at the completion token limit (${maxTokens}, finish_reason "length") and is incomplete.`, + ); +} + /** * Read an SSE stream from an OpenAI-compatible streaming response. */ async function readSSEStream( body: ReadableStream, onChunk: (charsReceived: number) => void, + maxTokens: number, ): Promise { const decoder = new TextDecoder(); const reader = body.getReader(); let content = ''; let buffer = ''; let contentFilterTriggered = false; + let lengthLimitReached = false; while (true) { const { done, value } = await reader.read(); @@ -503,6 +551,9 @@ async function readSSEStream( contentFilterTriggered = true; continue; } + if (choice?.finish_reason === 'length') { + lengthLimitReached = true; + } const delta = choice?.delta?.content; if (delta) { @@ -521,6 +572,10 @@ async function readSSEStream( ); } + if (lengthLimitReached) { + throw truncatedResponseError(maxTokens); + } + if (!content) { throw new Error('LLM returned empty streaming response'); } diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index c3c731418..7fb4ac5f9 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -1848,6 +1848,8 @@ export interface CLIConfig { apiVersion?: string; /** Set true when the deployment is a reasoning model (o1, o3, o4-mini). Auto-detected for OpenAI; must be set for Azure deployments. */ isReasoningModel?: boolean; + /** Completion-token limit for wiki LLM requests. Flag and GITNEXUS_MAX_COMPLETION_TOKENS win over this. */ + maxCompletionTokens?: number; } /** diff --git a/gitnexus/test/unit/wiki-flags.test.ts b/gitnexus/test/unit/wiki-flags.test.ts index 935b8953b..1721f504d 100644 --- a/gitnexus/test/unit/wiki-flags.test.ts +++ b/gitnexus/test/unit/wiki-flags.test.ts @@ -941,6 +941,75 @@ describe('wikiCommand --timeout mapping', () => { expect(harness.getCapturedConfig()?.maxAttempts).toBe(5); }); + it('passes --max-completion-tokens through to the LLM config', async () => { + const harness = await loadWikiCommandHarness(); + + await harness.wikiCommand('/tmp/repo', { maxCompletionTokens: '131072' }); + + expect(harness.resolveLLMConfig).toHaveBeenCalledWith( + expect.objectContaining({ maxTokens: 131072 }), + ); + expect(harness.generatorCtor).toHaveBeenCalledTimes(1); + expect(harness.getCapturedConfig()?.maxTokens).toBe(131072); + }); + + it.each([false, true])( + 'reports incomplete generation as failure (review: %s)', + async (review) => { + const harness = await loadWikiCommandHarness(); + const run = vi.fn().mockResolvedValue({ + mode: 'full', + pagesGenerated: 1, + failedModules: ['Billing'], + }); + if (review) { + run.mockResolvedValueOnce({ mode: 'full', pagesGenerated: 0, moduleTree: [] }); + } + harness.generatorCtor.mockImplementation(function () { + return { run }; + }); + + await harness.wikiCommand('/tmp/repo', { review, gist: true }); + + expect(process.exitCode).toBe(1); + expect(run).toHaveBeenCalledTimes(review ? 2 : 1); + expect(harness.consoleSpy).toHaveBeenCalledWith( + expect.stringContaining('Wiki generation incomplete'), + ); + expect(harness.consoleSpy).not.toHaveBeenCalledWith( + expect.stringContaining('Wiki generated successfully'), + ); + expect(harness.consoleSpy).toHaveBeenCalledWith(' - Billing'); + }, + ); + + it('reports invalid environment or saved limits before starting generation', async () => { + const harness = await loadWikiCommandHarness(); + harness.resolveLLMConfig.mockRejectedValueOnce( + new Error('GITNEXUS_MAX_COMPLETION_TOKENS must be a positive integer'), + ); + + await harness.wikiCommand('/tmp/repo'); + + expect(process.exitCode).toBe(1); + expect(harness.generatorCtor).not.toHaveBeenCalled(); + expect(harness.consoleSpy).toHaveBeenCalledWith( + ' Error: GITNEXUS_MAX_COMPLETION_TOKENS must be a positive integer\n', + ); + }); + + it('rejects a non-integer --max-completion-tokens value before starting generation', async () => { + const harness = await loadWikiCommandHarness(); + + await harness.wikiCommand('/tmp/repo', { maxCompletionTokens: 'abc' }); + + expect(process.exitCode).toBe(1); + expect(harness.generatorCtor).not.toHaveBeenCalled(); + expect(harness.consoleSpy).toHaveBeenCalledWith( + ' Error: --max-completion-tokens must be a positive integer\n', + ); + }); + it('maps --allow-insecure-connection to allowedInsecureHttpHosts', async () => { const harness = await loadWikiCommandHarness(); diff --git a/gitnexus/test/unit/wiki-llm-client.test.ts b/gitnexus/test/unit/wiki-llm-client.test.ts index 2a20c7591..b1f107667 100644 --- a/gitnexus/test/unit/wiki-llm-client.test.ts +++ b/gitnexus/test/unit/wiki-llm-client.test.ts @@ -2,9 +2,12 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; // Import the function we'll add in the next step import { + DEFAULT_MAX_COMPLETION_TOKENS, LLM_ALLOW_INSECURE_CONNECTION_ENV, + LLM_MAX_COMPLETION_TOKENS_ENV, MINIMAX_MODEL_IDS, MINIMAX_OPENAI_BASE_URLS, + callLLM, isAzureProvider, isReasoningModel, buildRequestUrl, @@ -130,6 +133,97 @@ describe('resolveLLMConfig provider isolation', () => { }); }); +describe('resolveLLMConfig max completion tokens', () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + async function mockSavedConfig(config: Record) { + vi.spyOn(await import('../../src/storage/repo-manager.js'), 'loadCLIConfig').mockResolvedValue( + config, + ); + } + + it('keeps 16384 when the flag, env var, and config are unset', async () => { + vi.stubEnv(LLM_MAX_COMPLETION_TOKENS_ENV, ''); + await mockSavedConfig({}); + + const config = await resolveLLMConfig(); + + expect(config.maxTokens).toBe(DEFAULT_MAX_COMPLETION_TOKENS); + }); + + it('uses maxCompletionTokens from ~/.gitnexus/config.json', async () => { + vi.stubEnv(LLM_MAX_COMPLETION_TOKENS_ENV, ''); + await mockSavedConfig({ maxCompletionTokens: 131072 }); + + const config = await resolveLLMConfig(); + + expect(config.maxTokens).toBe(131072); + }); + + it('lets GITNEXUS_MAX_COMPLETION_TOKENS override the config file', async () => { + vi.stubEnv(LLM_MAX_COMPLETION_TOKENS_ENV, '65536'); + await mockSavedConfig({ maxCompletionTokens: 131072 }); + + const config = await resolveLLMConfig(); + + expect(config.maxTokens).toBe(65536); + }); + + it('lets an explicit override beat the env var and the config file', async () => { + vi.stubEnv(LLM_MAX_COMPLETION_TOKENS_ENV, 'nope'); + await mockSavedConfig({ maxCompletionTokens: 1.5 }); + + const config = await resolveLLMConfig({ maxTokens: 131072 }); + + expect(config.maxTokens).toBe(131072); + }); + + it('sends the env limit as max_completion_tokens', async () => { + vi.stubEnv(LLM_MAX_COMPLETION_TOKENS_ENV, '131072'); + await mockSavedConfig({}); + const fetchSpy = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ choices: [{ message: { content: 'ok' } }], usage: {} }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchSpy); + + const config = await resolveLLMConfig({ + apiKey: 'sk-test', + baseUrl: 'https://api.openai.com/v1', + model: 'gpt-4o', + provider: 'openai', + }); + await callLLM('test', config); + + const body = JSON.parse((fetchSpy.mock.calls[0] as [string, RequestInit])[1].body as string); + expect(body.max_completion_tokens).toBe(131072); + }); + + it('rejects a non-integer env value', async () => { + vi.stubEnv(LLM_MAX_COMPLETION_TOKENS_ENV, 'nope'); + await mockSavedConfig({}); + + await expect(resolveLLMConfig()).rejects.toThrow( + `${LLM_MAX_COMPLETION_TOKENS_ENV} must be a positive integer`, + ); + }); + + it('rejects a non-integer config value', async () => { + vi.stubEnv(LLM_MAX_COMPLETION_TOKENS_ENV, ''); + await mockSavedConfig({ maxCompletionTokens: 1.5 }); + + await expect(resolveLLMConfig()).rejects.toThrow( + 'maxCompletionTokens in ~/.gitnexus/config.json must be a positive integer', + ); + }); +}); + describe('callLLM — auth header', () => { afterEach(() => vi.unstubAllGlobals()); diff --git a/gitnexus/test/unit/wiki-llm-truncation.test.ts b/gitnexus/test/unit/wiki-llm-truncation.test.ts new file mode 100644 index 000000000..37dabb22a --- /dev/null +++ b/gitnexus/test/unit/wiki-llm-truncation.test.ts @@ -0,0 +1,150 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { callLLM, type LLMConfig } from '../../src/core/wiki/llm-client.js'; + +const config: LLMConfig = { + apiKey: 'sk-test', + baseUrl: 'https://api.openai.com/v1', + model: 'gpt-4o', + maxTokens: 16_384, + temperature: 0, +}; + +function jsonResponse(choice: Record): Response { + return new Response(JSON.stringify({ choices: [choice], usage: {} }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function sseResponse(chunks: Array<{ content?: string; finish_reason: string | null }>): Response { + const lines = chunks.map( + ({ content, finish_reason }) => + `data: ${JSON.stringify({ choices: [{ delta: content ? { content } : {}, finish_reason }] })}\n\n`, + ); + const encoder = new TextEncoder(); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode([...lines, 'data: [DONE]\n\n'].join(''))); + controller.close(); + }, + }); + return new Response(body, { status: 200, headers: { 'Content-Type': 'text/event-stream' } }); +} + +const LIMIT_ERROR = + 'LLM response stopped at the completion token limit (16384, finish_reason "length") and is incomplete.'; + +describe('callLLM — responses cut off by the completion limit', () => { + afterEach(() => vi.unstubAllGlobals()); + + it('rejects a non-streaming answer that stopped at the limit', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + jsonResponse({ + message: { content: '| col | col |\n| --- | -' }, + finish_reason: 'length', + }), + ), + ); + + await expect(callLLM('prompt', config)).rejects.toThrow(LIMIT_ERROR); + }); + + it('names the limit when reasoning used it all and no answer came back', async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValue(jsonResponse({ message: { content: '' }, finish_reason: 'length' })), + ); + + await expect(callLLM('prompt', config)).rejects.toThrow(LIMIT_ERROR); + }); + + it('rejects a streamed answer that stopped at the limit', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + sseResponse([ + { content: '# Module\n\n| col | col |\n', finish_reason: null }, + { content: '| --- | -', finish_reason: 'length' }, + ]), + ), + ); + const onChunk = vi.fn(); + + await expect(callLLM('prompt', config, undefined, { onChunk })).rejects.toThrow(LIMIT_ERROR); + expect(onChunk).toHaveBeenCalled(); + }); + + it('keeps answers that finished on their own', async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce( + jsonResponse({ message: { content: 'done' }, finish_reason: 'stop' }), + ) + .mockResolvedValueOnce( + sseResponse([ + { content: 'do', finish_reason: null }, + { content: 'ne', finish_reason: 'stop' }, + ]), + ), + ); + + await expect(callLLM('prompt', config)).resolves.toMatchObject({ content: 'done' }); + await expect( + callLLM('prompt', config, undefined, { onChunk: () => {} }), + ).resolves.toMatchObject({ content: 'done' }); + }); + + it.each([false, true])('reports the configured limit (streaming: %s)', async (streaming) => { + const limit = 65_536; + const fetchMock = vi + .fn() + .mockResolvedValue( + streaming + ? sseResponse([{ finish_reason: 'length' }]) + : jsonResponse({ message: { content: '' }, finish_reason: 'length' }), + ); + vi.stubGlobal('fetch', fetchMock); + + await expect( + callLLM('prompt', { ...config, maxTokens: limit }, undefined, { + onChunk: streaming ? vi.fn() : undefined, + }), + ).rejects.toThrow( + 'LLM response stopped at the completion token limit (65536, finish_reason "length")', + ); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(JSON.parse(fetchMock.mock.calls[0][1].body).max_completion_tokens).toBe(limit); + }); + + it('recognizes a separate finish chunk split across network reads, followed by usage', async () => { + const chunks = [ + { choices: [{ delta: { content: 'Partial answer' }, finish_reason: null }] }, + { choices: [{ delta: {}, finish_reason: 'length' }] }, + { choices: [], usage: { completion_tokens: 16_384 } }, + ]; + const bytes = new TextEncoder().encode( + chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\r\n\r\n`).join('') + + 'data: [DONE]\r\n\r\n', + ); + const body = new ReadableStream({ + start(controller) { + for (let i = 0; i < bytes.length; i += 7) { + controller.enqueue(bytes.slice(i, i + 7)); + } + controller.close(); + }, + }); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(body))); + + await expect(callLLM('prompt', config, undefined, { onChunk: vi.fn() })).rejects.toThrow( + LIMIT_ERROR, + ); + }); +}); diff --git a/gitnexus/test/unit/wiki-truncation-recovery.test.ts b/gitnexus/test/unit/wiki-truncation-recovery.test.ts new file mode 100644 index 000000000..ad54926eb --- /dev/null +++ b/gitnexus/test/unit/wiki-truncation-recovery.test.ts @@ -0,0 +1,230 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { WikiGenerator, type WikiOptions } from '../../src/core/wiki/generator.js'; + +vi.mock('../../src/core/wiki/graph-queries.js', () => ({ + initWikiDb: vi.fn().mockResolvedValue(undefined), + closeWikiDb: vi.fn().mockResolvedValue(undefined), + touchWikiDb: vi.fn(), + pinWikiDb: vi.fn(() => vi.fn()), + getFilesWithExports: vi.fn().mockResolvedValue([]), + getAllFiles: vi.fn().mockResolvedValue(['billing.ts', 'shipping.ts']), + getIntraModuleCallEdges: vi.fn().mockResolvedValue([]), + getInterModuleCallEdges: vi.fn().mockResolvedValue({ incoming: [], outgoing: [] }), + getProcessesForFiles: vi.fn().mockResolvedValue([]), + getAllProcesses: vi.fn().mockResolvedValue([]), + getInterModuleEdgesForOverview: vi.fn().mockResolvedValue([]), +})); + +function response(content: string, finishReason = 'stop'): Response { + const chunk = { choices: [{ delta: { content }, finish_reason: finishReason }] }; + return new Response(`data: ${JSON.stringify(chunk)}\n\ndata: [DONE]\n\n`, { + headers: { 'Content-Type': 'text/event-stream' }, + }); +} + +describe('wiki generation after a completion-token limit', () => { + let repoPath: string; + let storagePath: string; + let wikiPath: string; + let fetchMock: ReturnType; + + function git(...args: string[]): string { + return execFileSync('git', args, { cwd: repoPath, encoding: 'utf8' }).trim(); + } + + function commit(): string { + git('add', 'billing.ts', 'shipping.ts'); + git( + '-c', + 'user.name=Wiki Test', + '-c', + 'user.email=wiki@example.test', + 'commit', + '-qm', + 'fixture', + ); + return git('rev-parse', 'HEAD'); + } + + function generator(options: WikiOptions = {}): WikiGenerator { + return new WikiGenerator( + repoPath, + storagePath, + '/unused/lbug', + { + apiKey: 'test', + baseUrl: 'https://wiki.example.test/v1', + model: 'test-model', + maxTokens: 65_536, + temperature: 0, + provider: 'custom', + }, + { concurrency: 1, ...options }, + ); + } + + async function metadata() { + return JSON.parse(await fs.readFile(path.join(wikiPath, 'meta.json'), 'utf8')); + } + + beforeEach(async () => { + repoPath = await fs.mkdtemp(path.join(os.tmpdir(), 'wiki-truncation-')); + storagePath = path.join(repoPath, '.gitnexus'); + wikiPath = path.join(storagePath, 'wiki'); + await fs.mkdir(wikiPath, { recursive: true }); + await fs.writeFile(path.join(repoPath, 'billing.ts'), 'export const bill = 1;\n'); + await fs.writeFile(path.join(repoPath, 'shipping.ts'), 'export const ship = 1;\n'); + git('init', '-q'); + commit(); + await fs.writeFile( + path.join(wikiPath, 'module_tree.json'), + JSON.stringify([ + { name: 'Billing', slug: 'billing', files: ['billing.ts'] }, + { name: 'Shipping', slug: 'shipping', files: ['shipping.ts'] }, + ]), + ); + fetchMock = vi.fn().mockImplementation(async () => response('Complete documentation.')); + vi.stubGlobal('fetch', fetchMock); + }); + + afterEach(async () => { + vi.unstubAllGlobals(); + await fs.rm(repoPath, { recursive: true, force: true }); + }); + + it('does not save a truncated page and retries it at the same commit', async () => { + fetchMock.mockResolvedValueOnce(response('Incomplete table | --- | -', 'length')); + + expect(await generator().run()).toEqual({ + pagesGenerated: 2, + mode: 'full', + failedModules: ['Billing'], + }); + await expect(fs.readFile(path.join(wikiPath, 'billing.md'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + expect((await metadata()).failedModules).toEqual(['Billing']); + const shippingBefore = await fs.readFile(path.join(wikiPath, 'shipping.md'), 'utf8'); + fetchMock.mockClear(); + + expect(await generator().run()).toEqual({ + pagesGenerated: 2, + mode: 'incremental', + failedModules: [], + }); + expect(fetchMock).toHaveBeenCalledTimes(2); + for (const [, init] of fetchMock.mock.calls) { + expect(JSON.parse(init.body).max_completion_tokens).toBe(65_536); + } + expect(await fs.readFile(path.join(wikiPath, 'billing.md'), 'utf8')).toContain( + 'Complete documentation.', + ); + expect(await fs.readFile(path.join(wikiPath, 'shipping.md'), 'utf8')).toBe(shippingBefore); + expect((await metadata()).failedModules).toEqual([]); + expect(await generator().run()).toEqual({ + pagesGenerated: 0, + mode: 'up-to-date', + failedModules: [], + }); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('retries a truncated incremental page after the new commit was recorded', async () => { + await generator().run(); + await fs.writeFile(path.join(repoPath, 'billing.ts'), 'export const bill = 2;\n'); + const currentCommit = commit(); + fetchMock.mockClear(); + fetchMock.mockResolvedValueOnce(response('', 'length')); + + expect(await generator().run()).toEqual({ + pagesGenerated: 0, + mode: 'incremental', + failedModules: ['Billing'], + }); + expect(await metadata()).toMatchObject({ + fromCommit: currentCommit, + failedModules: ['Billing'], + }); + expect(await generator().run()).toEqual({ + pagesGenerated: 2, + mode: 'incremental', + failedModules: [], + }); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('retries failed forced generation even when an older wiki covers the same commit', async () => { + await generator().run(); + fetchMock.mockClear(); + fetchMock.mockResolvedValueOnce(response('cut off', 'length')); + + expect((await generator({ force: true }).run()).failedModules).toEqual(['Billing']); + expect((await generator().run()).mode).toBe('incremental'); + expect((await metadata()).failedModules).toEqual([]); + expect(await fs.readFile(path.join(wikiPath, 'billing.md'), 'utf8')).toContain( + 'Complete documentation.', + ); + }); + + it('waits for a failed child to recover before generating its parent page', async () => { + await fs.writeFile( + path.join(wikiPath, 'module_tree.json'), + JSON.stringify([ + { + name: 'Commerce', + slug: 'commerce', + files: [], + children: [ + { name: 'Billing', slug: 'billing', files: ['billing.ts'] }, + { name: 'Shipping', slug: 'shipping', files: ['shipping.ts'] }, + ], + }, + ]), + ); + fetchMock.mockResolvedValueOnce(response('cut off', 'length')); + + expect((await generator().run()).failedModules).toEqual(['Billing', 'Commerce']); + await expect(fs.readFile(path.join(wikiPath, 'commerce.md'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + fetchMock.mockClear(); + + expect(await generator({ concurrency: 3 }).run()).toEqual({ + pagesGenerated: 3, + mode: 'incremental', + failedModules: [], + }); + expect(fetchMock).toHaveBeenCalledTimes(3); + const parentPrompt = JSON.parse(fetchMock.mock.calls[1][1].body).messages[1].content; + expect(parentPrompt).toContain('Complete documentation.'); + expect(parentPrompt).not.toContain('Documentation pending'); + expect((await metadata()).failedModules).toEqual([]); + }); + + it('does not retain a completion receipt when the forced overview is truncated', async () => { + await generator().run(); + fetchMock.mockClear(); + fetchMock + .mockResolvedValueOnce(response('Billing complete')) + .mockResolvedValueOnce(response('Shipping complete')) + .mockResolvedValueOnce(response('overview cut off', 'length')); + + await expect(generator({ force: true }).run()).rejects.toThrow('completion token limit'); + await expect(fs.readFile(path.join(wikiPath, 'meta.json'))).rejects.toMatchObject({ + code: 'ENOENT', + }); + expect(await generator().run()).toEqual({ + pagesGenerated: 1, + mode: 'full', + failedModules: [], + }); + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(await fs.readFile(path.join(wikiPath, 'overview.md'), 'utf8')).toContain( + 'Complete documentation.', + ); + }); +});