From 14a9ccfa871c3cf17976ad31999b6f91a2945473 Mon Sep 17 00:00:00 2001 From: Roo Code Date: Tue, 30 Dec 2025 05:07:57 +0000 Subject: [PATCH] feat: add configurable max batch retries for scanner This PR addresses Issue #10396 by adding a user-configurable setting for the maximum number of batch retries during codebase indexing. Changes: - Add batch retry constants to CODEBASE_INDEX_DEFAULTS (min: 1, max: 10, default: 3) - Add codebaseIndexMaxBatchRetries field to config schema - Update config-manager.ts to expose currentMaxBatchRetries getter - Update scanner.ts to accept and use configurable maxBatchRetries - Update service-factory.ts to pass configured value to DirectoryScanner - Add UI slider in Advanced Settings section of CodeIndexPopover - Add i18n translation strings for the new setting --- packages/types/src/codebase-index.ts | 10 ++++ src/services/code-index/config-manager.ts | 15 +++++- src/services/code-index/processors/scanner.ts | 12 +++-- src/services/code-index/service-factory.ts | 12 ++++- .../src/components/chat/CodeIndexPopover.tsx | 48 +++++++++++++++++++ webview-ui/src/i18n/locales/en/settings.json | 2 + 6 files changed, 92 insertions(+), 7 deletions(-) diff --git a/packages/types/src/codebase-index.ts b/packages/types/src/codebase-index.ts index 61009ba301..486ef9b76a 100644 --- a/packages/types/src/codebase-index.ts +++ b/packages/types/src/codebase-index.ts @@ -12,6 +12,11 @@ export const CODEBASE_INDEX_DEFAULTS = { MAX_SEARCH_SCORE: 1, DEFAULT_SEARCH_MIN_SCORE: 0.4, SEARCH_SCORE_STEP: 0.05, + // Batch retry settings + MIN_BATCH_RETRIES: 1, + MAX_BATCH_RETRIES: 10, + DEFAULT_BATCH_RETRIES: 3, + BATCH_RETRIES_STEP: 1, } as const /** @@ -42,6 +47,11 @@ export const codebaseIndexConfigSchema = z.object({ .min(CODEBASE_INDEX_DEFAULTS.MIN_SEARCH_RESULTS) .max(CODEBASE_INDEX_DEFAULTS.MAX_SEARCH_RESULTS) .optional(), + codebaseIndexMaxBatchRetries: z + .number() + .min(CODEBASE_INDEX_DEFAULTS.MIN_BATCH_RETRIES) + .max(CODEBASE_INDEX_DEFAULTS.MAX_BATCH_RETRIES) + .optional(), // OpenAI Compatible specific fields codebaseIndexOpenAiCompatibleBaseUrl: z.string().optional(), codebaseIndexOpenAiCompatibleModelDimension: z.number().optional(), diff --git a/src/services/code-index/config-manager.ts b/src/services/code-index/config-manager.ts index e7f239e621..a63d6f430d 100644 --- a/src/services/code-index/config-manager.ts +++ b/src/services/code-index/config-manager.ts @@ -2,7 +2,7 @@ import { ApiHandlerOptions } from "../../shared/api" import { ContextProxy } from "../../core/config/ContextProxy" import { EmbedderProvider } from "./interfaces/manager" import { CodeIndexConfig, PreviousConfigSnapshot } from "./interfaces/config" -import { DEFAULT_SEARCH_MIN_SCORE, DEFAULT_MAX_SEARCH_RESULTS } from "./constants" +import { DEFAULT_SEARCH_MIN_SCORE, DEFAULT_MAX_SEARCH_RESULTS, MAX_BATCH_RETRIES } from "./constants" import { getDefaultModelId, getModelDimension, getModelScoreThreshold } from "../../shared/embeddingModels" /** @@ -26,6 +26,7 @@ export class CodeIndexConfigManager { private qdrantApiKey?: string private searchMinScore?: number private searchMaxResults?: number + private maxBatchRetries?: number constructor(private readonly contextProxy: ContextProxy) { // Initialize with current configuration to avoid false restart triggers @@ -65,7 +66,8 @@ export class CodeIndexConfigManager { codebaseIndexEmbedderModelId, codebaseIndexSearchMinScore, codebaseIndexSearchMaxResults, - } = codebaseIndexConfig + codebaseIndexMaxBatchRetries, + } = codebaseIndexConfig as any const openAiKey = this.contextProxy?.getSecret("codeIndexOpenAiKey") ?? "" const qdrantApiKey = this.contextProxy?.getSecret("codeIndexQdrantApiKey") ?? "" @@ -86,6 +88,7 @@ export class CodeIndexConfigManager { this.qdrantApiKey = qdrantApiKey ?? "" this.searchMinScore = codebaseIndexSearchMinScore this.searchMaxResults = codebaseIndexSearchMaxResults + this.maxBatchRetries = codebaseIndexMaxBatchRetries // Validate and set model dimension const rawDimension = codebaseIndexConfig.codebaseIndexEmbedderModelDimension @@ -541,4 +544,12 @@ export class CodeIndexConfigManager { public get currentSearchMaxResults(): number { return this.searchMaxResults ?? DEFAULT_MAX_SEARCH_RESULTS } + + /** + * Gets the configured maximum batch retries for indexing. + * Returns user setting if configured, otherwise returns default. + */ + public get currentMaxBatchRetries(): number { + return this.maxBatchRetries ?? MAX_BATCH_RETRIES + } } diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index 92a7d77c27..aebb448cb1 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -33,6 +33,7 @@ import { Package } from "../../../shared/package" export class DirectoryScanner implements IDirectoryScanner { private readonly batchSegmentThreshold: number + private readonly maxBatchRetries: number constructor( private readonly embedder: IEmbedder, @@ -41,6 +42,7 @@ export class DirectoryScanner implements IDirectoryScanner { private readonly cacheManager: CacheManager, private readonly ignoreInstance: Ignore, batchSegmentThreshold?: number, + maxBatchRetries?: number, ) { // Get the configurable batch size from VSCode settings, fallback to default // If not provided in constructor, try to get from VSCode settings @@ -56,6 +58,8 @@ export class DirectoryScanner implements IDirectoryScanner { this.batchSegmentThreshold = BATCH_SEGMENT_THRESHOLD } } + // Set max batch retries from parameter or use default constant + this.maxBatchRetries = maxBatchRetries ?? MAX_BATCH_RETRIES } /** @@ -360,7 +364,7 @@ export class DirectoryScanner implements IDirectoryScanner { let success = false let lastError: Error | null = null - while (attempts < MAX_BATCH_RETRIES && !success) { + while (attempts < this.maxBatchRetries && !success) { attempts++ try { // --- Deletion Step --- @@ -450,7 +454,7 @@ export class DirectoryScanner implements IDirectoryScanner { batchSize: batchBlocks.length, }) - if (attempts < MAX_BATCH_RETRIES) { + if (attempts < this.maxBatchRetries) { const delay = INITIAL_RETRY_DELAY_MS * Math.pow(2, attempts - 1) await new Promise((resolve) => setTimeout(resolve, delay)) } @@ -458,7 +462,7 @@ export class DirectoryScanner implements IDirectoryScanner { } if (!success && lastError) { - console.error(`[DirectoryScanner] Failed to process batch after ${MAX_BATCH_RETRIES} attempts`) + console.error(`[DirectoryScanner] Failed to process batch after ${this.maxBatchRetries} attempts`) if (onError) { // Preserve the original error message from embedders which now have detailed i18n messages const errorMessage = lastError.message || "Unknown error" @@ -467,7 +471,7 @@ export class DirectoryScanner implements IDirectoryScanner { onError( new Error( t("embeddings:scanner.failedToProcessBatchWithError", { - maxRetries: MAX_BATCH_RETRIES, + maxRetries: this.maxBatchRetries, errorMessage, }), ), diff --git a/src/services/code-index/service-factory.ts b/src/services/code-index/service-factory.ts index c98c65d4c1..a1cf5ba9df 100644 --- a/src/services/code-index/service-factory.ts +++ b/src/services/code-index/service-factory.ts @@ -186,7 +186,17 @@ export class CodeIndexServiceFactory { // In test environment, vscode.workspace might not be available batchSize = BATCH_SEGMENT_THRESHOLD } - return new DirectoryScanner(embedder, vectorStore, parser, this.cacheManager, ignoreInstance, batchSize) + // Get max batch retries from config manager + const maxBatchRetries = this.configManager.currentMaxBatchRetries + return new DirectoryScanner( + embedder, + vectorStore, + parser, + this.cacheManager, + ignoreInstance, + batchSize, + maxBatchRetries, + ) } /** diff --git a/webview-ui/src/components/chat/CodeIndexPopover.tsx b/webview-ui/src/components/chat/CodeIndexPopover.tsx index 368f0395ea..a65da0ca55 100644 --- a/webview-ui/src/components/chat/CodeIndexPopover.tsx +++ b/webview-ui/src/components/chat/CodeIndexPopover.tsx @@ -69,6 +69,7 @@ interface LocalCodeIndexSettings { codebaseIndexEmbedderModelDimension?: number // Generic dimension for all providers codebaseIndexSearchMaxResults?: number codebaseIndexSearchMinScore?: number + codebaseIndexMaxBatchRetries?: number // Bedrock-specific settings codebaseIndexBedrockRegion?: string @@ -217,6 +218,7 @@ export const CodeIndexPopover: React.FC = ({ codebaseIndexEmbedderModelDimension: undefined, codebaseIndexSearchMaxResults: CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_RESULTS, codebaseIndexSearchMinScore: CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_MIN_SCORE, + codebaseIndexMaxBatchRetries: CODEBASE_INDEX_DEFAULTS.DEFAULT_BATCH_RETRIES, codebaseIndexBedrockRegion: "", codebaseIndexBedrockProfile: "", codeIndexOpenAiKey: "", @@ -256,6 +258,8 @@ export const CodeIndexPopover: React.FC = ({ codebaseIndexConfig.codebaseIndexSearchMaxResults ?? CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_RESULTS, codebaseIndexSearchMinScore: codebaseIndexConfig.codebaseIndexSearchMinScore ?? CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_MIN_SCORE, + codebaseIndexMaxBatchRetries: + codebaseIndexConfig.codebaseIndexMaxBatchRetries ?? CODEBASE_INDEX_DEFAULTS.DEFAULT_BATCH_RETRIES, codebaseIndexBedrockRegion: codebaseIndexConfig.codebaseIndexBedrockRegion || "", codebaseIndexBedrockProfile: codebaseIndexConfig.codebaseIndexBedrockProfile || "", codeIndexOpenAiKey: "", @@ -1589,6 +1593,50 @@ export const CodeIndexPopover: React.FC = ({ + + {/* Maximum Batch Retries Slider */} +
+
+ + + + +
+
+ + updateSetting("codebaseIndexMaxBatchRetries", values[0]) + } + className="flex-1" + data-testid="max-batch-retries-slider" + /> + + {currentSettings.codebaseIndexMaxBatchRetries ?? + CODEBASE_INDEX_DEFAULTS.DEFAULT_BATCH_RETRIES} + + + updateSetting( + "codebaseIndexMaxBatchRetries", + CODEBASE_INDEX_DEFAULTS.DEFAULT_BATCH_RETRIES, + ) + }> + + +
+
)} diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index c7fa7e0794..00fe834f27 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -133,6 +133,8 @@ "searchMinScoreResetTooltip": "Reset to default value (0.4)", "searchMaxResultsLabel": "Maximum Search Results", "searchMaxResultsDescription": "Maximum number of search results to return when querying the codebase index. Higher values provide more context but may include less relevant results.", + "maxBatchRetriesLabel": "Maximum Batch Retries", + "maxBatchRetriesDescription": "Maximum number of retry attempts for failed batch operations during indexing. Higher values improve reliability on unstable connections but may delay error detection.", "resetToDefault": "Reset to default", "startIndexingButton": "Start Indexing", "clearIndexDataButton": "Clear Index Data",