diff --git a/packages/types/src/codebase-index.ts b/packages/types/src/codebase-index.ts index 61009ba301..ea81d125ac 100644 --- a/packages/types/src/codebase-index.ts +++ b/packages/types/src/codebase-index.ts @@ -12,6 +12,21 @@ export const CODEBASE_INDEX_DEFAULTS = { MAX_SEARCH_SCORE: 1, DEFAULT_SEARCH_MIN_SCORE: 0.4, SEARCH_SCORE_STEP: 0.05, + // Embedding batch size settings + MIN_EMBEDDING_BATCH_SIZE: 10, + MAX_EMBEDDING_BATCH_SIZE: 200, + DEFAULT_EMBEDDING_BATCH_SIZE: 60, + EMBEDDING_BATCH_SIZE_STEP: 10, + // Max chunk size settings (characters per code chunk) + MIN_MAX_CHUNK_SIZE: 200, + MAX_MAX_CHUNK_SIZE: 5000, + DEFAULT_MAX_CHUNK_SIZE: 1000, + MAX_CHUNK_SIZE_STEP: 100, + // Parsing concurrency settings (concurrent file parsing) + MIN_PARSING_CONCURRENCY: 1, + MAX_PARSING_CONCURRENCY: 50, + DEFAULT_PARSING_CONCURRENCY: 10, + PARSING_CONCURRENCY_STEP: 1, } as const /** @@ -42,6 +57,22 @@ export const codebaseIndexConfigSchema = z.object({ .min(CODEBASE_INDEX_DEFAULTS.MIN_SEARCH_RESULTS) .max(CODEBASE_INDEX_DEFAULTS.MAX_SEARCH_RESULTS) .optional(), + // Advanced indexing parameters + codebaseIndexEmbeddingBatchSize: z + .number() + .min(CODEBASE_INDEX_DEFAULTS.MIN_EMBEDDING_BATCH_SIZE) + .max(CODEBASE_INDEX_DEFAULTS.MAX_EMBEDDING_BATCH_SIZE) + .optional(), + codebaseIndexMaxChunkSize: z + .number() + .min(CODEBASE_INDEX_DEFAULTS.MIN_MAX_CHUNK_SIZE) + .max(CODEBASE_INDEX_DEFAULTS.MAX_MAX_CHUNK_SIZE) + .optional(), + codebaseIndexParsingConcurrency: z + .number() + .min(CODEBASE_INDEX_DEFAULTS.MIN_PARSING_CONCURRENCY) + .max(CODEBASE_INDEX_DEFAULTS.MAX_PARSING_CONCURRENCY) + .optional(), // OpenAI Compatible specific fields codebaseIndexOpenAiCompatibleBaseUrl: z.string().optional(), codebaseIndexOpenAiCompatibleModelDimension: z.number().optional(), diff --git a/src/services/code-index/config-manager.ts b/src/services/code-index/config-manager.ts index e7f239e621..0c428532c1 100644 --- a/src/services/code-index/config-manager.ts +++ b/src/services/code-index/config-manager.ts @@ -2,7 +2,13 @@ import { ApiHandlerOptions } from "../../shared/api" import { ContextProxy } from "../../core/config/ContextProxy" import { EmbedderProvider } from "./interfaces/manager" import { CodeIndexConfig, PreviousConfigSnapshot } from "./interfaces/config" -import { DEFAULT_SEARCH_MIN_SCORE, DEFAULT_MAX_SEARCH_RESULTS } from "./constants" +import { + DEFAULT_SEARCH_MIN_SCORE, + DEFAULT_MAX_SEARCH_RESULTS, + BATCH_SEGMENT_THRESHOLD, + MAX_BLOCK_CHARS, + PARSING_CONCURRENCY, +} from "./constants" import { getDefaultModelId, getModelDimension, getModelScoreThreshold } from "../../shared/embeddingModels" /** @@ -26,6 +32,10 @@ export class CodeIndexConfigManager { private qdrantApiKey?: string private searchMinScore?: number private searchMaxResults?: number + // Advanced indexing parameters + private embeddingBatchSize?: number + private maxChunkSize?: number + private parsingConcurrency?: number constructor(private readonly contextProxy: ContextProxy) { // Initialize with current configuration to avoid false restart triggers @@ -87,6 +97,11 @@ export class CodeIndexConfigManager { this.searchMinScore = codebaseIndexSearchMinScore this.searchMaxResults = codebaseIndexSearchMaxResults + // Load advanced indexing parameters + this.embeddingBatchSize = codebaseIndexConfig.codebaseIndexEmbeddingBatchSize + this.maxChunkSize = codebaseIndexConfig.codebaseIndexMaxChunkSize + this.parsingConcurrency = codebaseIndexConfig.codebaseIndexParsingConcurrency + // Validate and set model dimension const rawDimension = codebaseIndexConfig.codebaseIndexEmbedderModelDimension if (rawDimension !== undefined && rawDimension !== null) { @@ -460,6 +475,10 @@ export class CodeIndexConfigManager { qdrantApiKey: this.qdrantApiKey, searchMinScore: this.currentSearchMinScore, searchMaxResults: this.currentSearchMaxResults, + // Advanced indexing parameters + embeddingBatchSize: this.currentEmbeddingBatchSize, + maxChunkSize: this.currentMaxChunkSize, + parsingConcurrency: this.currentParsingConcurrency, } } @@ -541,4 +560,28 @@ export class CodeIndexConfigManager { public get currentSearchMaxResults(): number { return this.searchMaxResults ?? DEFAULT_MAX_SEARCH_RESULTS } + + /** + * Gets the configured embedding batch size. + * Returns user setting if configured, otherwise returns default. + */ + public get currentEmbeddingBatchSize(): number { + return this.embeddingBatchSize ?? BATCH_SEGMENT_THRESHOLD + } + + /** + * Gets the configured max chunk size (characters per code chunk). + * Returns user setting if configured, otherwise returns default. + */ + public get currentMaxChunkSize(): number { + return this.maxChunkSize ?? MAX_BLOCK_CHARS + } + + /** + * Gets the configured parsing concurrency (concurrent file parsing). + * Returns user setting if configured, otherwise returns default. + */ + public get currentParsingConcurrency(): number { + return this.parsingConcurrency ?? PARSING_CONCURRENCY + } } diff --git a/src/services/code-index/constants/index.ts b/src/services/code-index/constants/index.ts index 6f0e0fe7e6..a69015da00 100644 --- a/src/services/code-index/constants/index.ts +++ b/src/services/code-index/constants/index.ts @@ -1,7 +1,7 @@ import { CODEBASE_INDEX_DEFAULTS } from "@roo-code/types" /**Parser */ -export const MAX_BLOCK_CHARS = 1000 +export const MAX_BLOCK_CHARS = CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE export const MIN_BLOCK_CHARS = 50 export const MIN_CHUNK_REMAINDER_CHARS = 200 // Minimum characters for the *next* chunk after a split export const MAX_CHARS_TOLERANCE_FACTOR = 1.15 // 15% tolerance for max chars @@ -16,10 +16,10 @@ export const MAX_FILE_SIZE_BYTES = 1 * 1024 * 1024 // 1MB /**Directory Scanner */ export const MAX_LIST_FILES_LIMIT_CODE_INDEX = 50_000 -export const BATCH_SEGMENT_THRESHOLD = 60 // Number of code segments to batch for embeddings/upserts +export const BATCH_SEGMENT_THRESHOLD = CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE // Number of code segments to batch for embeddings/upserts export const MAX_BATCH_RETRIES = 3 export const INITIAL_RETRY_DELAY_MS = 500 -export const PARSING_CONCURRENCY = 10 +export const PARSING_CONCURRENCY = CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY export const MAX_PENDING_BATCHES = 20 // Maximum number of batches to accumulate before waiting /**OpenAI Embedder */ diff --git a/src/services/code-index/interfaces/config.ts b/src/services/code-index/interfaces/config.ts index f52f98aaa0..a4bc14f01b 100644 --- a/src/services/code-index/interfaces/config.ts +++ b/src/services/code-index/interfaces/config.ts @@ -21,6 +21,10 @@ export interface CodeIndexConfig { qdrantApiKey?: string searchMinScore?: number searchMaxResults?: number + // Advanced indexing parameters + embeddingBatchSize?: number + maxChunkSize?: number + parsingConcurrency?: number } /** diff --git a/src/services/code-index/processors/scanner.ts b/src/services/code-index/processors/scanner.ts index 92a7d77c27..b9e8f3cea0 100644 --- a/src/services/code-index/processors/scanner.ts +++ b/src/services/code-index/processors/scanner.ts @@ -33,6 +33,7 @@ import { Package } from "../../../shared/package" export class DirectoryScanner implements IDirectoryScanner { private readonly batchSegmentThreshold: number + private readonly parsingConcurrencyLimit: number constructor( private readonly embedder: IEmbedder, @@ -41,6 +42,7 @@ export class DirectoryScanner implements IDirectoryScanner { private readonly cacheManager: CacheManager, private readonly ignoreInstance: Ignore, batchSegmentThreshold?: number, + parsingConcurrencyLimit?: number, ) { // Get the configurable batch size from VSCode settings, fallback to default // If not provided in constructor, try to get from VSCode settings @@ -56,6 +58,8 @@ export class DirectoryScanner implements IDirectoryScanner { this.batchSegmentThreshold = BATCH_SEGMENT_THRESHOLD } } + // Set parsing concurrency (default from constants if not provided) + this.parsingConcurrencyLimit = parsingConcurrencyLimit ?? PARSING_CONCURRENCY } /** @@ -109,7 +113,7 @@ export class DirectoryScanner implements IDirectoryScanner { let skippedCount = 0 // Initialize parallel processing tools - const parseLimiter = pLimit(PARSING_CONCURRENCY) // Concurrency for file parsing + const parseLimiter = pLimit(this.parsingConcurrencyLimit) // Concurrency for file parsing const batchLimiter = pLimit(BATCH_PROCESSING_CONCURRENCY) // Concurrency for batch processing const mutex = new Mutex() diff --git a/src/services/code-index/service-factory.ts b/src/services/code-index/service-factory.ts index c98c65d4c1..6c38e30726 100644 --- a/src/services/code-index/service-factory.ts +++ b/src/services/code-index/service-factory.ts @@ -176,17 +176,20 @@ export class CodeIndexServiceFactory { parser: ICodeParser, ignoreInstance: Ignore, ): DirectoryScanner { - // Get the configurable batch size from VSCode settings - let batchSize: number - try { - batchSize = vscode.workspace - .getConfiguration(Package.name) - .get("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) - } catch { - // In test environment, vscode.workspace might not be available - batchSize = BATCH_SEGMENT_THRESHOLD - } - return new DirectoryScanner(embedder, vectorStore, parser, this.cacheManager, ignoreInstance, batchSize) + // Get the configurable settings from config manager + const config = this.configManager.getConfig() + const batchSize = config.embeddingBatchSize ?? BATCH_SEGMENT_THRESHOLD + const parsingConcurrency = config.parsingConcurrency + + return new DirectoryScanner( + embedder, + vectorStore, + parser, + this.cacheManager, + ignoreInstance, + batchSize, + parsingConcurrency, + ) } /** @@ -200,16 +203,10 @@ export class CodeIndexServiceFactory { ignoreInstance: Ignore, rooIgnoreController?: RooIgnoreController, ): IFileWatcher { - // Get the configurable batch size from VSCode settings - let batchSize: number - try { - batchSize = vscode.workspace - .getConfiguration(Package.name) - .get("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) - } catch { - // In test environment, vscode.workspace might not be available - batchSize = BATCH_SEGMENT_THRESHOLD - } + // Get the configurable settings from config manager + const config = this.configManager.getConfig() + const batchSize = config.embeddingBatchSize ?? BATCH_SEGMENT_THRESHOLD + return new FileWatcher( this.workspacePath, context, diff --git a/webview-ui/src/components/chat/CodeIndexPopover.tsx b/webview-ui/src/components/chat/CodeIndexPopover.tsx index 368f0395ea..362da1d96b 100644 --- a/webview-ui/src/components/chat/CodeIndexPopover.tsx +++ b/webview-ui/src/components/chat/CodeIndexPopover.tsx @@ -69,6 +69,10 @@ interface LocalCodeIndexSettings { codebaseIndexEmbedderModelDimension?: number // Generic dimension for all providers codebaseIndexSearchMaxResults?: number codebaseIndexSearchMinScore?: number + // Advanced indexing parameters + codebaseIndexEmbeddingBatchSize?: number + codebaseIndexMaxChunkSize?: number + codebaseIndexParsingConcurrency?: number // Bedrock-specific settings codebaseIndexBedrockRegion?: string @@ -217,6 +221,9 @@ export const CodeIndexPopover: React.FC = ({ codebaseIndexEmbedderModelDimension: undefined, codebaseIndexSearchMaxResults: CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_RESULTS, codebaseIndexSearchMinScore: CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_MIN_SCORE, + codebaseIndexEmbeddingBatchSize: CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE, + codebaseIndexMaxChunkSize: CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE, + codebaseIndexParsingConcurrency: CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY, codebaseIndexBedrockRegion: "", codebaseIndexBedrockProfile: "", codeIndexOpenAiKey: "", @@ -256,6 +263,14 @@ export const CodeIndexPopover: React.FC = ({ codebaseIndexConfig.codebaseIndexSearchMaxResults ?? CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_RESULTS, codebaseIndexSearchMinScore: codebaseIndexConfig.codebaseIndexSearchMinScore ?? CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_MIN_SCORE, + codebaseIndexEmbeddingBatchSize: + codebaseIndexConfig.codebaseIndexEmbeddingBatchSize ?? + CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE, + codebaseIndexMaxChunkSize: + codebaseIndexConfig.codebaseIndexMaxChunkSize ?? CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE, + codebaseIndexParsingConcurrency: + codebaseIndexConfig.codebaseIndexParsingConcurrency ?? + CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY, codebaseIndexBedrockRegion: codebaseIndexConfig.codebaseIndexBedrockRegion || "", codebaseIndexBedrockProfile: codebaseIndexConfig.codebaseIndexBedrockProfile || "", codeIndexOpenAiKey: "", @@ -1589,6 +1604,138 @@ export const CodeIndexPopover: React.FC = ({ + + {/* Embedding Batch Size Slider */} +
+
+ + + + +
+
+ + updateSetting("codebaseIndexEmbeddingBatchSize", values[0]) + } + className="flex-1" + data-testid="embedding-batch-size-slider" + /> + + {currentSettings.codebaseIndexEmbeddingBatchSize ?? + CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE} + + + updateSetting( + "codebaseIndexEmbeddingBatchSize", + CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE, + ) + }> + + +
+
+ + {/* Max Chunk Size Slider */} +
+
+ + + + +
+
+ + updateSetting("codebaseIndexMaxChunkSize", values[0]) + } + className="flex-1" + data-testid="max-chunk-size-slider" + /> + + {currentSettings.codebaseIndexMaxChunkSize ?? + CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE} + + + updateSetting( + "codebaseIndexMaxChunkSize", + CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE, + ) + }> + + +
+
+ + {/* Parsing Concurrency Slider */} +
+
+ + + + +
+
+ + updateSetting("codebaseIndexParsingConcurrency", values[0]) + } + className="flex-1" + data-testid="parsing-concurrency-slider" + /> + + {currentSettings.codebaseIndexParsingConcurrency ?? + CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY} + + + updateSetting( + "codebaseIndexParsingConcurrency", + CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY, + ) + }> + + +
+
)} diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index c7fa7e0794..7ef1976e8e 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -133,6 +133,15 @@ "searchMinScoreResetTooltip": "Reset to default value (0.4)", "searchMaxResultsLabel": "Maximum Search Results", "searchMaxResultsDescription": "Maximum number of search results to return when querying the codebase index. Higher values provide more context but may include less relevant results.", + "embeddingBatchSizeLabel": "Embedding Batch Size", + "embeddingBatchSizeDescription": "Number of code segments to batch together for embeddings. Higher values can speed up indexing on powerful hardware. Lower values reduce memory usage.", + "embeddingBatchSizeResetTooltip": "Reset to default value (60)", + "maxChunkSizeLabel": "Max Chunk Size", + "maxChunkSizeDescription": "Maximum characters per code chunk. Larger chunks provide more context but may reduce search precision. Smaller chunks enable finer-grained search results.", + "maxChunkSizeResetTooltip": "Reset to default value (1000)", + "parsingConcurrencyLabel": "Parsing Concurrency", + "parsingConcurrencyDescription": "Number of files to parse concurrently during indexing. Higher values speed up indexing but use more CPU and memory.", + "parsingConcurrencyResetTooltip": "Reset to default value (10)", "resetToDefault": "Reset to default", "startIndexingButton": "Start Indexing", "clearIndexDataButton": "Clear Index Data",