diff --git a/src/package.json b/src/package.json index 023c470d4a..ca244079be 100644 --- a/src/package.json +++ b/src/package.json @@ -455,6 +455,13 @@ "type": "boolean", "default": false, "description": "%settings.codeIndex.lowResourceMode.description%" + }, + "roo-cline.codeIndex.fileProcessingConcurrency": { + "type": "number", + "default": 10, + "minimum": 1, + "maximum": 50, + "description": "%settings.codeIndex.fileProcessingConcurrency.description%" } } } diff --git a/src/package.nls.json b/src/package.nls.json index 819f36af8f..b519c7cf81 100644 --- a/src/package.nls.json +++ b/src/package.nls.json @@ -46,5 +46,6 @@ "settings.codeIndex.parsingConcurrency.description": "Maximum number of files to parse concurrently during code indexing. Lower values reduce memory and CPU usage. Default is 10.", "settings.codeIndex.batchProcessingConcurrency.description": "Maximum number of embedding batches to process concurrently. Lower values reduce API load and memory usage. Default is 10.", "settings.codeIndex.maxPendingBatches.description": "Maximum number of pending batches before throttling. Lower values reduce memory usage. Default is 20.", - "settings.codeIndex.lowResourceMode.description": "Enable low resource mode for better performance on modest hardware. This reduces concurrency, batch sizes, and memory usage." + "settings.codeIndex.lowResourceMode.description": "Enable low resource mode for better performance on modest hardware. This reduces concurrency, batch sizes, and memory usage.", + "settings.codeIndex.fileProcessingConcurrency.description": "Maximum number of files to process concurrently when watching for file changes. Lower values reduce CPU and memory usage. Default is 10." } diff --git a/src/services/code-index/constants/index.ts b/src/services/code-index/constants/index.ts index c254cc1df2..91fb7a8e53 100644 --- a/src/services/code-index/constants/index.ts +++ b/src/services/code-index/constants/index.ts @@ -22,12 +22,14 @@ export const INITIAL_RETRY_DELAY_MS = 500 export const PARSING_CONCURRENCY = 10 // Default number of files to parse concurrently export const MAX_PENDING_BATCHES = 20 // Default maximum number of batches to accumulate before waiting export const BATCH_PROCESSING_CONCURRENCY = 10 // Default number of batches to process concurrently +export const FILE_PROCESSING_CONCURRENCY = 10 // Default number of files to process concurrently in file watcher /**Low Resource Mode Defaults */ export const LOW_RESOURCE_PARSING_CONCURRENCY = 2 // Reduced concurrent file parsing for low-end hardware export const LOW_RESOURCE_BATCH_PROCESSING_CONCURRENCY = 2 // Reduced concurrent batch processing export const LOW_RESOURCE_MAX_PENDING_BATCHES = 5 // Reduced pending batches export const LOW_RESOURCE_BATCH_SEGMENT_THRESHOLD = 20 // Smaller batch size for low-end hardware +export const LOW_RESOURCE_FILE_PROCESSING_CONCURRENCY = 2 // Reduced file processing concurrency for low-end hardware /**OpenAI Embedder */ export const MAX_BATCH_TOKENS = 100000 diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index fdd918b524..c33c0409f8 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -6,6 +6,8 @@ import { MAX_BATCH_RETRIES, INITIAL_RETRY_DELAY_MS, LOW_RESOURCE_BATCH_SEGMENT_THRESHOLD, + FILE_PROCESSING_CONCURRENCY, + LOW_RESOURCE_FILE_PROCESSING_CONCURRENCY, } from "../constants" import { createHash } from "crypto" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" @@ -96,18 +98,24 @@ export class FileWatcher implements IFileWatcher { this.batchSegmentThreshold = batchSegmentThreshold ?? config.get("codeIndex.embeddingBatchSize", LOW_RESOURCE_BATCH_SEGMENT_THRESHOLD) - // Reduce file processing concurrency in low resource mode - this.FILE_PROCESSING_CONCURRENCY_LIMIT = 2 + // Use configurable file processing concurrency with low resource default + this.FILE_PROCESSING_CONCURRENCY_LIMIT = config.get( + "codeIndex.fileProcessingConcurrency", + LOW_RESOURCE_FILE_PROCESSING_CONCURRENCY, + ) } else { this.batchSegmentThreshold = batchSegmentThreshold ?? config.get("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD) - // Normal concurrency for file processing - this.FILE_PROCESSING_CONCURRENCY_LIMIT = 10 + // Use configurable file processing concurrency with normal default + this.FILE_PROCESSING_CONCURRENCY_LIMIT = config.get( + "codeIndex.fileProcessingConcurrency", + FILE_PROCESSING_CONCURRENCY, + ) } } catch { // In test environment, vscode.workspace might not be available this.batchSegmentThreshold = batchSegmentThreshold ?? BATCH_SEGMENT_THRESHOLD - this.FILE_PROCESSING_CONCURRENCY_LIMIT = 10 + this.FILE_PROCESSING_CONCURRENCY_LIMIT = FILE_PROCESSING_CONCURRENCY } }