feat: expose configurable chunking and batch processing parameters for codebase indexing

Adds three new configurable parameters for codebase indexing:
- Embedding Batch Size (10-200, default 60): Number of code segments batched for embeddings
- Max Chunk Size (200-5000 chars, default 1000): Maximum characters per code chunk
- Parsing Concurrency (1-50, default 10): Number of concurrent file parsing operations

Changes:
- Added new config fields in packages/types/src/codebase-index.ts
- Updated constants in src/services/code-index/constants/index.ts
- Updated config-manager.ts to load and expose new settings
- Updated DirectoryScanner to use configurable parsing concurrency
- Added UI sliders in CodeIndexPopover.tsx for the new settings
- Added i18n translations for new settings labels

Closes #10396

packages/types/src/codebase-index.ts

@@ -12,6 +12,21 @@ export const CODEBASE_INDEX_DEFAULTS = {
 	MAX_SEARCH_SCORE: 1,
 	DEFAULT_SEARCH_MIN_SCORE: 0.4,
 	SEARCH_SCORE_STEP: 0.05,
+	// Embedding batch size settings
+	MIN_EMBEDDING_BATCH_SIZE: 10,
+	MAX_EMBEDDING_BATCH_SIZE: 200,
+	DEFAULT_EMBEDDING_BATCH_SIZE: 60,
+	EMBEDDING_BATCH_SIZE_STEP: 10,
+	// Max chunk size settings (characters per code chunk)
+	MIN_MAX_CHUNK_SIZE: 200,
+	MAX_MAX_CHUNK_SIZE: 5000,
+	DEFAULT_MAX_CHUNK_SIZE: 1000,
+	MAX_CHUNK_SIZE_STEP: 100,
+	// Parsing concurrency settings (concurrent file parsing)
+	MIN_PARSING_CONCURRENCY: 1,
+	MAX_PARSING_CONCURRENCY: 50,
+	DEFAULT_PARSING_CONCURRENCY: 10,
+	PARSING_CONCURRENCY_STEP: 1,
 } as const
 
 /**
@@ -42,6 +57,22 @@ export const codebaseIndexConfigSchema = z.object({
 		.min(CODEBASE_INDEX_DEFAULTS.MIN_SEARCH_RESULTS)
 		.max(CODEBASE_INDEX_DEFAULTS.MAX_SEARCH_RESULTS)
 		.optional(),
+	// Advanced indexing parameters
+	codebaseIndexEmbeddingBatchSize: z
+		.number()
+		.min(CODEBASE_INDEX_DEFAULTS.MIN_EMBEDDING_BATCH_SIZE)
+		.max(CODEBASE_INDEX_DEFAULTS.MAX_EMBEDDING_BATCH_SIZE)
+		.optional(),
+	codebaseIndexMaxChunkSize: z
+		.number()
+		.min(CODEBASE_INDEX_DEFAULTS.MIN_MAX_CHUNK_SIZE)
+		.max(CODEBASE_INDEX_DEFAULTS.MAX_MAX_CHUNK_SIZE)
+		.optional(),
+	codebaseIndexParsingConcurrency: z
+		.number()
+		.min(CODEBASE_INDEX_DEFAULTS.MIN_PARSING_CONCURRENCY)
+		.max(CODEBASE_INDEX_DEFAULTS.MAX_PARSING_CONCURRENCY)
+		.optional(),
 	// OpenAI Compatible specific fields
 	codebaseIndexOpenAiCompatibleBaseUrl: z.string().optional(),
 	codebaseIndexOpenAiCompatibleModelDimension: z.number().optional(),

src/services/code-index/config-manager.ts

@@ -2,7 +2,13 @@ import { ApiHandlerOptions } from "../../shared/api"
 import { ContextProxy } from "../../core/config/ContextProxy"
 import { EmbedderProvider } from "./interfaces/manager"
 import { CodeIndexConfig, PreviousConfigSnapshot } from "./interfaces/config"
-import { DEFAULT_SEARCH_MIN_SCORE, DEFAULT_MAX_SEARCH_RESULTS } from "./constants"
+import {
+	DEFAULT_SEARCH_MIN_SCORE,
+	DEFAULT_MAX_SEARCH_RESULTS,
+	BATCH_SEGMENT_THRESHOLD,
+	MAX_BLOCK_CHARS,
+	PARSING_CONCURRENCY,
+} from "./constants"
 import { getDefaultModelId, getModelDimension, getModelScoreThreshold } from "../../shared/embeddingModels"
 
 /**
@@ -26,6 +32,10 @@ export class CodeIndexConfigManager {
 	private qdrantApiKey?: string
 	private searchMinScore?: number
 	private searchMaxResults?: number
+	// Advanced indexing parameters
+	private embeddingBatchSize?: number
+	private maxChunkSize?: number
+	private parsingConcurrency?: number
 
 	constructor(private readonly contextProxy: ContextProxy) {
 		// Initialize with current configuration to avoid false restart triggers
@@ -87,6 +97,11 @@ export class CodeIndexConfigManager {
 		this.searchMinScore = codebaseIndexSearchMinScore
 		this.searchMaxResults = codebaseIndexSearchMaxResults
 
+		// Load advanced indexing parameters
+		this.embeddingBatchSize = codebaseIndexConfig.codebaseIndexEmbeddingBatchSize
+		this.maxChunkSize = codebaseIndexConfig.codebaseIndexMaxChunkSize
+		this.parsingConcurrency = codebaseIndexConfig.codebaseIndexParsingConcurrency
+
 		// Validate and set model dimension
 		const rawDimension = codebaseIndexConfig.codebaseIndexEmbedderModelDimension
 		if (rawDimension !== undefined && rawDimension !== null) {
@@ -460,6 +475,10 @@ export class CodeIndexConfigManager {
 			qdrantApiKey: this.qdrantApiKey,
 			searchMinScore: this.currentSearchMinScore,
 			searchMaxResults: this.currentSearchMaxResults,
+			// Advanced indexing parameters
+			embeddingBatchSize: this.currentEmbeddingBatchSize,
+			maxChunkSize: this.currentMaxChunkSize,
+			parsingConcurrency: this.currentParsingConcurrency,
 		}
 	}
 
@@ -541,4 +560,28 @@ export class CodeIndexConfigManager {
 	public get currentSearchMaxResults(): number {
 		return this.searchMaxResults ?? DEFAULT_MAX_SEARCH_RESULTS
 	}
+
+	/**
+	 * Gets the configured embedding batch size.
+	 * Returns user setting if configured, otherwise returns default.
+	 */
+	public get currentEmbeddingBatchSize(): number {
+		return this.embeddingBatchSize ?? BATCH_SEGMENT_THRESHOLD
+	}
+
+	/**
+	 * Gets the configured max chunk size (characters per code chunk).
+	 * Returns user setting if configured, otherwise returns default.
+	 */
+	public get currentMaxChunkSize(): number {
+		return this.maxChunkSize ?? MAX_BLOCK_CHARS
+	}
+
+	/**
+	 * Gets the configured parsing concurrency (concurrent file parsing).
+	 * Returns user setting if configured, otherwise returns default.
+	 */
+	public get currentParsingConcurrency(): number {
+		return this.parsingConcurrency ?? PARSING_CONCURRENCY
+	}
 }

src/services/code-index/constants/index.ts

@@ -1,7 +1,7 @@
 import { CODEBASE_INDEX_DEFAULTS } from "@roo-code/types"
 
 /**Parser */
-export const MAX_BLOCK_CHARS = 1000
+export const MAX_BLOCK_CHARS = CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE
 export const MIN_BLOCK_CHARS = 50
 export const MIN_CHUNK_REMAINDER_CHARS = 200 // Minimum characters for the *next* chunk after a split
 export const MAX_CHARS_TOLERANCE_FACTOR = 1.15 // 15% tolerance for max chars
@@ -16,10 +16,10 @@ export const MAX_FILE_SIZE_BYTES = 1 * 1024 * 1024 // 1MB
 
 /**Directory Scanner */
 export const MAX_LIST_FILES_LIMIT_CODE_INDEX = 50_000
-export const BATCH_SEGMENT_THRESHOLD = 60 // Number of code segments to batch for embeddings/upserts
+export const BATCH_SEGMENT_THRESHOLD = CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE // Number of code segments to batch for embeddings/upserts
 export const MAX_BATCH_RETRIES = 3
 export const INITIAL_RETRY_DELAY_MS = 500
-export const PARSING_CONCURRENCY = 10
+export const PARSING_CONCURRENCY = CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY
 export const MAX_PENDING_BATCHES = 20 // Maximum number of batches to accumulate before waiting
 
 /**OpenAI Embedder */

src/services/code-index/interfaces/config.ts

@@ -21,6 +21,10 @@ export interface CodeIndexConfig {
 	qdrantApiKey?: string
 	searchMinScore?: number
 	searchMaxResults?: number
+	// Advanced indexing parameters
+	embeddingBatchSize?: number
+	maxChunkSize?: number
+	parsingConcurrency?: number
 }
 
 /**

src/services/code-index/processors/scanner.ts

@@ -33,6 +33,7 @@ import { Package } from "../../../shared/package"
 
 export class DirectoryScanner implements IDirectoryScanner {
 	private readonly batchSegmentThreshold: number
+	private readonly parsingConcurrencyLimit: number
 
 	constructor(
 		private readonly embedder: IEmbedder,
@@ -41,6 +42,7 @@ export class DirectoryScanner implements IDirectoryScanner {
 		private readonly cacheManager: CacheManager,
 		private readonly ignoreInstance: Ignore,
 		batchSegmentThreshold?: number,
+		parsingConcurrencyLimit?: number,
 	) {
 		// Get the configurable batch size from VSCode settings, fallback to default
 		// If not provided in constructor, try to get from VSCode settings
@@ -56,6 +58,8 @@ export class DirectoryScanner implements IDirectoryScanner {
 				this.batchSegmentThreshold = BATCH_SEGMENT_THRESHOLD
 			}
 		}
+		// Set parsing concurrency (default from constants if not provided)
+		this.parsingConcurrencyLimit = parsingConcurrencyLimit ?? PARSING_CONCURRENCY
 	}
 
 	/**
@@ -109,7 +113,7 @@ export class DirectoryScanner implements IDirectoryScanner {
 		let skippedCount = 0
 
 		// Initialize parallel processing tools
-		const parseLimiter = pLimit(PARSING_CONCURRENCY) // Concurrency for file parsing
+		const parseLimiter = pLimit(this.parsingConcurrencyLimit) // Concurrency for file parsing
 		const batchLimiter = pLimit(BATCH_PROCESSING_CONCURRENCY) // Concurrency for batch processing
 		const mutex = new Mutex()
 

src/services/code-index/service-factory.ts

@@ -176,17 +176,20 @@ export class CodeIndexServiceFactory {
 		parser: ICodeParser,
 		ignoreInstance: Ignore,
 	): DirectoryScanner {
-		// Get the configurable batch size from VSCode settings
-		let batchSize: number
-		try {
-			batchSize = vscode.workspace
-				.getConfiguration(Package.name)
-				.get<number>("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD)
-		} catch {
-			// In test environment, vscode.workspace might not be available
-			batchSize = BATCH_SEGMENT_THRESHOLD
-		}
-		return new DirectoryScanner(embedder, vectorStore, parser, this.cacheManager, ignoreInstance, batchSize)
+		// Get the configurable settings from config manager
+		const config = this.configManager.getConfig()
+		const batchSize = config.embeddingBatchSize ?? BATCH_SEGMENT_THRESHOLD
+		const parsingConcurrency = config.parsingConcurrency
+
+		return new DirectoryScanner(
+			embedder,
+			vectorStore,
+			parser,
+			this.cacheManager,
+			ignoreInstance,
+			batchSize,
+			parsingConcurrency,
+		)
 	}
 
 	/**
@@ -200,16 +203,10 @@ export class CodeIndexServiceFactory {
 		ignoreInstance: Ignore,
 		rooIgnoreController?: RooIgnoreController,
 	): IFileWatcher {
-		// Get the configurable batch size from VSCode settings
-		let batchSize: number
-		try {
-			batchSize = vscode.workspace
-				.getConfiguration(Package.name)
-				.get<number>("codeIndex.embeddingBatchSize", BATCH_SEGMENT_THRESHOLD)
-		} catch {
-			// In test environment, vscode.workspace might not be available
-			batchSize = BATCH_SEGMENT_THRESHOLD
-		}
+		// Get the configurable settings from config manager
+		const config = this.configManager.getConfig()
+		const batchSize = config.embeddingBatchSize ?? BATCH_SEGMENT_THRESHOLD
+
 		return new FileWatcher(
 			this.workspacePath,
 			context,

webview-ui/src/components/chat/CodeIndexPopover.tsx

@@ -69,6 +69,10 @@ interface LocalCodeIndexSettings {
 	codebaseIndexEmbedderModelDimension?: number // Generic dimension for all providers
 	codebaseIndexSearchMaxResults?: number
 	codebaseIndexSearchMinScore?: number
+	// Advanced indexing parameters
+	codebaseIndexEmbeddingBatchSize?: number
+	codebaseIndexMaxChunkSize?: number
+	codebaseIndexParsingConcurrency?: number
 
 	// Bedrock-specific settings
 	codebaseIndexBedrockRegion?: string
@@ -217,6 +221,9 @@ export const CodeIndexPopover: React.FC<CodeIndexPopoverProps> = ({
 		codebaseIndexEmbedderModelDimension: undefined,
 		codebaseIndexSearchMaxResults: CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_RESULTS,
 		codebaseIndexSearchMinScore: CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_MIN_SCORE,
+		codebaseIndexEmbeddingBatchSize: CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE,
+		codebaseIndexMaxChunkSize: CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE,
+		codebaseIndexParsingConcurrency: CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY,
 		codebaseIndexBedrockRegion: "",
 		codebaseIndexBedrockProfile: "",
 		codeIndexOpenAiKey: "",
@@ -256,6 +263,14 @@ export const CodeIndexPopover: React.FC<CodeIndexPopoverProps> = ({
 					codebaseIndexConfig.codebaseIndexSearchMaxResults ?? CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_RESULTS,
 				codebaseIndexSearchMinScore:
 					codebaseIndexConfig.codebaseIndexSearchMinScore ?? CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_MIN_SCORE,
+				codebaseIndexEmbeddingBatchSize:
+					codebaseIndexConfig.codebaseIndexEmbeddingBatchSize ??
+					CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE,
+				codebaseIndexMaxChunkSize:
+					codebaseIndexConfig.codebaseIndexMaxChunkSize ?? CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE,
+				codebaseIndexParsingConcurrency:
+					codebaseIndexConfig.codebaseIndexParsingConcurrency ??
+					CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY,
 				codebaseIndexBedrockRegion: codebaseIndexConfig.codebaseIndexBedrockRegion || "",
 				codebaseIndexBedrockProfile: codebaseIndexConfig.codebaseIndexBedrockProfile || "",
 				codeIndexOpenAiKey: "",
@@ -1589,6 +1604,138 @@ export const CodeIndexPopover: React.FC<CodeIndexPopoverProps> = ({
 											</VSCodeButton>
 										</div>
 									</div>
+
+									{/* Embedding Batch Size Slider */}
+									<div className="space-y-2">
+										<div className="flex items-center gap-2">
+											<label className="text-sm font-medium">
+												{t("settings:codeIndex.embeddingBatchSizeLabel")}
+											</label>
+											<StandardTooltip
+												content={t("settings:codeIndex.embeddingBatchSizeDescription")}>
+												<span className="codicon codicon-info text-xs text-vscode-descriptionForeground cursor-help" />
+											</StandardTooltip>
+										</div>
+										<div className="flex items-center gap-2">
+											<Slider
+												min={CODEBASE_INDEX_DEFAULTS.MIN_EMBEDDING_BATCH_SIZE}
+												max={CODEBASE_INDEX_DEFAULTS.MAX_EMBEDDING_BATCH_SIZE}
+												step={CODEBASE_INDEX_DEFAULTS.EMBEDDING_BATCH_SIZE_STEP}
+												value={[
+													currentSettings.codebaseIndexEmbeddingBatchSize ??
+														CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE,
+												]}
+												onValueChange={(values) =>
+													updateSetting("codebaseIndexEmbeddingBatchSize", values[0])
+												}
+												className="flex-1"
+												data-testid="embedding-batch-size-slider"
+											/>
+											<span className="w-12 text-center">
+												{currentSettings.codebaseIndexEmbeddingBatchSize ??
+													CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE}
+											</span>
+											<VSCodeButton
+												appearance="icon"
+												title={t("settings:codeIndex.resetToDefault")}
+												onClick={() =>
+													updateSetting(
+														"codebaseIndexEmbeddingBatchSize",
+														CODEBASE_INDEX_DEFAULTS.DEFAULT_EMBEDDING_BATCH_SIZE,
+													)
+												}>
+												<span className="codicon codicon-discard" />
+											</VSCodeButton>
+										</div>
+									</div>
+
+									{/* Max Chunk Size Slider */}
+									<div className="space-y-2">
+										<div className="flex items-center gap-2">
+											<label className="text-sm font-medium">
+												{t("settings:codeIndex.maxChunkSizeLabel")}
+											</label>
+											<StandardTooltip
+												content={t("settings:codeIndex.maxChunkSizeDescription")}>
+												<span className="codicon codicon-info text-xs text-vscode-descriptionForeground cursor-help" />
+											</StandardTooltip>
+										</div>
+										<div className="flex items-center gap-2">
+											<Slider
+												min={CODEBASE_INDEX_DEFAULTS.MIN_MAX_CHUNK_SIZE}
+												max={CODEBASE_INDEX_DEFAULTS.MAX_MAX_CHUNK_SIZE}
+												step={CODEBASE_INDEX_DEFAULTS.MAX_CHUNK_SIZE_STEP}
+												value={[
+													currentSettings.codebaseIndexMaxChunkSize ??
+														CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE,
+												]}
+												onValueChange={(values) =>
+													updateSetting("codebaseIndexMaxChunkSize", values[0])
+												}
+												className="flex-1"
+												data-testid="max-chunk-size-slider"
+											/>
+											<span className="w-12 text-center">
+												{currentSettings.codebaseIndexMaxChunkSize ??
+													CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE}
+											</span>
+											<VSCodeButton
+												appearance="icon"
+												title={t("settings:codeIndex.resetToDefault")}
+												onClick={() =>
+													updateSetting(
+														"codebaseIndexMaxChunkSize",
+														CODEBASE_INDEX_DEFAULTS.DEFAULT_MAX_CHUNK_SIZE,
+													)
+												}>
+												<span className="codicon codicon-discard" />
+											</VSCodeButton>
+										</div>
+									</div>
+
+									{/* Parsing Concurrency Slider */}
+									<div className="space-y-2">
+										<div className="flex items-center gap-2">
+											<label className="text-sm font-medium">
+												{t("settings:codeIndex.parsingConcurrencyLabel")}
+											</label>
+											<StandardTooltip
+												content={t("settings:codeIndex.parsingConcurrencyDescription")}>
+												<span className="codicon codicon-info text-xs text-vscode-descriptionForeground cursor-help" />
+											</StandardTooltip>
+										</div>
+										<div className="flex items-center gap-2">
+											<Slider
+												min={CODEBASE_INDEX_DEFAULTS.MIN_PARSING_CONCURRENCY}
+												max={CODEBASE_INDEX_DEFAULTS.MAX_PARSING_CONCURRENCY}
+												step={CODEBASE_INDEX_DEFAULTS.PARSING_CONCURRENCY_STEP}
+												value={[
+													currentSettings.codebaseIndexParsingConcurrency ??
+														CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY,
+												]}
+												onValueChange={(values) =>
+													updateSetting("codebaseIndexParsingConcurrency", values[0])
+												}
+												className="flex-1"
+												data-testid="parsing-concurrency-slider"
+											/>
+											<span className="w-12 text-center">
+												{currentSettings.codebaseIndexParsingConcurrency ??
+													CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY}
+											</span>
+											<VSCodeButton
+												appearance="icon"
+												title={t("settings:codeIndex.resetToDefault")}
+												onClick={() =>
+													updateSetting(
+														"codebaseIndexParsingConcurrency",
+														CODEBASE_INDEX_DEFAULTS.DEFAULT_PARSING_CONCURRENCY,
+													)
+												}>
+												<span className="codicon codicon-discard" />
+											</VSCodeButton>
+										</div>
+									</div>
 								</div>
 							)}
 						</div>

webview-ui/src/i18n/locales/en/settings.json

@@ -133,6 +133,15 @@
 		"searchMinScoreResetTooltip": "Reset to default value (0.4)",
 		"searchMaxResultsLabel": "Maximum Search Results",
 		"searchMaxResultsDescription": "Maximum number of search results to return when querying the codebase index. Higher values provide more context but may include less relevant results.",
+		"embeddingBatchSizeLabel": "Embedding Batch Size",
+		"embeddingBatchSizeDescription": "Number of code segments to batch together for embeddings. Higher values can speed up indexing on powerful hardware. Lower values reduce memory usage.",
+		"embeddingBatchSizeResetTooltip": "Reset to default value (60)",
+		"maxChunkSizeLabel": "Max Chunk Size",
+		"maxChunkSizeDescription": "Maximum characters per code chunk. Larger chunks provide more context but may reduce search precision. Smaller chunks enable finer-grained search results.",
+		"maxChunkSizeResetTooltip": "Reset to default value (1000)",
+		"parsingConcurrencyLabel": "Parsing Concurrency",
+		"parsingConcurrencyDescription": "Number of files to parse concurrently during indexing. Higher values speed up indexing but use more CPU and memory.",
+		"parsingConcurrencyResetTooltip": "Reset to default value (10)",
 		"resetToDefault": "Reset to default",
 		"startIndexingButton": "Start Indexing",
 		"clearIndexDataButton": "Clear Index Data",