From e60b32f8cf42a6e070783306a971bc5e072bec17 Mon Sep 17 00:00:00 2001 From: Roo Code Date: Tue, 30 Dec 2025 05:19:52 +0000 Subject: [PATCH] fix: apply maxChunkSize configuration to CodeParser - Modified CodeParser to accept maxBlockChars as constructor parameter - Updated FileWatcher to receive ICodeParser via constructor injection - Updated service-factory to create CodeParser with configured maxChunkSize - This ensures the codebaseIndexMaxChunkSize setting actually affects parsing --- .../code-index/processors/file-watcher.ts | 18 ++++++++++++--- src/services/code-index/processors/parser.ts | 23 +++++++++++++------ src/services/code-index/service-factory.ts | 15 ++++++++++-- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index 1e5ebcbceb..ffbcb84fd7 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -18,8 +18,8 @@ import { IVectorStore, PointStruct, BatchProcessingSummary, + ICodeParser, } from "../interfaces" -import { codeParser } from "./parser" import { CacheManager } from "../cache-manager" import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" @@ -68,9 +68,13 @@ export class FileWatcher implements IFileWatcher { * Creates a new file watcher * @param workspacePath Path to the workspace * @param context VS Code extension context + * @param cacheManager Cache manager * @param embedder Optional embedder * @param vectorStore Optional vector store - * @param cacheManager Cache manager + * @param ignoreInstance Optional ignore instance + * @param ignoreController Optional ignore controller + * @param batchSegmentThreshold Optional batch segment threshold + * @param codeParser Code parser for parsing files */ constructor( private workspacePath: string, @@ -81,6 +85,7 @@ export class FileWatcher implements IFileWatcher { ignoreInstance?: Ignore, ignoreController?: RooIgnoreController, batchSegmentThreshold?: number, + private readonly codeParser?: ICodeParser, ) { this.ignoreController = ignoreController || new RooIgnoreController(workspacePath) if (ignoreInstance) { @@ -557,7 +562,14 @@ export class FileWatcher implements IFileWatcher { } // Parse file - const blocks = await codeParser.parseFile(filePath, { content, fileHash: newHash }) + if (!this.codeParser) { + return { + path: filePath, + status: "local_error" as const, + error: new Error("No code parser configured"), + } + } + const blocks = await this.codeParser.parseFile(filePath, { content, fileHash: newHash }) // Prepare points for batch processing let pointsToUpsert: PointStruct[] = [] diff --git a/src/services/code-index/processors/parser.ts b/src/services/code-index/processors/parser.ts index 8611884ade..493d8f7a6a 100644 --- a/src/services/code-index/processors/parser.ts +++ b/src/services/code-index/processors/parser.ts @@ -17,9 +17,18 @@ import { sanitizeErrorMessage } from "../shared/validation-helpers" export class CodeParser implements ICodeParser { private loadedParsers: LanguageParser = {} private pendingLoads: Map> = new Map() + private readonly maxBlockChars: number // Markdown files are now supported using the custom markdown parser // which extracts headers and sections for semantic indexing + /** + * Creates a new CodeParser instance + * @param maxBlockChars Maximum characters per code chunk (default: MAX_BLOCK_CHARS from constants) + */ + constructor(maxBlockChars?: number) { + this.maxBlockChars = maxBlockChars ?? MAX_BLOCK_CHARS + } + /** * Parses a code file into code blocks * @param filePath Path to the file to parse @@ -179,7 +188,7 @@ export class CodeParser implements ICodeParser { // Check if the node meets the minimum character requirement if (currentNode.text.length >= MIN_BLOCK_CHARS) { // If it also exceeds the maximum character limit, try to break it down - if (currentNode.text.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR) { + if (currentNode.text.length > this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR) { if (currentNode.children.filter((child) => child !== null).length > 0) { // If it has children, process them instead queue.push(...currentNode.children.filter((child) => child !== null)) @@ -244,7 +253,7 @@ export class CodeParser implements ICodeParser { let currentChunkLines: string[] = [] let currentChunkLength = 0 let chunkStartLineIndex = 0 // 0-based index within the `lines` array - const effectiveMaxChars = MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR + const effectiveMaxChars = this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR const finalizeChunk = (endLineIndex: number) => { if (currentChunkLength >= MIN_BLOCK_CHARS && currentChunkLines.length > 0) { @@ -314,10 +323,10 @@ export class CodeParser implements ICodeParser { let remainingLineContent = line let currentSegmentStartChar = 0 while (remainingLineContent.length > 0) { - const segment = remainingLineContent.substring(0, MAX_BLOCK_CHARS) - remainingLineContent = remainingLineContent.substring(MAX_BLOCK_CHARS) + const segment = remainingLineContent.substring(0, this.maxBlockChars) + remainingLineContent = remainingLineContent.substring(this.maxBlockChars) createSegmentBlock(segment, originalLineNumber, currentSegmentStartChar) - currentSegmentStartChar += MAX_BLOCK_CHARS + currentSegmentStartChar += this.maxBlockChars } // Update chunkStartLineIndex to continue processing from the next line chunkStartLineIndex = i + 1 @@ -425,8 +434,8 @@ export class CodeParser implements ICodeParser { // Check if content needs chunking (either total size or individual line size) const needsChunking = - content.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR || - lines.some((line) => line.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR) + content.length > this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR || + lines.some((line) => line.length > this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR) if (needsChunking) { // Apply chunking for large content or oversized lines diff --git a/src/services/code-index/service-factory.ts b/src/services/code-index/service-factory.ts index 6c38e30726..dc165b6dd7 100644 --- a/src/services/code-index/service-factory.ts +++ b/src/services/code-index/service-factory.ts @@ -9,7 +9,7 @@ import { BedrockEmbedder } from "./embedders/bedrock" import { OpenRouterEmbedder } from "./embedders/openrouter" import { EmbedderProvider, getDefaultModelId, getModelDimension } from "../../shared/embeddingModels" import { QdrantVectorStore } from "./vector-store/qdrant-client" -import { codeParser, DirectoryScanner, FileWatcher } from "./processors" +import { CodeParser, DirectoryScanner, FileWatcher } from "./processors" import { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces" import { CodeIndexConfigManager } from "./config-manager" import { CacheManager } from "./cache-manager" @@ -167,6 +167,14 @@ export class CodeIndexServiceFactory { return new QdrantVectorStore(this.workspacePath, config.qdrantUrl, vectorSize, config.qdrantApiKey) } + /** + * Creates a code parser instance with the configured max chunk size. + */ + public createCodeParser(): ICodeParser { + const config = this.configManager.getConfig() + return new CodeParser(config.maxChunkSize) + } + /** * Creates a directory scanner instance with its required dependencies. */ @@ -202,6 +210,7 @@ export class CodeIndexServiceFactory { cacheManager: CacheManager, ignoreInstance: Ignore, rooIgnoreController?: RooIgnoreController, + parser?: ICodeParser, ): IFileWatcher { // Get the configurable settings from config manager const config = this.configManager.getConfig() @@ -216,6 +225,7 @@ export class CodeIndexServiceFactory { ignoreInstance, rooIgnoreController, batchSize, + parser, ) } @@ -241,7 +251,7 @@ export class CodeIndexServiceFactory { const embedder = this.createEmbedder() const vectorStore = this.createVectorStore() - const parser = codeParser + const parser = this.createCodeParser() const scanner = this.createDirectoryScanner(embedder, vectorStore, parser, ignoreInstance) const fileWatcher = this.createFileWatcher( context, @@ -250,6 +260,7 @@ export class CodeIndexServiceFactory { cacheManager, ignoreInstance, rooIgnoreController, + parser, ) return {