fix: apply maxChunkSize configuration to CodeParser

- Modified CodeParser to accept maxBlockChars as constructor parameter
- Updated FileWatcher to receive ICodeParser via constructor injection
- Updated service-factory to create CodeParser with configured maxChunkSize
- This ensures the codebaseIndexMaxChunkSize setting actually affects parsing
This commit is contained in:
Roo Code 2025-12-30 05:19:52 +00:00
parent 19f24dac68
commit e60b32f8cf
3 changed files with 44 additions and 12 deletions

View file

@ -18,8 +18,8 @@ import {
IVectorStore,
PointStruct,
BatchProcessingSummary,
ICodeParser,
} from "../interfaces"
import { codeParser } from "./parser"
import { CacheManager } from "../cache-manager"
import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path"
import { isPathInIgnoredDirectory } from "../../glob/ignore-utils"
@ -68,9 +68,13 @@ export class FileWatcher implements IFileWatcher {
* Creates a new file watcher
* @param workspacePath Path to the workspace
* @param context VS Code extension context
* @param cacheManager Cache manager
* @param embedder Optional embedder
* @param vectorStore Optional vector store
* @param cacheManager Cache manager
* @param ignoreInstance Optional ignore instance
* @param ignoreController Optional ignore controller
* @param batchSegmentThreshold Optional batch segment threshold
* @param codeParser Code parser for parsing files
*/
constructor(
private workspacePath: string,
@ -81,6 +85,7 @@ export class FileWatcher implements IFileWatcher {
ignoreInstance?: Ignore,
ignoreController?: RooIgnoreController,
batchSegmentThreshold?: number,
private readonly codeParser?: ICodeParser,
) {
this.ignoreController = ignoreController || new RooIgnoreController(workspacePath)
if (ignoreInstance) {
@ -557,7 +562,14 @@ export class FileWatcher implements IFileWatcher {
}
// Parse file
const blocks = await codeParser.parseFile(filePath, { content, fileHash: newHash })
if (!this.codeParser) {
return {
path: filePath,
status: "local_error" as const,
error: new Error("No code parser configured"),
}
}
const blocks = await this.codeParser.parseFile(filePath, { content, fileHash: newHash })
// Prepare points for batch processing
let pointsToUpsert: PointStruct[] = []

View file

@ -17,9 +17,18 @@ import { sanitizeErrorMessage } from "../shared/validation-helpers"
export class CodeParser implements ICodeParser {
private loadedParsers: LanguageParser = {}
private pendingLoads: Map<string, Promise<LanguageParser>> = new Map()
private readonly maxBlockChars: number
// Markdown files are now supported using the custom markdown parser
// which extracts headers and sections for semantic indexing
/**
* Creates a new CodeParser instance
* @param maxBlockChars Maximum characters per code chunk (default: MAX_BLOCK_CHARS from constants)
*/
constructor(maxBlockChars?: number) {
this.maxBlockChars = maxBlockChars ?? MAX_BLOCK_CHARS
}
/**
* Parses a code file into code blocks
* @param filePath Path to the file to parse
@ -179,7 +188,7 @@ export class CodeParser implements ICodeParser {
// Check if the node meets the minimum character requirement
if (currentNode.text.length >= MIN_BLOCK_CHARS) {
// If it also exceeds the maximum character limit, try to break it down
if (currentNode.text.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR) {
if (currentNode.text.length > this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR) {
if (currentNode.children.filter((child) => child !== null).length > 0) {
// If it has children, process them instead
queue.push(...currentNode.children.filter((child) => child !== null))
@ -244,7 +253,7 @@ export class CodeParser implements ICodeParser {
let currentChunkLines: string[] = []
let currentChunkLength = 0
let chunkStartLineIndex = 0 // 0-based index within the `lines` array
const effectiveMaxChars = MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR
const effectiveMaxChars = this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR
const finalizeChunk = (endLineIndex: number) => {
if (currentChunkLength >= MIN_BLOCK_CHARS && currentChunkLines.length > 0) {
@ -314,10 +323,10 @@ export class CodeParser implements ICodeParser {
let remainingLineContent = line
let currentSegmentStartChar = 0
while (remainingLineContent.length > 0) {
const segment = remainingLineContent.substring(0, MAX_BLOCK_CHARS)
remainingLineContent = remainingLineContent.substring(MAX_BLOCK_CHARS)
const segment = remainingLineContent.substring(0, this.maxBlockChars)
remainingLineContent = remainingLineContent.substring(this.maxBlockChars)
createSegmentBlock(segment, originalLineNumber, currentSegmentStartChar)
currentSegmentStartChar += MAX_BLOCK_CHARS
currentSegmentStartChar += this.maxBlockChars
}
// Update chunkStartLineIndex to continue processing from the next line
chunkStartLineIndex = i + 1
@ -425,8 +434,8 @@ export class CodeParser implements ICodeParser {
// Check if content needs chunking (either total size or individual line size)
const needsChunking =
content.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR ||
lines.some((line) => line.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR)
content.length > this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR ||
lines.some((line) => line.length > this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR)
if (needsChunking) {
// Apply chunking for large content or oversized lines

View file

@ -9,7 +9,7 @@ import { BedrockEmbedder } from "./embedders/bedrock"
import { OpenRouterEmbedder } from "./embedders/openrouter"
import { EmbedderProvider, getDefaultModelId, getModelDimension } from "../../shared/embeddingModels"
import { QdrantVectorStore } from "./vector-store/qdrant-client"
import { codeParser, DirectoryScanner, FileWatcher } from "./processors"
import { CodeParser, DirectoryScanner, FileWatcher } from "./processors"
import { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces"
import { CodeIndexConfigManager } from "./config-manager"
import { CacheManager } from "./cache-manager"
@ -167,6 +167,14 @@ export class CodeIndexServiceFactory {
return new QdrantVectorStore(this.workspacePath, config.qdrantUrl, vectorSize, config.qdrantApiKey)
}
/**
* Creates a code parser instance with the configured max chunk size.
*/
public createCodeParser(): ICodeParser {
const config = this.configManager.getConfig()
return new CodeParser(config.maxChunkSize)
}
/**
* Creates a directory scanner instance with its required dependencies.
*/
@ -202,6 +210,7 @@ export class CodeIndexServiceFactory {
cacheManager: CacheManager,
ignoreInstance: Ignore,
rooIgnoreController?: RooIgnoreController,
parser?: ICodeParser,
): IFileWatcher {
// Get the configurable settings from config manager
const config = this.configManager.getConfig()
@ -216,6 +225,7 @@ export class CodeIndexServiceFactory {
ignoreInstance,
rooIgnoreController,
batchSize,
parser,
)
}
@ -241,7 +251,7 @@ export class CodeIndexServiceFactory {
const embedder = this.createEmbedder()
const vectorStore = this.createVectorStore()
const parser = codeParser
const parser = this.createCodeParser()
const scanner = this.createDirectoryScanner(embedder, vectorStore, parser, ignoreInstance)
const fileWatcher = this.createFileWatcher(
context,
@ -250,6 +260,7 @@ export class CodeIndexServiceFactory {
cacheManager,
ignoreInstance,
rooIgnoreController,
parser,
)
return {