mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-05 08:10:14 +00:00
fix: apply maxChunkSize configuration to CodeParser
- Modified CodeParser to accept maxBlockChars as constructor parameter - Updated FileWatcher to receive ICodeParser via constructor injection - Updated service-factory to create CodeParser with configured maxChunkSize - This ensures the codebaseIndexMaxChunkSize setting actually affects parsing
This commit is contained in:
parent
19f24dac68
commit
e60b32f8cf
3 changed files with 44 additions and 12 deletions
|
|
@ -18,8 +18,8 @@ import {
|
|||
IVectorStore,
|
||||
PointStruct,
|
||||
BatchProcessingSummary,
|
||||
ICodeParser,
|
||||
} from "../interfaces"
|
||||
import { codeParser } from "./parser"
|
||||
import { CacheManager } from "../cache-manager"
|
||||
import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path"
|
||||
import { isPathInIgnoredDirectory } from "../../glob/ignore-utils"
|
||||
|
|
@ -68,9 +68,13 @@ export class FileWatcher implements IFileWatcher {
|
|||
* Creates a new file watcher
|
||||
* @param workspacePath Path to the workspace
|
||||
* @param context VS Code extension context
|
||||
* @param cacheManager Cache manager
|
||||
* @param embedder Optional embedder
|
||||
* @param vectorStore Optional vector store
|
||||
* @param cacheManager Cache manager
|
||||
* @param ignoreInstance Optional ignore instance
|
||||
* @param ignoreController Optional ignore controller
|
||||
* @param batchSegmentThreshold Optional batch segment threshold
|
||||
* @param codeParser Code parser for parsing files
|
||||
*/
|
||||
constructor(
|
||||
private workspacePath: string,
|
||||
|
|
@ -81,6 +85,7 @@ export class FileWatcher implements IFileWatcher {
|
|||
ignoreInstance?: Ignore,
|
||||
ignoreController?: RooIgnoreController,
|
||||
batchSegmentThreshold?: number,
|
||||
private readonly codeParser?: ICodeParser,
|
||||
) {
|
||||
this.ignoreController = ignoreController || new RooIgnoreController(workspacePath)
|
||||
if (ignoreInstance) {
|
||||
|
|
@ -557,7 +562,14 @@ export class FileWatcher implements IFileWatcher {
|
|||
}
|
||||
|
||||
// Parse file
|
||||
const blocks = await codeParser.parseFile(filePath, { content, fileHash: newHash })
|
||||
if (!this.codeParser) {
|
||||
return {
|
||||
path: filePath,
|
||||
status: "local_error" as const,
|
||||
error: new Error("No code parser configured"),
|
||||
}
|
||||
}
|
||||
const blocks = await this.codeParser.parseFile(filePath, { content, fileHash: newHash })
|
||||
|
||||
// Prepare points for batch processing
|
||||
let pointsToUpsert: PointStruct[] = []
|
||||
|
|
|
|||
|
|
@ -17,9 +17,18 @@ import { sanitizeErrorMessage } from "../shared/validation-helpers"
|
|||
export class CodeParser implements ICodeParser {
|
||||
private loadedParsers: LanguageParser = {}
|
||||
private pendingLoads: Map<string, Promise<LanguageParser>> = new Map()
|
||||
private readonly maxBlockChars: number
|
||||
// Markdown files are now supported using the custom markdown parser
|
||||
// which extracts headers and sections for semantic indexing
|
||||
|
||||
/**
|
||||
* Creates a new CodeParser instance
|
||||
* @param maxBlockChars Maximum characters per code chunk (default: MAX_BLOCK_CHARS from constants)
|
||||
*/
|
||||
constructor(maxBlockChars?: number) {
|
||||
this.maxBlockChars = maxBlockChars ?? MAX_BLOCK_CHARS
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a code file into code blocks
|
||||
* @param filePath Path to the file to parse
|
||||
|
|
@ -179,7 +188,7 @@ export class CodeParser implements ICodeParser {
|
|||
// Check if the node meets the minimum character requirement
|
||||
if (currentNode.text.length >= MIN_BLOCK_CHARS) {
|
||||
// If it also exceeds the maximum character limit, try to break it down
|
||||
if (currentNode.text.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR) {
|
||||
if (currentNode.text.length > this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR) {
|
||||
if (currentNode.children.filter((child) => child !== null).length > 0) {
|
||||
// If it has children, process them instead
|
||||
queue.push(...currentNode.children.filter((child) => child !== null))
|
||||
|
|
@ -244,7 +253,7 @@ export class CodeParser implements ICodeParser {
|
|||
let currentChunkLines: string[] = []
|
||||
let currentChunkLength = 0
|
||||
let chunkStartLineIndex = 0 // 0-based index within the `lines` array
|
||||
const effectiveMaxChars = MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR
|
||||
const effectiveMaxChars = this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR
|
||||
|
||||
const finalizeChunk = (endLineIndex: number) => {
|
||||
if (currentChunkLength >= MIN_BLOCK_CHARS && currentChunkLines.length > 0) {
|
||||
|
|
@ -314,10 +323,10 @@ export class CodeParser implements ICodeParser {
|
|||
let remainingLineContent = line
|
||||
let currentSegmentStartChar = 0
|
||||
while (remainingLineContent.length > 0) {
|
||||
const segment = remainingLineContent.substring(0, MAX_BLOCK_CHARS)
|
||||
remainingLineContent = remainingLineContent.substring(MAX_BLOCK_CHARS)
|
||||
const segment = remainingLineContent.substring(0, this.maxBlockChars)
|
||||
remainingLineContent = remainingLineContent.substring(this.maxBlockChars)
|
||||
createSegmentBlock(segment, originalLineNumber, currentSegmentStartChar)
|
||||
currentSegmentStartChar += MAX_BLOCK_CHARS
|
||||
currentSegmentStartChar += this.maxBlockChars
|
||||
}
|
||||
// Update chunkStartLineIndex to continue processing from the next line
|
||||
chunkStartLineIndex = i + 1
|
||||
|
|
@ -425,8 +434,8 @@ export class CodeParser implements ICodeParser {
|
|||
|
||||
// Check if content needs chunking (either total size or individual line size)
|
||||
const needsChunking =
|
||||
content.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR ||
|
||||
lines.some((line) => line.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR)
|
||||
content.length > this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR ||
|
||||
lines.some((line) => line.length > this.maxBlockChars * MAX_CHARS_TOLERANCE_FACTOR)
|
||||
|
||||
if (needsChunking) {
|
||||
// Apply chunking for large content or oversized lines
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { BedrockEmbedder } from "./embedders/bedrock"
|
|||
import { OpenRouterEmbedder } from "./embedders/openrouter"
|
||||
import { EmbedderProvider, getDefaultModelId, getModelDimension } from "../../shared/embeddingModels"
|
||||
import { QdrantVectorStore } from "./vector-store/qdrant-client"
|
||||
import { codeParser, DirectoryScanner, FileWatcher } from "./processors"
|
||||
import { CodeParser, DirectoryScanner, FileWatcher } from "./processors"
|
||||
import { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces"
|
||||
import { CodeIndexConfigManager } from "./config-manager"
|
||||
import { CacheManager } from "./cache-manager"
|
||||
|
|
@ -167,6 +167,14 @@ export class CodeIndexServiceFactory {
|
|||
return new QdrantVectorStore(this.workspacePath, config.qdrantUrl, vectorSize, config.qdrantApiKey)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a code parser instance with the configured max chunk size.
|
||||
*/
|
||||
public createCodeParser(): ICodeParser {
|
||||
const config = this.configManager.getConfig()
|
||||
return new CodeParser(config.maxChunkSize)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a directory scanner instance with its required dependencies.
|
||||
*/
|
||||
|
|
@ -202,6 +210,7 @@ export class CodeIndexServiceFactory {
|
|||
cacheManager: CacheManager,
|
||||
ignoreInstance: Ignore,
|
||||
rooIgnoreController?: RooIgnoreController,
|
||||
parser?: ICodeParser,
|
||||
): IFileWatcher {
|
||||
// Get the configurable settings from config manager
|
||||
const config = this.configManager.getConfig()
|
||||
|
|
@ -216,6 +225,7 @@ export class CodeIndexServiceFactory {
|
|||
ignoreInstance,
|
||||
rooIgnoreController,
|
||||
batchSize,
|
||||
parser,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -241,7 +251,7 @@ export class CodeIndexServiceFactory {
|
|||
|
||||
const embedder = this.createEmbedder()
|
||||
const vectorStore = this.createVectorStore()
|
||||
const parser = codeParser
|
||||
const parser = this.createCodeParser()
|
||||
const scanner = this.createDirectoryScanner(embedder, vectorStore, parser, ignoreInstance)
|
||||
const fileWatcher = this.createFileWatcher(
|
||||
context,
|
||||
|
|
@ -250,6 +260,7 @@ export class CodeIndexServiceFactory {
|
|||
cacheManager,
|
||||
ignoreInstance,
|
||||
rooIgnoreController,
|
||||
parser,
|
||||
)
|
||||
|
||||
return {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue