Roo-Code/src/services/code-index/interfaces/file-processor.ts
Daniel 61e122dc34
Codebase indexing (Clean history) (#3137)
* feat: apply changes from local main

* fix: add missing types

* feat: deduplicate code blocks coming out of parser

* feat: implement a cache manager to improve cache handling

* refactor: move code index service initialization to extension and remove await from indexing process

* fix: return undefined instead of throwing if no workspace is detected

* feat: allow auto approve if it is active for read tools

* refactor: improve UI of the results and allow opening the ranges directly in the editor

* refactor: use dependency injection to improve performance

* feat: implement result filtering by directory path

* refactor: centralize path normalization logic

* refactor: remove unnecessary barrel file

* refactor: prevent restarting the service if no settings change

* fix: the indexing process should never be awaited

* refactor: cleanup unused method

* refactor: remove batch limits for ollama

* refactor(parser): simplify method signatures and improve chunking logic

- Remove redundant min/max chars parameters
- Add better handling for oversized lines
- Improve chunking logic with segment handling
- Clean up method signatures and parameter ordering

* fix(settings): make select inputs full width in CodeIndexSettings

* refactor: increase max list file limit

* feat(ui): improve codebase search result display formatting

* test: add tests for cache and config managers

* test: create unit tests for parser and scanner

* feat(parser): improve segment hash uniqueness

- Added startCharIndex to segment hash calculation in _chunkTextByLines
- Track character position when splitting oversized lines
- Ensures unique identification of segments from same line

* feat(file-watcher): add error logging and optional ignoreController injection

* fix: allow getting the state if the service is disabled

* fix: set the embedding models when cline provider is initialized

* feat: use zod to validate form

* feat(file-watcher): enhance file watcher for batched deletions and improved vector store interactions

Improve file watcher to handle file deletions in batches and optimize vector store operations.

* feat(CodeIndexSettings): move OpenAI key input to a conditional rendering block

* feat(CodeIndexSettings): update button visibility based on indexing status

* feat(file-watcher): refactor vscode mock and enhance file watcher tests

* fix(CodeIndexManager): do not await startIndexing on configuration changes

* feat(types): add codeIndexOpenAiKey and codeIndexQdrantApiKey to ProviderSettings and IpcMessage

* feat(FileWatcher): enhance file processing with batch operations and new status handling

* fix(webviewMessageHandler): handle errors during CodeIndexManager initialization

* refactor(CodeIndexManager): streamline service creation by consolidating into a single method

* feat(CodeIndex): implement minimum search score configuration and update search methods

* refactor(CodeIndexSettings): replace ApiConfiguration with ProviderSettings and update related methods

* refactor: move contants to centralized file

* refactor(constants): rename CODEBASE_INDEX_SEARCH_MIN_SCORE to SEARCH_MIN_SCORE

* feat(QdrantVectorStore): enhance search functionality with new query structure and indexing

* feat(FileWatcher): implement batch processing and retry logic for upserting points

* fix(CodeIndexSettings): rename setProviderSettingsField to setApiConfigurationField and move model label

* fix(ChatRow): remove limit from search query messages

* refactor(CodebaseSearchResult): remove unused props from component

* feat: implement batch processing for file events in FileWatcher

- Introduced a new mechanism to accumulate file events (create, change, delete) and process them in batches.
- Added debounce functionality to optimize processing frequency.
- Emitted events for batch processing start, progress updates, and completion with detailed summaries.
- Refactored existing processing logic to handle batch deletions and upserts efficiently.
- Enhanced error handling and logging for better traceability during batch operations.

* feat(CodeIndex): implement batch processing and update progress reporting

* fix: define a default url for qdrant

* feat(CodeIndexManager): add initialization check and update startIndexing logic

* feat(CodeIndexSettings): validate Qdrant URL and update settings commitment logic

* feat: refactor progress calculation and update progress bar rendering

* refactor: remove webview provider and related methods

* fix: simplify indexing status update by directly using update values

* feat: integrate .gitignore support into file processing and scanning logic

* fix: update clearCacheFile method to write an empty object instead of deleting the cache file

* Revert this

* Run prettier

* fix: add new dependencies for qdrant client and directory scanner

* feat: add codebase search functionality to localization files

* feat: add localization strings for codebase indexing settings

* feat: integrate CodeIndexSettings into ExperimentalSettings and update settings localization

* refactor: remove console logs from various components for cleaner output

* feat: enhance capabilities section and codebase search tool description

* feat: add code indexing localization for multiple languages

* fix: correct indentation for CodeIndexSettings component in ExperimentalSettings

* refactor: update unit tests to properly test current functionality

* feat: add mock implementation for p-limit and update Jest config

* feat: track file creation, change, and deletion events in accumulatedEvents

* refactor: simplify file watcher tests by removing waitForFileProcessingToFinish and using direct event accumulation

* refactor: mock ContextProxy's getValue method to return current config name in ClineProvider tests

* refactor: mock missing properties required by codebase indexing manager

---------

Co-authored-by: cte <cestreich@gmail.com>
2025-05-23 19:07:19 -06:00

117 lines
2.8 KiB
TypeScript

import * as vscode from "vscode"
import { PointStruct } from "./vector-store"
/**
* Interface for code file parser
*/
export interface ICodeParser {
/**
* Parses a code file into code blocks
* @param filePath Path to the file to parse
* @param options Optional parsing options
* @returns Promise resolving to array of code blocks
*/
parseFile(
filePath: string,
options?: {
minBlockLines?: number
maxBlockLines?: number
content?: string
fileHash?: string
},
): Promise<CodeBlock[]>
}
/**
* Interface for directory scanner
*/
export interface IDirectoryScanner {
/**
* Scans a directory for code blocks
* @param directoryPath Path to the directory to scan
* @param options Optional scanning options
* @returns Promise resolving to scan results
*/
scanDirectory(
directory: string,
onError?: (error: Error) => void,
onBlocksIndexed?: (indexedCount: number) => void,
onFileParsed?: (fileBlockCount: number) => void,
): Promise<{
codeBlocks: CodeBlock[]
stats: {
processed: number
skipped: number
}
totalBlockCount: number
}>
}
/**
* Interface for file watcher
*/
export interface IFileWatcher extends vscode.Disposable {
/**
* Initializes the file watcher
*/
initialize(): Promise<void>
/**
* Event emitted when a batch of files begins processing.
* The event payload is an array of file paths included in the batch.
*/
readonly onDidStartBatchProcessing: vscode.Event<string[]>
/**
* Event emitted to report progress during batch processing.
*/
readonly onBatchProgressUpdate: vscode.Event<{
processedInBatch: number
totalInBatch: number
currentFile?: string
}>
/**
* Event emitted when a batch of files has finished processing.
* The event payload contains a summary of the batch operation.
*/
readonly onDidFinishBatchProcessing: vscode.Event<BatchProcessingSummary>
/**
* Processes a file
* @param filePath Path to the file to process
* @returns Promise resolving to processing result
*/
processFile(filePath: string): Promise<FileProcessingResult>
}
export interface BatchProcessingSummary {
/** All files attempted in the batch, including their final status. */
processedFiles: FileProcessingResult[]
/** Optional error if the entire batch operation failed (e.g., database connection issue). */
batchError?: Error
}
export interface FileProcessingResult {
path: string
status: "success" | "skipped" | "error" | "processed_for_batching" | "local_error"
error?: Error
reason?: string
newHash?: string
pointsToUpsert?: PointStruct[]
}
/**
* Common types used across the code-index service
*/
export interface CodeBlock {
file_path: string
identifier: string | null
type: string
start_line: number
end_line: number
content: string
fileHash: string
segmentHash: string
}