diff --git a/src/esbuild.mjs b/src/esbuild.mjs index 6269eaf3c7..05d1284d6a 100644 --- a/src/esbuild.mjs +++ b/src/esbuild.mjs @@ -110,6 +110,7 @@ async function main() { ...buildOptions, entryPoints: ["workers/countTokens.ts", "workers/indexing-worker.ts"], outdir: "dist/workers", + external: ["vscode"], } const [extensionCtx, workerCtx] = await Promise.all([ diff --git a/src/services/code-index/worker-utils/RooIgnoreController.ts b/src/services/code-index/worker-utils/RooIgnoreController.ts new file mode 100644 index 0000000000..6ae5d98874 --- /dev/null +++ b/src/services/code-index/worker-utils/RooIgnoreController.ts @@ -0,0 +1,90 @@ +// Worker-compatible version of RooIgnoreController without vscode dependencies +import * as fs from "fs/promises" +import * as path from "path" +import ignore, { Ignore } from "ignore" + +/** + * Worker-compatible controller for managing .rooignore files + */ +export class RooIgnoreController { + private ignoreInstance: Ignore + private rooIgnorePath: string + private initialized = false + + constructor(private workspacePath: string) { + this.rooIgnorePath = path.join(workspacePath, ".rooignore") + this.ignoreInstance = ignore() + } + + /** + * Initialize the controller by loading .rooignore patterns + */ + async initialize(): Promise { + if (this.initialized) { + return + } + + try { + const content = await fs.readFile(this.rooIgnorePath, "utf8") + const patterns = content + .split("\n") + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith("#")) + + this.ignoreInstance.add(patterns) + this.initialized = true + } catch (error) { + // If .rooignore doesn't exist, that's fine - no patterns to add + this.initialized = true + } + } + + /** + * Check if a path should be ignored + */ + isIgnored(filePath: string): boolean { + if (!this.initialized) { + throw new Error("RooIgnoreController not initialized. Call initialize() first.") + } + + const relativePath = path.relative(this.workspacePath, filePath) + return this.ignoreInstance.ignores(relativePath) + } + + /** + * Filter an array of paths, removing ignored ones + */ + filterPaths(paths: string[]): string[] { + if (!this.initialized) { + throw new Error("RooIgnoreController not initialized. Call initialize() first.") + } + + return paths.filter((filePath) => { + const relativePath = path.relative(this.workspacePath, filePath) + return !this.ignoreInstance.ignores(relativePath) + }) + } + + /** + * Add patterns to the ignore list + */ + addPatterns(patterns: string[]): void { + this.ignoreInstance.add(patterns) + } + + /** + * Get the current ignore instance + */ + getIgnoreInstance(): Ignore { + return this.ignoreInstance + } + + /** + * Reload patterns from the .rooignore file + */ + async reload(): Promise { + this.ignoreInstance = ignore() + this.initialized = false + await this.initialize() + } +} diff --git a/src/services/code-index/worker-utils/cache-manager.ts b/src/services/code-index/worker-utils/cache-manager.ts new file mode 100644 index 0000000000..729d7fe243 --- /dev/null +++ b/src/services/code-index/worker-utils/cache-manager.ts @@ -0,0 +1,115 @@ +// Worker-compatible version of cache-manager without vscode dependencies +import { createHash } from "crypto" +import { ICacheManager } from "../interfaces/cache" +import debounce from "lodash.debounce" +import * as fs from "fs/promises" +import * as path from "path" + +/** + * Manages the cache for code indexing (worker-compatible version) + */ +export class CacheManager implements ICacheManager { + private cachePath: string + private fileHashes: Record = {} + private _debouncedSaveCache: () => void + + /** + * Creates a new cache manager + * @param context Mock context with globalStorageUri + * @param workspacePath Path to the workspace + */ + constructor( + private context: { globalStorageUri: { fsPath: string } }, + private workspacePath: string, + ) { + const cacheFileName = `roo-index-cache-${createHash("sha256").update(workspacePath).digest("hex")}.json` + this.cachePath = path.join(context.globalStorageUri.fsPath, cacheFileName) + + this._debouncedSaveCache = debounce(async () => { + await this._performSave() + }, 1500) + } + + /** + * Initializes the cache manager by loading the cache file + */ + async initialize(): Promise { + try { + const cacheData = await fs.readFile(this.cachePath, "utf8") + this.fileHashes = JSON.parse(cacheData) + } catch (error) { + this.fileHashes = {} + } + } + + /** + * Saves the cache to disk + */ + private async _performSave(): Promise { + try { + // Ensure directory exists + const dir = path.dirname(this.cachePath) + await fs.mkdir(dir, { recursive: true }) + + // Write file atomically + const tempPath = `${this.cachePath}.tmp` + await fs.writeFile(tempPath, JSON.stringify(this.fileHashes, null, 2)) + await fs.rename(tempPath, this.cachePath) + } catch (error) { + console.error("Failed to save cache:", error) + } + } + + /** + * Clears the cache file by writing an empty object to it + */ + async clearCacheFile(): Promise { + try { + // Ensure directory exists + const dir = path.dirname(this.cachePath) + await fs.mkdir(dir, { recursive: true }) + + // Write empty cache + await fs.writeFile(this.cachePath, JSON.stringify({}, null, 2)) + this.fileHashes = {} + } catch (error) { + console.error("Failed to clear cache file:", error, this.cachePath) + } + } + + /** + * Gets the hash for a file path + * @param filePath Path to the file + * @returns The hash for the file or undefined if not found + */ + getHash(filePath: string): string | undefined { + return this.fileHashes[filePath] + } + + /** + * Updates the hash for a file path + * @param filePath Path to the file + * @param hash New hash value + */ + updateHash(filePath: string, hash: string): void { + this.fileHashes[filePath] = hash + this._debouncedSaveCache() + } + + /** + * Deletes the hash for a file path + * @param filePath Path to the file + */ + deleteHash(filePath: string): void { + delete this.fileHashes[filePath] + this._debouncedSaveCache() + } + + /** + * Gets a copy of all file hashes + * @returns A copy of the file hashes record + */ + getAllHashes(): Record { + return { ...this.fileHashes } + } +} diff --git a/src/services/code-index/worker-utils/file-watcher.ts b/src/services/code-index/worker-utils/file-watcher.ts new file mode 100644 index 0000000000..5ac32d13b6 --- /dev/null +++ b/src/services/code-index/worker-utils/file-watcher.ts @@ -0,0 +1,166 @@ +// Worker-compatible version of file-watcher without vscode dependencies +import * as chokidar from "chokidar" +import * as path from "path" +import ignore from "ignore" + +export interface FileWatcherOptions { + excludePatterns?: string[] + includePatterns?: string[] +} + +export interface FileChangeEvent { + type: "created" | "changed" | "deleted" + path: string +} + +/** + * Worker-compatible file watcher that monitors file system changes + */ +export class FileWatcher { + private watcher: chokidar.FSWatcher | null = null + private ignoreInstance: ReturnType + private listeners: ((event: FileChangeEvent) => void)[] = [] + + constructor( + private workspacePath: string, + private options: FileWatcherOptions = {}, + ) { + // Initialize ignore instance with exclude patterns + this.ignoreInstance = ignore() + if (options.excludePatterns) { + this.ignoreInstance.add(options.excludePatterns) + } + } + + /** + * Start watching for file changes + */ + start(): void { + if (this.watcher) { + return // Already watching + } + + // Configure chokidar options + const watchOptions = { + cwd: this.workspacePath, + ignored: (filePath: string) => { + const relativePath = path.relative(this.workspacePath, filePath) + return this.ignoreInstance.ignores(relativePath) + }, + persistent: true, + ignoreInitial: true, + followSymlinks: false, + usePolling: false, + interval: 100, + binaryInterval: 300, + awaitWriteFinish: { + stabilityThreshold: 200, + pollInterval: 100, + }, + } + + // Create watcher + this.watcher = chokidar.watch(this.workspacePath, watchOptions) + + // Set up event handlers + this.watcher + .on("add", (filePath: string) => { + this.emitEvent({ type: "created", path: path.join(this.workspacePath, filePath) }) + }) + .on("change", (filePath: string) => { + this.emitEvent({ type: "changed", path: path.join(this.workspacePath, filePath) }) + }) + .on("unlink", (filePath: string) => { + this.emitEvent({ type: "deleted", path: path.join(this.workspacePath, filePath) }) + }) + .on("error", (error: unknown) => { + console.error("[FileWatcher] Error:", error) + }) + } + + /** + * Stop watching for file changes + */ + async stop(): Promise { + if (this.watcher) { + await this.watcher.close() + this.watcher = null + } + } + + /** + * Add a listener for file change events + */ + onFileChange(listener: (event: FileChangeEvent) => void): void { + this.listeners.push(listener) + } + + /** + * Remove a listener + */ + removeListener(listener: (event: FileChangeEvent) => void): void { + const index = this.listeners.indexOf(listener) + if (index !== -1) { + this.listeners.splice(index, 1) + } + } + + /** + * Check if a file should be watched based on patterns + */ + shouldWatchFile(filePath: string): boolean { + const relativePath = path.relative(this.workspacePath, filePath) + + // Check if ignored + if (this.ignoreInstance.ignores(relativePath)) { + return false + } + + // If include patterns are specified, check if file matches any + if (this.options.includePatterns && this.options.includePatterns.length > 0) { + const ext = path.extname(filePath).toLowerCase() + const fileName = path.basename(filePath) + + return this.options.includePatterns.some((pattern) => { + // Handle simple extension patterns + if (pattern.startsWith("**/*") && pattern.indexOf("*", 4) === -1) { + const patternExt = pattern.substring(3) + return filePath.endsWith(patternExt) + } + // Handle specific file name patterns + if (pattern.startsWith("**/") && !pattern.includes("*", 3)) { + const patternName = pattern.substring(3) + return fileName === patternName + } + // For complex patterns, use simple matching + return filePath.includes(pattern.replace(/\*\*/g, "").replace(/\*/g, "")) + }) + } + + // If no include patterns, watch all non-ignored files + return true + } + + /** + * Emit an event to all listeners + */ + private emitEvent(event: FileChangeEvent): void { + // Check if file should be watched before emitting + if (this.shouldWatchFile(event.path)) { + for (const listener of this.listeners) { + try { + listener(event) + } catch (error) { + console.error("[FileWatcher] Error in listener:", error) + } + } + } + } + + /** + * Get watcher status + */ + isWatching(): boolean { + return this.watcher !== null + } +} diff --git a/src/services/code-index/worker-utils/scanner.ts b/src/services/code-index/worker-utils/scanner.ts new file mode 100644 index 0000000000..4af8705b73 --- /dev/null +++ b/src/services/code-index/worker-utils/scanner.ts @@ -0,0 +1,400 @@ +// Worker-compatible version of scanner without vscode dependencies +import * as fs from "fs/promises" +import * as path from "path" +import { glob } from "glob" +import ignore from "ignore" + +export interface ScanResult { + files: string[] + totalFiles: number +} + +/** + * Scans the workspace for files to index (worker-compatible version) + */ +export class Scanner { + private static readonly DEFAULT_EXCLUDE_PATTERNS = [ + "**/node_modules/**", + "**/.git/**", + "**/dist/**", + "**/build/**", + "**/out/**", + "**/.next/**", + "**/.nuxt/**", + "**/coverage/**", + "**/.nyc_output/**", + "**/.cache/**", + "**/.parcel-cache/**", + "**/.vscode/**", + "**/.idea/**", + "**/*.min.js", + "**/*.map", + "**/vendor/**", + "**/bower_components/**", + "**/.svn/**", + "**/.hg/**", + "**/.DS_Store", + "**/Thumbs.db", + "**/*.log", + "**/logs/**", + "**/tmp/**", + "**/temp/**", + "**/.env*", + "**/.git/**", + "**/.gitignore", + "**/.gitmodules", + "**/package-lock.json", + "**/yarn.lock", + "**/pnpm-lock.yaml", + "**/composer.lock", + "**/Gemfile.lock", + "**/Cargo.lock", + "**/poetry.lock", + "**/Pipfile.lock", + "**/.terraform/**", + "**/*.tfstate*", + "**/.serverless/**", + "**/cdk.out/**", + ] + + private static readonly INCLUDE_PATTERNS = [ + "**/*.ts", + "**/*.tsx", + "**/*.js", + "**/*.jsx", + "**/*.mjs", + "**/*.cjs", + "**/*.vue", + "**/*.svelte", + "**/*.py", + "**/*.pyw", + "**/*.pyx", + "**/*.pxd", + "**/*.pyi", + "**/*.java", + "**/*.kt", + "**/*.kts", + "**/*.scala", + "**/*.sc", + "**/*.go", + "**/*.rs", + "**/*.c", + "**/*.cc", + "**/*.cpp", + "**/*.cxx", + "**/*.c++", + "**/*.h", + "**/*.hh", + "**/*.hpp", + "**/*.hxx", + "**/*.h++", + "**/*.cs", + "**/*.fs", + "**/*.fsx", + "**/*.fsi", + "**/*.ml", + "**/*.mli", + "**/*.rb", + "**/*.rake", + "**/*.php", + "**/*.php3", + "**/*.php4", + "**/*.php5", + "**/*.phtml", + "**/*.swift", + "**/*.m", + "**/*.mm", + "**/*.dart", + "**/*.lua", + "**/*.pl", + "**/*.pm", + "**/*.t", + "**/*.sh", + "**/*.bash", + "**/*.zsh", + "**/*.fish", + "**/*.ps1", + "**/*.psm1", + "**/*.psd1", + "**/*.bat", + "**/*.cmd", + "**/*.r", + "**/*.R", + "**/*.jl", + "**/*.ex", + "**/*.exs", + "**/*.elm", + "**/*.clj", + "**/*.cljs", + "**/*.cljc", + "**/*.edn", + "**/*.erl", + "**/*.hrl", + "**/*.nim", + "**/*.nims", + "**/*.cr", + "**/*.d", + "**/*.zig", + "**/*.v", + "**/*.vsh", + "**/*.sql", + "**/*.md", + "**/*.mdx", + "**/*.rst", + "**/*.txt", + "**/*.json", + "**/*.jsonc", + "**/*.json5", + "**/*.yaml", + "**/*.yml", + "**/*.toml", + "**/*.xml", + "**/*.html", + "**/*.htm", + "**/*.xhtml", + "**/*.css", + "**/*.scss", + "**/*.sass", + "**/*.less", + "**/*.styl", + "**/Dockerfile", + "**/Containerfile", + "**/*.dockerfile", + "**/*.containerfile", + "**/docker-compose.yml", + "**/docker-compose.yaml", + "**/.env.example", + "**/.env.sample", + "**/Makefile", + "**/makefile", + "**/GNUmakefile", + "**/CMakeLists.txt", + "**/*.cmake", + "**/meson.build", + "**/BUILD", + "**/BUILD.bazel", + "**/WORKSPACE", + "**/*.bzl", + "**/*.gradle", + "**/*.gradle.kts", + "**/pom.xml", + "**/build.xml", + "**/*.sbt", + "**/Cargo.toml", + "**/go.mod", + "**/go.sum", + "**/package.json", + "**/tsconfig.json", + "**/jsconfig.json", + "**/webpack.config.js", + "**/webpack.config.ts", + "**/rollup.config.js", + "**/rollup.config.ts", + "**/vite.config.js", + "**/vite.config.ts", + "**/.eslintrc", + "**/.eslintrc.js", + "**/.eslintrc.json", + "**/.prettierrc", + "**/.prettierrc.js", + "**/.prettierrc.json", + "**/jest.config.js", + "**/jest.config.ts", + "**/vitest.config.js", + "**/vitest.config.ts", + "**/playwright.config.js", + "**/playwright.config.ts", + "**/cypress.config.js", + "**/cypress.config.ts", + "**/*.proto", + "**/*.graphql", + "**/*.gql", + "**/*.prisma", + "**/*.tf", + "**/*.tfvars", + "**/*.hcl", + "**/ansible.cfg", + "**/*.playbook.yml", + "**/*.playbook.yaml", + "**/requirements.txt", + "**/requirements.in", + "**/Pipfile", + "**/pyproject.toml", + "**/setup.py", + "**/setup.cfg", + "**/Gemfile", + "**/Rakefile", + "**/composer.json", + "**/*.gemspec", + "**/pubspec.yaml", + "**/pubspec.yml", + "**/*.cabal", + "**/stack.yaml", + "**/elm.json", + "**/deno.json", + "**/deno.jsonc", + "**/*.nimble", + "**/shard.yml", + "**/Project.toml", + "**/Manifest.toml", + "**/*.opam", + "**/rebar.config", + "**/erlang.mk", + "**/mix.exs", + "**/*.app.src", + "**/info.rkt", + "**/.gitignore", + "**/.dockerignore", + "**/.npmignore", + "**/.gitattributes", + "**/.editorconfig", + "**/LICENSE", + "**/LICENSE.txt", + "**/LICENSE.md", + "**/COPYING", + "**/README", + "**/README.txt", + "**/README.md", + "**/CHANGELOG", + "**/CHANGELOG.txt", + "**/CHANGELOG.md", + "**/CONTRIBUTING", + "**/CONTRIBUTING.txt", + "**/CONTRIBUTING.md", + "**/AUTHORS", + "**/AUTHORS.txt", + "**/AUTHORS.md", + "**/CONTRIBUTORS", + "**/CONTRIBUTORS.txt", + "**/CONTRIBUTORS.md", + "**/.github/workflows/*.yml", + "**/.github/workflows/*.yaml", + "**/.gitlab-ci.yml", + "**/.travis.yml", + "**/appveyor.yml", + "**/.circleci/config.yml", + "**/bitbucket-pipelines.yml", + "**/azure-pipelines.yml", + "**/Jenkinsfile", + "**/.drone.yml", + "**/.woodpecker.yml", + "**/cloudbuild.yaml", + "**/cloudbuild.yml", + "**/buildspec.yml", + "**/.buildkite/*.yml", + "**/netlify.toml", + "**/vercel.json", + "**/now.json", + "**/render.yaml", + "**/render.yml", + "**/app.json", + "**/Procfile", + "**/heroku.yml", + "**/fly.toml", + "**/.replit", + "**/replit.nix", + "**/.devcontainer/devcontainer.json", + "**/.devcontainer.json", + "**/Vagrantfile", + "**/.vagrant/**", + "**/*.code-workspace", + "**/.vscode/settings.json", + "**/.vscode/tasks.json", + "**/.vscode/launch.json", + "**/.vscode/extensions.json", + "**/.idea/*.xml", + "**/.fleet/settings.json", + ] + + constructor( + private workspacePath: string, + private excludePatterns: string[] = [], + private includePatterns: string[] = [], + ) {} + + /** + * Scans the workspace for files to index + */ + async scan(onProgress?: (processed: number, total: number) => void): Promise { + const allExcludePatterns = [...Scanner.DEFAULT_EXCLUDE_PATTERNS, ...this.excludePatterns] + const allIncludePatterns = this.includePatterns.length > 0 ? this.includePatterns : Scanner.INCLUDE_PATTERNS + + // Use glob to find all matching files + const files: string[] = [] + let processedCount = 0 + + for (const pattern of allIncludePatterns) { + const matches = await glob(pattern, { + cwd: this.workspacePath, + absolute: true, + nodir: true, + dot: true, + ignore: allExcludePatterns, + }) + + for (const file of matches) { + // Double-check exclusion patterns using ignore + const relativePath = path.relative(this.workspacePath, file) + const ig = ignore().add(allExcludePatterns) + const isExcluded = ig.ignores(relativePath) + + if (!isExcluded && !files.includes(file)) { + files.push(file) + } + + processedCount++ + if (onProgress && processedCount % 100 === 0) { + onProgress(processedCount, files.length) + } + } + } + + // Sort files for consistent ordering + files.sort() + + return { + files, + totalFiles: files.length, + } + } + + /** + * Checks if a file should be indexed based on include/exclude patterns + */ + shouldIndexFile(filePath: string): boolean { + const relativePath = path.relative(this.workspacePath, filePath) + + // Check exclude patterns first using ignore + const allExcludePatterns = [...Scanner.DEFAULT_EXCLUDE_PATTERNS, ...this.excludePatterns] + const ig = ignore().add(allExcludePatterns) + const isExcluded = ig.ignores(relativePath) + + if (isExcluded) { + return false + } + + // Check include patterns - for include patterns, we need to check if any pattern matches + const allIncludePatterns = this.includePatterns.length > 0 ? this.includePatterns : Scanner.INCLUDE_PATTERNS + + // Convert glob patterns to check if file matches any include pattern + const ext = path.extname(filePath).toLowerCase() + const fileName = path.basename(filePath) + + const isIncluded = allIncludePatterns.some((pattern) => { + // Handle simple extension patterns + if (pattern.startsWith("**/*") && pattern.indexOf("*", 4) === -1) { + const patternExt = pattern.substring(3) + return filePath.endsWith(patternExt) + } + // Handle specific file name patterns + if (pattern.startsWith("**/") && !pattern.includes("*", 3)) { + const patternName = pattern.substring(3) + return fileName === patternName + } + // For complex patterns, use glob matching + return filePath.includes(pattern.replace(/\*\*/g, "").replace(/\*/g, "")) + }) + + return isIncluded + } +} diff --git a/src/services/code-index/worker-utils/state-manager.ts b/src/services/code-index/worker-utils/state-manager.ts new file mode 100644 index 0000000000..b43e9ac93b --- /dev/null +++ b/src/services/code-index/worker-utils/state-manager.ts @@ -0,0 +1,125 @@ +// Worker-compatible version of state-manager without vscode dependencies +export type IndexingState = "Standby" | "Indexing" | "Indexed" | "Error" + +interface ProgressListener { + (status: ReturnType): void +} + +export class CodeIndexStateManager { + private _systemStatus: IndexingState = "Standby" + private _statusMessage: string = "" + private _processedItems: number = 0 + private _totalItems: number = 0 + private _currentItemUnit: string = "blocks" + private _progressListeners: ProgressListener[] = [] + + // --- Public API --- + + public onProgressUpdate(listener: ProgressListener): void { + this._progressListeners.push(listener) + } + + public get state(): IndexingState { + return this._systemStatus + } + + public getCurrentStatus() { + return { + systemStatus: this._systemStatus, + message: this._statusMessage, + processedItems: this._processedItems, + totalItems: this._totalItems, + currentItemUnit: this._currentItemUnit, + } + } + + // --- State Management --- + + public setSystemState(newState: IndexingState, message?: string): void { + const stateChanged = + newState !== this._systemStatus || (message !== undefined && message !== this._statusMessage) + + if (stateChanged) { + this._systemStatus = newState + if (message !== undefined) { + this._statusMessage = message + } + + // Reset progress counters if moving to a non-indexing state or starting fresh + if (newState !== "Indexing") { + this._processedItems = 0 + this._totalItems = 0 + this._currentItemUnit = "blocks" // Reset to default unit + // Optionally clear the message or set a default for non-indexing states + if (newState === "Standby" && message === undefined) this._statusMessage = "Ready." + if (newState === "Indexed" && message === undefined) this._statusMessage = "Index up-to-date." + if (newState === "Error" && message === undefined) this._statusMessage = "An error occurred." + } + + this._fireProgressUpdate() + } + } + + public reportBlockIndexingProgress(processedItems: number, totalItems: number): void { + const progressChanged = processedItems !== this._processedItems || totalItems !== this._totalItems + + // Update if progress changes OR if the system wasn't already in 'Indexing' state + if (progressChanged || this._systemStatus !== "Indexing") { + this._processedItems = processedItems + this._totalItems = totalItems + this._currentItemUnit = "blocks" + + const message = `Indexed ${this._processedItems} / ${this._totalItems} ${this._currentItemUnit} found` + const oldStatus = this._systemStatus + const oldMessage = this._statusMessage + + this._systemStatus = "Indexing" // Ensure state is Indexing + this._statusMessage = message + + // Only fire update if status, message or progress actually changed + if (oldStatus !== this._systemStatus || oldMessage !== this._statusMessage || progressChanged) { + this._fireProgressUpdate() + } + } + } + + public reportFileQueueProgress(processedFiles: number, totalFiles: number, currentFileBasename?: string): void { + const progressChanged = processedFiles !== this._processedItems || totalFiles !== this._totalItems + + if (progressChanged || this._systemStatus !== "Indexing") { + this._processedItems = processedFiles + this._totalItems = totalFiles + this._currentItemUnit = "files" + this._systemStatus = "Indexing" + + let message: string + if (totalFiles > 0 && processedFiles < totalFiles) { + message = `Processing ${processedFiles} / ${totalFiles} ${this._currentItemUnit}. Current: ${ + currentFileBasename || "..." + }` + } else if (totalFiles > 0 && processedFiles === totalFiles) { + message = `Finished processing ${totalFiles} ${this._currentItemUnit} from queue.` + } else { + message = `File queue processed.` + } + + const oldStatus = this._systemStatus + const oldMessage = this._statusMessage + + this._statusMessage = message + + if (oldStatus !== this._systemStatus || oldMessage !== this._statusMessage || progressChanged) { + this._fireProgressUpdate() + } + } + } + + private _fireProgressUpdate(): void { + const status = this.getCurrentStatus() + this._progressListeners.forEach((listener) => listener(status)) + } + + public dispose(): void { + this._progressListeners = [] + } +} diff --git a/src/workers/indexing-worker.ts b/src/workers/indexing-worker.ts index 04af80469e..7fdcd65da6 100644 --- a/src/workers/indexing-worker.ts +++ b/src/workers/indexing-worker.ts @@ -1,27 +1,33 @@ import { parentPort } from "worker_threads" import { WorkerCommand, WorkerResponse, WorkerMessage, WorkerInitConfig } from "../services/code-index/worker-messenger" -import { CodeIndexConfigManager } from "../services/code-index/config-manager" -import { CodeIndexStateManager, IndexingState } from "../services/code-index/state-manager" -import { CodeIndexServiceFactory } from "../services/code-index/service-factory" -import { CodeIndexOrchestrator } from "../services/code-index/orchestrator" -import { CodeIndexSearchService } from "../services/code-index/search-service" -import { CacheManager } from "../services/code-index/cache-manager" -import { DirectoryScanner } from "../services/code-index/processors" -import { IEmbedder, IVectorStore, IFileWatcher, VectorStoreSearchResult } from "../services/code-index/interfaces" +import { CodeIndexStateManager } from "../services/code-index/worker-utils/state-manager" +import { CacheManager } from "../services/code-index/worker-utils/cache-manager" +import { Scanner } from "../services/code-index/worker-utils/scanner" +import { FileWatcher } from "../services/code-index/worker-utils/file-watcher" +import { RooIgnoreController } from "../services/code-index/worker-utils/RooIgnoreController" +import { VectorStoreSearchResult } from "../services/code-index/interfaces" import ignore from "ignore" import * as fs from "fs/promises" import * as path from "path" +// Import embedders and vector store directly +import { OpenAiEmbedder } from "../services/code-index/embedders/openai" +import { CodeIndexOllamaEmbedder } from "../services/code-index/embedders/ollama" +import { OpenAICompatibleEmbedder } from "../services/code-index/embedders/openai-compatible" +import { QdrantVectorStore } from "../services/code-index/vector-store/qdrant-client" +import { codeParser } from "../services/code-index/processors" +import { EmbedderProvider, getDefaultModelId, getModelDimension } from "../shared/embeddingModels" + class IndexingWorker { private config: WorkerInitConfig | null = null - private orchestrator: CodeIndexOrchestrator | null = null - private searchService: CodeIndexSearchService | null = null private stateManager: CodeIndexStateManager | null = null private cacheManager: CacheManager | null = null - private embedder: IEmbedder | null = null - private vectorStore: IVectorStore | null = null - private scanner: DirectoryScanner | null = null - private fileWatcher: IFileWatcher | null = null + private scanner: Scanner | null = null + private fileWatcher: FileWatcher | null = null + private embedder: any = null + private vectorStore: any = null + private ignoreInstance: any = null + private rooIgnoreController: RooIgnoreController | null = null constructor() { if (!parentPort) { @@ -108,103 +114,256 @@ class IndexingWorker { }) // Initialize cache manager - this.cacheManager = new CacheManager( - { globalStorageUri: { fsPath: config.contextPath } } as any, - config.workspacePath, - ) + this.cacheManager = new CacheManager({ globalStorageUri: { fsPath: config.contextPath } }, config.workspacePath) await this.cacheManager.initialize() - // Create a mock config manager with the provided config - const configManager = this.createMockConfigManager(config) + // Initialize embedder based on config + this.embedder = this.createEmbedder(config) - // Initialize service factory - const serviceFactory = new CodeIndexServiceFactory(configManager, config.workspacePath, this.cacheManager) + // Initialize vector store + this.vectorStore = this.createVectorStore(config) // Load .gitignore - const ignoreInstance = ignore() + this.ignoreInstance = ignore() const ignorePath = path.join(config.workspacePath, ".gitignore") try { const content = await fs.readFile(ignorePath, "utf8") - ignoreInstance.add(content) - ignoreInstance.add(".gitignore") + this.ignoreInstance.add(content) + this.ignoreInstance.add(".gitignore") } catch (error) { console.error("Failed to load .gitignore:", error) } - // Create services - const services = serviceFactory.createServices( - { globalStorageUri: { fsPath: config.contextPath } } as any, - this.cacheManager, - ignoreInstance, - ) + // Initialize RooIgnoreController + this.rooIgnoreController = new RooIgnoreController(config.workspacePath) + await this.rooIgnoreController.initialize() - this.embedder = services.embedder - this.vectorStore = services.vectorStore - this.scanner = services.scanner - this.fileWatcher = services.fileWatcher + // Initialize scanner + this.scanner = new Scanner(config.workspacePath) - // Initialize orchestrator - this.orchestrator = new CodeIndexOrchestrator( - configManager, - this.stateManager, - config.workspacePath, - this.cacheManager, - this.vectorStore, - this.scanner, - this.fileWatcher, - ) - - // Initialize search service - this.searchService = new CodeIndexSearchService( - configManager, - this.stateManager, - this.embedder, - this.vectorStore, - ) + // Initialize file watcher + this.fileWatcher = new FileWatcher(config.workspacePath, { + excludePatterns: ["**/node_modules/**", "**/.git/**", "**/dist/**", "**/build/**", "**/out/**"], + }) } - private createMockConfigManager(config: WorkerInitConfig): CodeIndexConfigManager { - // Create a mock config manager that returns the worker config - return { - isFeatureEnabled: config.isFeatureEnabled, - isFeatureConfigured: config.isFeatureConfigured, - currentQdrantUrl: config.qdrantUrl || "http://localhost:6333", - currentEmbedderProvider: config.embedderProvider || "openai", - currentEmbedderBaseUrl: config.embedderBaseUrl, - currentEmbedderModelId: config.embedderModelId, - currentEmbedderApiKey: config.embedderApiKey, - currentSearchMinScore: config.searchMinScore || 0.7, - loadConfiguration: async () => ({ requiresRestart: false }), - } as any + private createEmbedder(config: WorkerInitConfig): any { + const provider = config.embedderProvider as EmbedderProvider + + if (provider === "openai") { + if (!config.embedderApiKey) { + throw new Error("OpenAI API key missing for embedder creation") + } + return new OpenAiEmbedder({ + openAiNativeApiKey: config.embedderApiKey, + openAiNativeBaseUrl: config.embedderBaseUrl, + openAiEmbeddingModelId: config.embedderModelId || getDefaultModelId(provider), + }) + } else if (provider === "ollama") { + if (!config.embedderBaseUrl) { + throw new Error("Ollama base URL missing for embedder creation") + } + return new CodeIndexOllamaEmbedder({ + ollamaBaseUrl: config.embedderBaseUrl, + ollamaModelId: config.embedderModelId || getDefaultModelId(provider), + }) + } else if (provider === "openai-compatible") { + if (!config.embedderBaseUrl || !config.embedderApiKey) { + throw new Error("OpenAI Compatible configuration missing for embedder creation") + } + return new OpenAICompatibleEmbedder( + config.embedderBaseUrl, + config.embedderApiKey, + config.embedderModelId || getDefaultModelId(provider), + ) + } + + throw new Error(`Invalid embedder type configured: ${provider}`) + } + + private createVectorStore(config: WorkerInitConfig): any { + const provider = config.embedderProvider as EmbedderProvider + const defaultModel = getDefaultModelId(provider) + const modelId = config.embedderModelId || defaultModel + + let vectorSize: number | undefined + + if (provider === "openai-compatible") { + // For openai-compatible, we need to get the dimension from somewhere + // Default to 1536 for now (OpenAI's dimension) + vectorSize = 1536 + } else { + vectorSize = getModelDimension(provider, modelId) + } + + if (vectorSize === undefined) { + throw new Error(`Could not determine vector dimension for model '${modelId}' with provider '${provider}'`) + } + + if (!config.qdrantUrl) { + throw new Error("Qdrant URL missing for vector store creation") + } + + return new QdrantVectorStore(config.workspacePath, config.qdrantUrl, vectorSize) } private async startIndexing() { - if (!this.orchestrator) { + if ( + !this.config || + !this.stateManager || + !this.scanner || + !this.embedder || + !this.vectorStore || + !this.cacheManager + ) { throw new Error("Worker not initialized") } - await this.orchestrator.startIndexing() + + this.stateManager.setSystemState("Indexing", "Initializing services...") + + try { + // Initialize vector store + const collectionCreated = await this.vectorStore.initialize() + if (collectionCreated) { + await this.cacheManager.clearCacheFile() + } + + this.stateManager.setSystemState("Indexing", "Services ready. Starting workspace scan...") + + // Perform the scan + const scanResult = await this.scanner.scan((processed, total) => { + this.stateManager!.reportFileQueueProgress(processed, total) + }) + + // Process files for indexing + let totalBlocksIndexed = 0 + const batchSize = 50 + const files = scanResult.files + + for (let i = 0; i < files.length; i += batchSize) { + const batch = files.slice(i, i + batchSize) + const blocks = [] + + for (const filePath of batch) { + try { + // Check if file should be indexed + if (this.rooIgnoreController?.isIgnored(filePath)) { + continue + } + + // Read file content + const content = await fs.readFile(filePath, "utf8") + + // Calculate hash + const { createHash } = await import("crypto") + const fileHash = createHash("sha256").update(content).digest("hex") + + // Check cache + const cachedHash = this.cacheManager.getHash(filePath) + if (cachedHash === fileHash) { + continue + } + + // Parse file + const fileBlocks = await codeParser.parseFile(filePath, { content, fileHash }) + blocks.push(...fileBlocks) + + // Update cache + this.cacheManager.updateHash(filePath, fileHash) + } catch (error) { + console.error(`Error processing file ${filePath}:`, error) + } + } + + // Create embeddings and store in vector store + if (blocks.length > 0) { + const texts = blocks.map((block) => block.content.trim()).filter((text) => text.length > 0) + if (texts.length > 0) { + const { embeddings } = await this.embedder.createEmbeddings(texts) + + // Prepare points for vector store + const points = blocks.map((block, index) => ({ + id: `${block.file_path}:${block.start_line}`, + vector: embeddings[index], + payload: { + filePath: block.file_path, + codeChunk: block.content, + startLine: block.start_line, + endLine: block.end_line, + }, + })) + + await this.vectorStore.upsertPoints(points) + totalBlocksIndexed += blocks.length + + this.stateManager.reportBlockIndexingProgress(totalBlocksIndexed, scanResult.totalFiles) + } + } + } + + // Start file watcher + if (this.fileWatcher) { + this.fileWatcher.onFileChange(async (event) => { + // Handle file changes + console.log(`File ${event.type}: ${event.path}`) + // TODO: Implement file change handling + }) + this.fileWatcher.start() + } + + this.stateManager.setSystemState("Indexed", "Indexing complete. File watcher started.") + } catch (error) { + console.error("Error during indexing:", error) + this.stateManager.setSystemState( + "Error", + `Indexing failed: ${error instanceof Error ? error.message : String(error)}`, + ) + throw error + } } private async stopIndexing() { - if (!this.orchestrator) { - throw new Error("Worker not initialized") + if (this.fileWatcher) { + await this.fileWatcher.stop() + } + if (this.stateManager) { + this.stateManager.setSystemState("Standby", "Indexing stopped") } - await this.orchestrator.stopWatcher() } private async clearIndex() { - if (!this.orchestrator || !this.cacheManager) { + if (!this.vectorStore || !this.cacheManager) { throw new Error("Worker not initialized") } - await this.orchestrator.clearIndexData() + + await this.vectorStore.deleteCollection() await this.cacheManager.clearCacheFile() + + if (this.stateManager) { + this.stateManager.setSystemState("Standby", "Index cleared") + } } private async search(query: string, directoryPrefix?: string): Promise { - if (!this.searchService) { + if (!this.embedder || !this.vectorStore) { throw new Error("Worker not initialized") } - return await this.searchService.searchIndex(query, directoryPrefix) + + // Create embedding for query + const { embeddings } = await this.embedder.createEmbeddings([query]) + const queryVector = embeddings[0] + + // Search in vector store + const results = await this.vectorStore.search(queryVector, 10, this.config?.searchMinScore || 0.7) + + // Filter by directory prefix if provided + if (directoryPrefix) { + return results.filter( + (result: VectorStoreSearchResult) => result.payload?.filePath?.startsWith(directoryPrefix) ?? false, + ) + } + + return results } private sendResponse(id: string, response: WorkerResponse) {