fix: resolve codebase indexing ignore pattern issues

- Add .gitignore fallback support when .rooignore does not exist
- Fix .rooignore changes requiring VSCode restart by enhancing file watchers
- Fix clearIndexData() ignoring .rooignore by recreating services
- Remove duplicate ignore filtering in DirectoryScanner
- Update RooIgnoreController to handle both .gitignore and .rooignore patterns
- Add proper cleanup and service lifecycle management

Fixes #5656

src/core/ignore/RooIgnoreController.ts

@@ -17,12 +17,15 @@ export class RooIgnoreController {
 	private ignoreInstance: Ignore
 	private disposables: vscode.Disposable[] = []
 	rooIgnoreContent: string | undefined
+	private gitIgnoreContent: string | undefined
+	private usingGitIgnoreFallback: boolean = false
 
 	constructor(cwd: string) {
 		this.cwd = cwd
 		this.ignoreInstance = ignore()
 		this.rooIgnoreContent = undefined
-		// Set up file watcher for .rooignore
+		this.gitIgnoreContent = undefined
+		// Set up file watcher for .rooignore and .gitignore
 		this.setupFileWatcher()
 	}
 
@@ -31,52 +34,85 @@ export class RooIgnoreController {
 	 * Must be called after construction and before using the controller
 	 */
 	async initialize(): Promise<void> {
-		await this.loadRooIgnore()
+		await this.loadIgnorePatterns()
 	}
 
 	/**
-	 * Set up the file watcher for .rooignore changes
+	 * Set up the file watcher for .rooignore and .gitignore changes
 	 */
 	private setupFileWatcher(): void {
 		const rooignorePattern = new vscode.RelativePattern(this.cwd, ".rooignore")
-		const fileWatcher = vscode.workspace.createFileSystemWatcher(rooignorePattern)
+		const rooignoreWatcher = vscode.workspace.createFileSystemWatcher(rooignorePattern)
 
-		// Watch for changes and updates
+		const gitignorePattern = new vscode.RelativePattern(this.cwd, ".gitignore")
+		const gitignoreWatcher = vscode.workspace.createFileSystemWatcher(gitignorePattern)
+
+		// Watch for .rooignore changes and updates
 		this.disposables.push(
-			fileWatcher.onDidChange(() => {
-				this.loadRooIgnore()
+			rooignoreWatcher.onDidChange(() => {
+				this.loadIgnorePatterns()
 			}),
-			fileWatcher.onDidCreate(() => {
-				this.loadRooIgnore()
+			rooignoreWatcher.onDidCreate(() => {
+				this.loadIgnorePatterns()
 			}),
-			fileWatcher.onDidDelete(() => {
-				this.loadRooIgnore()
+			rooignoreWatcher.onDidDelete(() => {
+				this.loadIgnorePatterns()
 			}),
 		)
 
-		// Add fileWatcher itself to disposables
-		this.disposables.push(fileWatcher)
+		// Watch for .gitignore changes and updates
+		this.disposables.push(
+			gitignoreWatcher.onDidChange(() => {
+				this.loadIgnorePatterns()
+			}),
+			gitignoreWatcher.onDidCreate(() => {
+				this.loadIgnorePatterns()
+			}),
+			gitignoreWatcher.onDidDelete(() => {
+				this.loadIgnorePatterns()
+			}),
+		)
+
+		// Add fileWatchers themselves to disposables
+		this.disposables.push(rooignoreWatcher, gitignoreWatcher)
 	}
 
 	/**
-	 * Load custom patterns from .rooignore if it exists
+	 * Load ignore patterns from .rooignore and .gitignore files
+	 * .rooignore takes precedence, but .gitignore is used as fallback
 	 */
-	private async loadRooIgnore(): Promise<void> {
+	private async loadIgnorePatterns(): Promise<void> {
 		try {
 			// Reset ignore instance to prevent duplicate patterns
 			this.ignoreInstance = ignore()
-			const ignorePath = path.join(this.cwd, ".rooignore")
-			if (await fileExistsAtPath(ignorePath)) {
-				const content = await fs.readFile(ignorePath, "utf8")
+			this.usingGitIgnoreFallback = false
+
+			const rooIgnorePath = path.join(this.cwd, ".rooignore")
+			const gitIgnorePath = path.join(this.cwd, ".gitignore")
+
+			// Check for .rooignore first
+			if (await fileExistsAtPath(rooIgnorePath)) {
+				const content = await fs.readFile(rooIgnorePath, "utf8")
 				this.rooIgnoreContent = content
 				this.ignoreInstance.add(content)
 				this.ignoreInstance.add(".rooignore")
 			} else {
 				this.rooIgnoreContent = undefined
+
+				// Fallback to .gitignore if .rooignore doesn't exist
+				if (await fileExistsAtPath(gitIgnorePath)) {
+					const content = await fs.readFile(gitIgnorePath, "utf8")
+					this.gitIgnoreContent = content
+					this.ignoreInstance.add(content)
+					this.ignoreInstance.add(".gitignore")
+					this.usingGitIgnoreFallback = true
+				} else {
+					this.gitIgnoreContent = undefined
+				}
 			}
 		} catch (error) {
 			// Should never happen: reading file failed even though it exists
-			console.error("Unexpected error loading .rooignore:", error)
+			console.error("Unexpected error loading ignore patterns:", error)
 		}
 	}
 
@@ -87,8 +123,8 @@ export class RooIgnoreController {
 	 * @returns true if file is accessible, false if ignored
 	 */
 	validateAccess(filePath: string): boolean {
-		// Always allow access if .rooignore does not exist
-		if (!this.rooIgnoreContent) {
+		// Always allow access if no ignore patterns are loaded
+		if (!this.rooIgnoreContent && !this.usingGitIgnoreFallback) {
 			return true
 		}
 		try {
@@ -121,8 +157,8 @@ export class RooIgnoreController {
 	 * @returns path of file that is being accessed if it is being accessed, undefined if command is allowed
 	 */
 	validateCommand(command: string): string | undefined {
-		// Always allow if no .rooignore exists
-		if (!this.rooIgnoreContent) {
+		// Always allow if no ignore patterns are loaded
+		if (!this.rooIgnoreContent && !this.usingGitIgnoreFallback) {
 			return undefined
 		}
 
@@ -200,14 +236,23 @@ export class RooIgnoreController {
 	}
 
 	/**
-	 * Get formatted instructions about the .rooignore file for the LLM
-	 * @returns Formatted instructions or undefined if .rooignore doesn't exist
+	 * Get formatted instructions about the ignore file for the LLM
+	 * @returns Formatted instructions or undefined if no ignore patterns exist
 	 */
 	getInstructions(): string | undefined {
-		if (!this.rooIgnoreContent) {
-			return undefined
+		if (this.rooIgnoreContent) {
+			return `# .rooignore\n\n(The following is provided by a root-level .rooignore file where the user has specified files and directories that should not be accessed. When using list_files, you'll notice a ${LOCK_TEXT_SYMBOL} next to files that are blocked. Attempting to access the file's contents e.g. through read_file will result in an error.)\n\n${this.rooIgnoreContent}\n.rooignore`
+		} else if (this.usingGitIgnoreFallback && this.gitIgnoreContent) {
+			return `# .gitignore (fallback)\n\n(The following is provided by a root-level .gitignore file that is being used as fallback ignore patterns since no .rooignore file exists. When using list_files, you'll notice a ${LOCK_TEXT_SYMBOL} next to files that are blocked. Attempting to access the file's contents e.g. through read_file will result in an error.)\n\n${this.gitIgnoreContent}\n.gitignore`
 		}
+		return undefined
+	}
 
-		return `# .rooignore\n\n(The following is provided by a root-level .rooignore file where the user has specified files and directories that should not be accessed. When using list_files, you'll notice a ${LOCK_TEXT_SYMBOL} next to files that are blocked. Attempting to access the file's contents e.g. through read_file will result in an error.)\n\n${this.rooIgnoreContent}\n.rooignore`
+	/**
+	 * Check if the controller is using .gitignore as fallback
+	 * @returns true if using .gitignore patterns because .rooignore doesn't exist
+	 */
+	isUsingGitIgnoreFallback(): boolean {
+		return this.usingGitIgnoreFallback
 	}
 }

src/core/ignore/__tests__/RooIgnoreController.spec.ts

@@ -153,7 +153,7 @@ describe("RooIgnoreController", () => {
 			await controller.initialize()
 
 			// Verify error was logged
-			expect(consoleSpy).toHaveBeenCalledWith("Unexpected error loading .rooignore:", expect.any(Error))
+			expect(consoleSpy).toHaveBeenCalledWith("Unexpected error loading ignore patterns:", expect.any(Error))
 
 			// Cleanup
 			consoleSpy.mockRestore()

src/services/code-index/manager.ts

@@ -27,6 +27,7 @@ export class CodeIndexManager {
 	private _orchestrator: CodeIndexOrchestrator | undefined
 	private _searchService: CodeIndexSearchService | undefined
 	private _cacheManager: CacheManager | undefined
+	private _rooIgnoreController: RooIgnoreController | undefined
 
 	// Flag to prevent race conditions during error recovery
 	private _isRecoveringFromError = false
@@ -250,12 +251,15 @@ export class CodeIndexManager {
 		if (this._orchestrator) {
 			this.stopWatcher()
 		}
+		if (this._rooIgnoreController) {
+			this._rooIgnoreController.dispose()
+		}
 		this._stateManager.dispose()
 	}
 
 	/**
 	 * Clears all index data by stopping the watcher, clearing the Qdrant collection,
-	 * and deleting the cache file.
+	 * and deleting the cache file. Also recreates services to ensure ignore patterns are reloaded.
 	 */
 	public async clearIndexData(): Promise<void> {
 		if (!this.isFeatureEnabled) {
@@ -264,6 +268,10 @@ export class CodeIndexManager {
 		this.assertInitialized()
 		await this._orchestrator!.clearIndexData()
 		await this._cacheManager!.clearCacheFile()
+
+		// Recreate services to ensure ignore patterns are properly reloaded
+		// This fixes the issue where .rooignore is ignored after clearing index data
+		await this._recreateServices()
 	}
 
 	// --- Private Helpers ---
@@ -332,6 +340,9 @@ export class CodeIndexManager {
 		const rooIgnoreController = new RooIgnoreController(workspacePath)
 		await rooIgnoreController.initialize()
 
+		// Store reference to the RooIgnoreController for cleanup
+		this._rooIgnoreController = rooIgnoreController
+
 		// (Re)Create shared service instances
 		const { embedder, vectorStore, scanner, fileWatcher } = this._serviceFactory.createServices(
 			this.context,

src/services/code-index/processors/scanner.ts

@@ -84,23 +84,25 @@ export class DirectoryScanner implements IDirectoryScanner {
 
 		// Initialize RooIgnoreController if not provided
 		const ignoreController = new RooIgnoreController(directoryPath)
-
 		await ignoreController.initialize()
 
-		// Filter paths using .rooignore
+		// Filter paths using .rooignore/.gitignore patterns
 		const allowedPaths = ignoreController.filterPaths(filePaths)
 
-		// Filter by supported extensions, ignore patterns, and excluded directories
+		// Filter by supported extensions and excluded directories
+		// Note: We removed the duplicate this.ignoreInstance.ignores() check here because:
+		// 1. The RooIgnoreController already handles both .gitignore and .rooignore patterns
+		// 2. The listFiles() function already applies .gitignore filtering via ripgrep
+		// 3. This prevents double-filtering and ensures consistent ignore behavior
 		const supportedPaths = allowedPaths.filter((filePath) => {
 			const ext = path.extname(filePath).toLowerCase()
-			const relativeFilePath = generateRelativeFilePath(filePath, scanWorkspace)
 
 			// Check if file is in an ignored directory using the shared helper
 			if (isPathInIgnoredDirectory(filePath)) {
 				return false
 			}
 
-			return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath)
+			return scannerExtensions.includes(ext)
 		})
 
 		// Initialize tracking variables