feat: add .rooindex support for code indexer to override gitignore

- Created RooIndexController to handle .rooindex file patterns
- Updated list-files.ts to support includeGitignored parameter
- Modified scanner.ts to use RooIndexController for override logic
- Added comprehensive tests for RooIndexController
- Allows developers to specify files that should be indexed even if gitignored

This addresses the need for indexing generated code, nested repositories,
and other files excluded from version control but valuable for AI context.
This commit is contained in:
Roo Code 2025-09-11 01:25:50 +00:00
parent 8fee3127ff
commit b1bde61ae3
4 changed files with 512 additions and 23 deletions

View file

@ -0,0 +1,173 @@
import path from "path"
import { fileExistsAtPath } from "../../utils/fs"
import fs from "fs/promises"
import ignore, { Ignore } from "ignore"
import * as vscode from "vscode"
/**
* Controls code indexer file inclusion by providing override patterns for gitignored files.
* Uses the 'ignore' library to support standard .gitignore syntax in .rooindex files.
*
* The .rooindex file allows developers to specify patterns for files that should be
* indexed even if they are gitignored. This is useful for:
* - Generated code (TypeScript definitions, API clients)
* - Meta-repository patterns with nested repositories
* - Monorepos with selective version control
* - Projects with generated documentation or configuration
*/
export class RooIndexController {
private cwd: string
private includeInstance: Ignore
private disposables: vscode.Disposable[] = []
rooIndexContent: string | undefined
constructor(cwd: string) {
this.cwd = cwd
this.includeInstance = ignore()
this.rooIndexContent = undefined
// Set up file watcher for .rooindex
this.setupFileWatcher()
}
/**
* Initialize the controller by loading custom patterns
* Must be called after construction and before using the controller
*/
async initialize(): Promise<void> {
await this.loadRooIndex()
}
/**
* Set up the file watcher for .rooindex changes
*/
private setupFileWatcher(): void {
const rooindexPattern = new vscode.RelativePattern(this.cwd, ".rooindex")
const fileWatcher = vscode.workspace.createFileSystemWatcher(rooindexPattern)
// Watch for changes and updates
this.disposables.push(
fileWatcher.onDidChange(() => {
this.loadRooIndex()
}),
fileWatcher.onDidCreate(() => {
this.loadRooIndex()
}),
fileWatcher.onDidDelete(() => {
this.loadRooIndex()
}),
)
// Add fileWatcher itself to disposables
this.disposables.push(fileWatcher)
}
/**
* Load custom patterns from .rooindex if it exists
*/
private async loadRooIndex(): Promise<void> {
try {
// Reset include instance to prevent duplicate patterns
this.includeInstance = ignore()
const indexPath = path.join(this.cwd, ".rooindex")
if (await fileExistsAtPath(indexPath)) {
const content = await fs.readFile(indexPath, "utf8")
this.rooIndexContent = content
// Add patterns to the include instance
// Note: We're using ignore library in reverse - patterns match what to INCLUDE
this.includeInstance.add(content)
} else {
this.rooIndexContent = undefined
}
} catch (error) {
// Should never happen: reading file failed even though it exists
console.error("Unexpected error loading .rooindex:", error)
}
}
/**
* Check if a file should be included for indexing based on .rooindex patterns
* @param filePath - Path to check (relative to cwd or absolute)
* @returns true if file matches an inclusion pattern, false otherwise
*/
shouldInclude(filePath: string): boolean {
// If .rooindex does not exist, no overrides
if (!this.rooIndexContent) {
return false
}
try {
// Convert to relative path for pattern matching
let relativePath: string
if (path.isAbsolute(filePath)) {
relativePath = path.relative(this.cwd, filePath)
} else {
relativePath = filePath
}
// Normalize path separators for cross-platform compatibility
relativePath = relativePath.replace(/\\/g, "/")
// Check if the path matches any include pattern
// We're using the ignore library to match patterns, but we want inclusion behavior
// The library returns true if a path should be ignored, but we're using it for inclusion
// So if ignores() returns true, it means the path matches our inclusion pattern
return this.includeInstance.ignores(relativePath)
} catch (error) {
// On error, don't include the file
return false
}
}
/**
* Filter an array of paths to include only those that match .rooindex patterns
* @param paths - Array of paths to filter
* @returns Array of paths that match inclusion patterns
*/
filterForInclusion(paths: string[]): string[] {
if (!this.rooIndexContent) {
return []
}
try {
return paths.filter((p) => this.shouldInclude(p))
} catch (error) {
console.error("Error filtering paths for inclusion:", error)
return []
}
}
/**
* Check if a file that would normally be gitignored should be included for indexing
* @param filePath - Path to check
* @param isGitignored - Whether the file is gitignored
* @returns true if the file should be included despite being gitignored
*/
shouldOverrideGitignore(filePath: string, isGitignored: boolean): boolean {
// If not gitignored, no need to override
if (!isGitignored) {
return false
}
// Check if .rooindex says to include this gitignored file
return this.shouldInclude(filePath)
}
/**
* Clean up resources when the controller is no longer needed
*/
dispose(): void {
this.disposables.forEach((d) => d.dispose())
this.disposables = []
}
/**
* Get formatted instructions about the .rooindex file
* @returns Formatted instructions or undefined if .rooindex doesn't exist
*/
getInstructions(): string | undefined {
if (!this.rooIndexContent) {
return undefined
}
return `# .rooindex\n\n(The following patterns from .rooindex specify files that should be indexed even if they are gitignored. This allows the code indexer to access generated code, nested repositories, and other files excluded from version control but valuable for AI context.)\n\n${this.rooIndexContent}`
}
}

View file

@ -0,0 +1,251 @@
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"
import { RooIndexController } from "../RooIndexController"
import * as fs from "fs/promises"
import * as path from "path"
import * as vscode from "vscode"
import { fileExistsAtPath } from "../../../utils/fs"
// Mock dependencies
vi.mock("fs/promises")
vi.mock("../../../utils/fs")
vi.mock("vscode", () => ({
workspace: {
createFileSystemWatcher: vi.fn(() => ({
onDidChange: vi.fn(() => ({ dispose: vi.fn() })),
onDidCreate: vi.fn(() => ({ dispose: vi.fn() })),
onDidDelete: vi.fn(() => ({ dispose: vi.fn() })),
dispose: vi.fn(),
})),
},
RelativePattern: vi.fn((base, pattern) => ({ base, pattern })),
}))
describe("RooIndexController", () => {
let controller: RooIndexController
const testCwd = "/test/workspace"
beforeEach(() => {
vi.clearAllMocks()
controller = new RooIndexController(testCwd)
})
afterEach(() => {
controller.dispose()
})
describe("initialization", () => {
it("should initialize with no content when .rooindex doesn't exist", async () => {
vi.mocked(fileExistsAtPath).mockResolvedValue(false)
await controller.initialize()
expect(controller.rooIndexContent).toBeUndefined()
})
it("should load .rooindex content when file exists", async () => {
const mockContent = "generated/\nnode_modules/\n*.min.js"
vi.mocked(fileExistsAtPath).mockResolvedValue(true)
vi.mocked(fs.readFile).mockResolvedValue(mockContent)
await controller.initialize()
expect(controller.rooIndexContent).toBe(mockContent)
expect(fs.readFile).toHaveBeenCalledWith(path.join(testCwd, ".rooindex"), "utf8")
})
it("should handle read errors gracefully", async () => {
vi.mocked(fileExistsAtPath).mockResolvedValue(true)
vi.mocked(fs.readFile).mockRejectedValue(new Error("Read error"))
const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {})
await controller.initialize()
expect(controller.rooIndexContent).toBeUndefined()
expect(consoleSpy).toHaveBeenCalledWith("Unexpected error loading .rooindex:", expect.any(Error))
consoleSpy.mockRestore()
})
})
describe("shouldInclude", () => {
it("should return false when .rooindex doesn't exist", () => {
controller.rooIndexContent = undefined
expect(controller.shouldInclude("generated/api.ts")).toBe(false)
expect(controller.shouldInclude("node_modules/package/index.js")).toBe(false)
})
it("should match patterns from .rooindex", async () => {
const mockContent = "generated/\n*.min.js\napi-client/**"
vi.mocked(fileExistsAtPath).mockResolvedValue(true)
vi.mocked(fs.readFile).mockResolvedValue(mockContent)
await controller.initialize()
// These should match the patterns
expect(controller.shouldInclude("generated/api.ts")).toBe(true)
expect(controller.shouldInclude("generated/types.ts")).toBe(true)
expect(controller.shouldInclude("bundle.min.js")).toBe(true)
expect(controller.shouldInclude("api-client/index.ts")).toBe(true)
expect(controller.shouldInclude("api-client/models/user.ts")).toBe(true)
// These should not match
expect(controller.shouldInclude("src/index.ts")).toBe(false)
expect(controller.shouldInclude("test.js")).toBe(false)
})
it("should handle absolute paths correctly", async () => {
const mockContent = "generated/"
vi.mocked(fileExistsAtPath).mockResolvedValue(true)
vi.mocked(fs.readFile).mockResolvedValue(mockContent)
await controller.initialize()
// Absolute path that should match
expect(controller.shouldInclude(path.join(testCwd, "generated/api.ts"))).toBe(true)
// Relative path that should match
expect(controller.shouldInclude("generated/api.ts")).toBe(true)
})
it("should normalize path separators for cross-platform compatibility", async () => {
const mockContent = "generated/**/*.ts"
vi.mocked(fileExistsAtPath).mockResolvedValue(true)
vi.mocked(fs.readFile).mockResolvedValue(mockContent)
await controller.initialize()
// Windows-style path should work
expect(controller.shouldInclude("generated\\models\\user.ts")).toBe(true)
// Unix-style path should work
expect(controller.shouldInclude("generated/models/user.ts")).toBe(true)
})
})
describe("filterForInclusion", () => {
it("should return empty array when .rooindex doesn't exist", () => {
controller.rooIndexContent = undefined
const paths = ["generated/api.ts", "src/index.ts", "node_modules/pkg/index.js"]
const result = controller.filterForInclusion(paths)
expect(result).toEqual([])
})
it("should filter paths based on .rooindex patterns", async () => {
const mockContent = "generated/\n*.min.js"
vi.mocked(fileExistsAtPath).mockResolvedValue(true)
vi.mocked(fs.readFile).mockResolvedValue(mockContent)
await controller.initialize()
const paths = ["generated/api.ts", "generated/types.ts", "src/index.ts", "bundle.min.js", "test.js"]
const result = controller.filterForInclusion(paths)
expect(result).toEqual(["generated/api.ts", "generated/types.ts", "bundle.min.js"])
})
})
describe("shouldOverrideGitignore", () => {
it("should return false if file is not gitignored", async () => {
const mockContent = "generated/"
vi.mocked(fileExistsAtPath).mockResolvedValue(true)
vi.mocked(fs.readFile).mockResolvedValue(mockContent)
await controller.initialize()
expect(controller.shouldOverrideGitignore("src/index.ts", false)).toBe(false)
})
it("should return false if file is gitignored but not in .rooindex", async () => {
const mockContent = "generated/"
vi.mocked(fileExistsAtPath).mockResolvedValue(true)
vi.mocked(fs.readFile).mockResolvedValue(mockContent)
await controller.initialize()
// node_modules/pkg/index.js doesn't match "generated/" pattern
expect(controller.shouldOverrideGitignore("node_modules/pkg/index.js", true)).toBe(false)
})
it("should return true if file is gitignored and matches .rooindex pattern", async () => {
const mockContent = "generated/\nnode_modules/my-local-package/"
vi.mocked(fileExistsAtPath).mockResolvedValue(true)
vi.mocked(fs.readFile).mockResolvedValue(mockContent)
await controller.initialize()
expect(controller.shouldOverrideGitignore("generated/api.ts", true)).toBe(true)
expect(controller.shouldOverrideGitignore("node_modules/my-local-package/index.js", true)).toBe(true)
})
})
describe("getInstructions", () => {
it("should return undefined when .rooindex doesn't exist", () => {
controller.rooIndexContent = undefined
expect(controller.getInstructions()).toBeUndefined()
})
it("should return formatted instructions when .rooindex exists", async () => {
const mockContent = "generated/\n*.min.js"
vi.mocked(fileExistsAtPath).mockResolvedValue(true)
vi.mocked(fs.readFile).mockResolvedValue(mockContent)
await controller.initialize()
const instructions = controller.getInstructions()
expect(instructions).toContain("# .rooindex")
expect(instructions).toContain(mockContent)
expect(instructions).toContain("files that should be indexed even if they are gitignored")
})
})
describe("file watching", () => {
it("should set up file watcher on construction", () => {
const mockWatcher = {
onDidChange: vi.fn(() => ({ dispose: vi.fn() })),
onDidCreate: vi.fn(() => ({ dispose: vi.fn() })),
onDidDelete: vi.fn(() => ({ dispose: vi.fn() })),
dispose: vi.fn(),
}
vi.mocked(vscode.workspace.createFileSystemWatcher).mockReturnValue(mockWatcher as any)
const newController = new RooIndexController(testCwd)
expect(vscode.workspace.createFileSystemWatcher).toHaveBeenCalledWith(
expect.objectContaining({
base: testCwd,
pattern: ".rooindex",
}),
)
expect(mockWatcher.onDidChange).toHaveBeenCalled()
expect(mockWatcher.onDidCreate).toHaveBeenCalled()
expect(mockWatcher.onDidDelete).toHaveBeenCalled()
newController.dispose()
})
})
describe("dispose", () => {
it("should dispose all resources", () => {
const mockDisposable = { dispose: vi.fn() }
const mockWatcher = {
onDidChange: vi.fn(() => mockDisposable),
onDidCreate: vi.fn(() => mockDisposable),
onDidDelete: vi.fn(() => mockDisposable),
dispose: vi.fn(),
}
vi.mocked(vscode.workspace.createFileSystemWatcher).mockReturnValue(mockWatcher as any)
const newController = new RooIndexController(testCwd)
newController.dispose()
expect(mockDisposable.dispose).toHaveBeenCalledTimes(3) // For each event handler
expect(mockWatcher.dispose).toHaveBeenCalled()
})
})
})

View file

@ -1,6 +1,7 @@
import { listFiles } from "../../glob/list-files"
import { Ignore } from "ignore"
import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController"
import { RooIndexController } from "../../../core/index/RooIndexController"
import { stat } from "fs/promises"
import * as path from "path"
import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path"
@ -76,8 +77,12 @@ export class DirectoryScanner implements IDirectoryScanner {
// Capture workspace context at scan start
const scanWorkspace = getWorkspacePathForContext(directoryPath)
// Get all files recursively (handles .gitignore automatically)
const [allPaths, _] = await listFiles(directoryPath, true, MAX_LIST_FILES_LIMIT_CODE_INDEX)
// Initialize RooIndexController to handle .rooindex overrides
const rooIndexController = new RooIndexController(directoryPath)
await rooIndexController.initialize()
// Get all files recursively (handles .gitignore automatically, but includes overrides from .rooindex)
const [allPaths, _] = await listFiles(directoryPath, true, MAX_LIST_FILES_LIMIT_CODE_INDEX, true)
// Filter out directories (marked with trailing '/')
const filePaths = allPaths.filter((p) => !p.endsWith("/"))
@ -97,10 +102,21 @@ export class DirectoryScanner implements IDirectoryScanner {
// Check if file is in an ignored directory using the shared helper
if (isPathInIgnoredDirectory(filePath)) {
return false
// Check if .rooindex overrides this
if (!rooIndexController.shouldInclude(filePath)) {
return false
}
}
return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath)
// Check gitignore patterns
const isGitignored = this.ignoreInstance.ignores(relativeFilePath)
// If gitignored, check if .rooindex overrides it
if (isGitignored && rooIndexController.shouldInclude(filePath)) {
return scannerExtensions.includes(ext)
}
return scannerExtensions.includes(ext) && !isGitignored
})
// Initialize tracking variables

View file

@ -7,6 +7,7 @@ import ignore from "ignore"
import { arePathsEqual } from "../../utils/path"
import { getBinPath } from "../../services/ripgrep"
import { DIRS_TO_IGNORE } from "./constants"
import { RooIndexController } from "../../core/index/RooIndexController"
/**
* Context object for directory scanning operations
@ -20,6 +21,8 @@ interface ScanContext {
basePath: string
/** The ignore instance for gitignore handling */
ignoreInstance: ReturnType<typeof ignore>
/** The RooIndexController for handling .rooindex overrides */
rooIndexController?: RooIndexController
}
/**
@ -30,7 +33,12 @@ interface ScanContext {
* @param limit - Maximum number of files to return
* @returns Tuple of [file paths array, whether the limit was reached]
*/
export async function listFiles(dirPath: string, recursive: boolean, limit: number): Promise<[string[], boolean]> {
export async function listFiles(
dirPath: string,
recursive: boolean,
limit: number,
includeGitignored: boolean = false,
): Promise<[string[], boolean]> {
// Early return for limit of 0 - no need to scan anything
if (limit === 0) {
return [[], false]
@ -46,29 +54,42 @@ export async function listFiles(dirPath: string, recursive: boolean, limit: numb
// Get ripgrep path
const rgPath = await getRipgrepPath()
// Initialize RooIndexController if we need to include gitignored files
let rooIndexController: RooIndexController | undefined
if (includeGitignored) {
rooIndexController = new RooIndexController(dirPath)
await rooIndexController.initialize()
}
if (!recursive) {
// For non-recursive, use the existing approach
const files = await listFilesWithRipgrep(rgPath, dirPath, false, limit)
const files = await listFilesWithRipgrep(rgPath, dirPath, false, limit, includeGitignored)
const ignoreInstance = await createIgnoreInstance(dirPath)
// Calculate remaining limit for directories
const remainingLimit = Math.max(0, limit - files.length)
const directories = await listFilteredDirectories(dirPath, false, ignoreInstance, remainingLimit)
const directories = await listFilteredDirectories(
dirPath,
false,
ignoreInstance,
remainingLimit,
rooIndexController,
)
return formatAndCombineResults(files, directories, limit)
}
// For recursive mode, use the original approach but ensure first-level directories are included
const files = await listFilesWithRipgrep(rgPath, dirPath, true, limit)
const files = await listFilesWithRipgrep(rgPath, dirPath, true, limit, includeGitignored)
const ignoreInstance = await createIgnoreInstance(dirPath)
// Calculate remaining limit for directories
const remainingLimit = Math.max(0, limit - files.length)
const directories = await listFilteredDirectories(dirPath, true, ignoreInstance, remainingLimit)
const directories = await listFilteredDirectories(dirPath, true, ignoreInstance, remainingLimit, rooIndexController)
// Combine and check if we hit the limits
const [results, limitReached] = formatAndCombineResults(files, directories, limit)
// If we hit the limit, ensure all first-level directories are included
if (limitReached) {
const firstLevelDirs = await getFirstLevelDirectories(dirPath, ignoreInstance)
const firstLevelDirs = await getFirstLevelDirectories(dirPath, ignoreInstance, rooIndexController)
return ensureFirstLevelDirectoriesIncluded(results, firstLevelDirs, limit)
}
@ -78,7 +99,11 @@ export async function listFiles(dirPath: string, recursive: boolean, limit: numb
/**
* Get only the first-level directories in a path
*/
async function getFirstLevelDirectories(dirPath: string, ignoreInstance: ReturnType<typeof ignore>): Promise<string[]> {
async function getFirstLevelDirectories(
dirPath: string,
ignoreInstance: ReturnType<typeof ignore>,
rooIndexController?: RooIndexController,
): Promise<string[]> {
const absolutePath = path.resolve(dirPath)
const directories: string[] = []
@ -93,6 +118,7 @@ async function getFirstLevelDirectories(dirPath: string, ignoreInstance: ReturnT
insideExplicitHiddenTarget: false,
basePath: dirPath,
ignoreInstance,
rooIndexController,
}
if (shouldIncludeDirectory(entry.name, fullDirPath, context)) {
const formattedPath = fullDirPath.endsWith("/") ? fullDirPath : `${fullDirPath}/`
@ -202,8 +228,9 @@ async function listFilesWithRipgrep(
dirPath: string,
recursive: boolean,
limit: number,
includeGitignored: boolean = false,
): Promise<string[]> {
const rgArgs = buildRipgrepArgs(dirPath, recursive)
const rgArgs = buildRipgrepArgs(dirPath, recursive, includeGitignored)
const relativePaths = await execRipgrep(rgPath, rgArgs, limit)
@ -216,25 +243,28 @@ async function listFilesWithRipgrep(
/**
* Build appropriate ripgrep arguments based on whether we're doing a recursive search
*/
function buildRipgrepArgs(dirPath: string, recursive: boolean): string[] {
function buildRipgrepArgs(dirPath: string, recursive: boolean, includeGitignored: boolean = false): string[] {
// Base arguments to list files
const args = ["--files", "--hidden", "--follow"]
if (recursive) {
return [...args, ...buildRecursiveArgs(dirPath), dirPath]
return [...args, ...buildRecursiveArgs(dirPath, includeGitignored), dirPath]
} else {
return [...args, ...buildNonRecursiveArgs(), dirPath]
return [...args, ...buildNonRecursiveArgs(includeGitignored), dirPath]
}
}
/**
* Build ripgrep arguments for recursive directory traversal
*/
function buildRecursiveArgs(dirPath: string): string[] {
function buildRecursiveArgs(dirPath: string, includeGitignored: boolean = false): string[] {
const args: string[] = []
// In recursive mode, respect .gitignore by default
// (ripgrep does this automatically)
// If we want to include gitignored files (for .rooindex support), disable gitignore
if (includeGitignored) {
args.push("--no-ignore-vcs")
}
// Otherwise, respect .gitignore by default (ripgrep does this automatically)
// Check if we're explicitly targeting a hidden directory
// Normalize the path first to handle edge cases
@ -295,15 +325,18 @@ function buildRecursiveArgs(dirPath: string): string[] {
/**
* Build ripgrep arguments for non-recursive directory listing
*/
function buildNonRecursiveArgs(): string[] {
function buildNonRecursiveArgs(includeGitignored: boolean = false): string[] {
const args: string[] = []
// For non-recursive, limit to the current directory level
args.push("-g", "*")
args.push("--maxdepth", "1") // ripgrep uses maxdepth, not max-depth
// Respect .gitignore in non-recursive mode too
// (ripgrep respects .gitignore by default)
// If we want to include gitignored files (for .rooindex support), disable gitignore
if (includeGitignored) {
args.push("--no-ignore-vcs")
}
// Otherwise, respect .gitignore in non-recursive mode too (ripgrep respects .gitignore by default)
// Apply directory exclusions for non-recursive searches
for (const dir of DIRS_TO_IGNORE) {
@ -389,6 +422,7 @@ async function listFilteredDirectories(
recursive: boolean,
ignoreInstance: ReturnType<typeof ignore>,
limit?: number,
rooIndexController?: RooIndexController,
): Promise<string[]> {
const absolutePath = path.resolve(dirPath)
const directories: string[] = []
@ -406,6 +440,7 @@ async function listFilteredDirectories(
insideExplicitHiddenTarget: isExplicitHiddenTarget,
basePath: dirPath,
ignoreInstance,
rooIndexController,
}
async function scanDirectory(currentPath: string, context: ScanContext): Promise<boolean> {
@ -554,7 +589,14 @@ function shouldIncludeInsideHiddenTarget(dirName: string, fullDirPath: string, c
}
// Check against gitignore patterns
return !isIgnoredByGitignore(fullDirPath, context.basePath, context.ignoreInstance)
const isGitignored = isIgnoredByGitignore(fullDirPath, context.basePath, context.ignoreInstance)
// If we have a RooIndexController and the directory is gitignored, check for override
if (context.rooIndexController && isGitignored) {
return context.rooIndexController.shouldInclude(fullDirPath)
}
return !isGitignored
}
/**
@ -568,7 +610,14 @@ function shouldIncludeRegularDirectory(dirName: string, fullDirPath: string, con
}
// Check against gitignore patterns
return !isIgnoredByGitignore(fullDirPath, context.basePath, context.ignoreInstance)
const isGitignored = isIgnoredByGitignore(fullDirPath, context.basePath, context.ignoreInstance)
// If we have a RooIndexController and the directory is gitignored, check for override
if (context.rooIndexController && isGitignored) {
return context.rooIndexController.shouldInclude(fullDirPath)
}
return !isGitignored
}
/**