scriptoriumadmin/Roo-Code
1
0
Fork 0
mirror of https://github.com/RooVetGit/Roo-Code.git synced 2026-10-06 02:47:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix #5301: Prevent early termination in list-files when directory has many files

Browse source 
- Remove early process termination in execRipgrep that was causing incomplete directory traversal
- Implement balanced sampling algorithm to ensure fair representation across all directories
- Increase timeout values to allow complete directory scanning before applying limits
- Add applyBalancedSampling function to distribute file selection evenly across directories
- Maintain existing 200-file limit while ensuring all directories are represented

This fixes the issue where having 200+ files in one directory (e.g., 'a/') would cause
other directories (e.g., 'b/') to be completely ignored in the file listing results.
This commit is contained in:
Roo Code 2025-07-01 15:59:57 +00:00
parent 3a8ba27615
commit aa77ada1f9
219 changed files with 381 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

123
src/services/glob/list-files.ts
View file

@ -26,15 +26,22 @@ export async function listFiles(dirPath: string, recursive: boolean, limit: numb
// Get ripgrep path
const rgPath = await getRipgrepPath()
// Get files using ripgrep
const files = await listFilesWithRipgrep(rgPath, dirPath, recursive, limit)
// Get directories with proper filtering
// Get directories with proper filtering first to ensure we capture the structure
const gitignorePatterns = await parseGitignoreFile(dirPath, recursive)
const directories = await listFilteredDirectories(dirPath, recursive, gitignorePatterns)
// Get files using ripgrep with a higher limit to avoid early termination
// We'll use a higher internal limit and then apply balanced sampling
const internalLimit = Math.max(limit * 3, 1000) // Use 3x the requested limit or 1000, whichever is higher
const files = await listFilesWithRipgrep(rgPath, dirPath, recursive, internalLimit)
// Apply balanced sampling to ensure fair representation across directories
// Reserve some space for directories in the limit
const filesLimit = Math.max(limit - directories.length, Math.floor(limit * 0.8))
const balancedFiles = applyBalancedSampling(files, directories, filesLimit)
// Combine and format the results
return formatAndCombineResults(files, directories, limit)
return formatAndCombineResults(balancedFiles, directories, limit)
}
/**
@ -304,6 +311,91 @@ function isIgnoredByGitignore(dirName: string, gitignorePatterns: string[]): boo
return false
}
/**
* Apply balanced sampling to ensure fair representation across directories
* This prevents one large directory from dominating the file list
*/
function applyBalancedSampling(files: string[], directories: string[], limit: number): string[] {
if (files.length <= limit) {
return files
}
// Group files by their parent directory
const filesByDirectory = new Map<string, string[]>()
for (const file of files) {
const dir = path.dirname(file)
if (!filesByDirectory.has(dir)) {
filesByDirectory.set(dir, [])
}
filesByDirectory.get(dir)!.push(file)
}
// Improved balanced sampling algorithm
const dirEntries = Array.from(filesByDirectory.entries())
dirEntries.sort(([a], [b]) => a.localeCompare(b))
const result: string[] = []
const dirCount = dirEntries.length
// Ensure each directory gets at least a minimum number of files
const minFilesPerDir = Math.max(3, Math.floor(limit / (dirCount * 4))) // At least 3 files per dir, or 1/4 of fair share
const maxFilesPerDir = Math.floor(limit / dirCount) + 10 // Allow some directories to have more files
// First pass: give each directory its minimum allocation
let remainingLimit = limit
const dirAllocations = new Map<string, number>()
for (const [dir, dirFiles] of dirEntries) {
const allocation = Math.min(minFilesPerDir, dirFiles.length, remainingLimit)
dirAllocations.set(dir, allocation)
remainingLimit -= allocation
}
// Second pass: distribute remaining slots proportionally to directory sizes
if (remainingLimit > 0) {
const totalFiles = dirEntries.reduce((sum, [, dirFiles]) => sum + dirFiles.length, 0)
for (const [dir, dirFiles] of dirEntries) {
const currentAllocation = dirAllocations.get(dir)!
const proportion = dirFiles.length / totalFiles
const additionalSlots = Math.min(
Math.floor(remainingLimit * proportion),
maxFilesPerDir - currentAllocation,
dirFiles.length - currentAllocation,
)
if (additionalSlots > 0) {
dirAllocations.set(dir, currentAllocation + additionalSlots)
remainingLimit -= additionalSlots
}
}
}
// Third pass: distribute any remaining slots to directories that can take them
for (const [dir, dirFiles] of dirEntries) {
if (remainingLimit <= 0) break
const currentAllocation = dirAllocations.get(dir)!
const canTakeMore = Math.min(remainingLimit, dirFiles.length - currentAllocation)
if (canTakeMore > 0) {
dirAllocations.set(dir, currentAllocation + canTakeMore)
remainingLimit -= canTakeMore
}
}
// Collect the files based on allocations
for (const [dir, dirFiles] of dirEntries) {
const allocation = dirAllocations.get(dir)!
dirFiles.sort() // Ensure consistent ordering
const selectedFiles = dirFiles.slice(0, allocation)
result.push(...selectedFiles)
}
return result
}
/**
* Combine file and directory results and format them properly
*/
@ -338,23 +430,20 @@ async function execRipgrep(rgPath: string, args: string[], limit: number): Promi
let output = ""
let results: string[] = []
// Set timeout to avoid hanging
// Set timeout to avoid hanging - increased to allow more complete traversal
const timeoutId = setTimeout(() => {
rgProcess.kill()
console.warn("ripgrep timed out, returning partial results")
resolve(results.slice(0, limit))
}, 10_000)
resolve(results) // Don't slice here either
}, 15_000)
// Process stdout data as it comes in
rgProcess.stdout.on("data", (data) => {
output += data.toString()
processRipgrepOutput()
// Kill the process if we've reached the limit
if (results.length >= limit) {
rgProcess.kill()
clearTimeout(timeoutId) // Clear the timeout when we kill the process due to reaching the limit
}
// Don't kill the process early - let it complete to get full directory structure
// The balanced sampling will be applied later in applyBalancedSampling
})
// Process stderr but don't fail on non-zero exit codes
@ -375,7 +464,7 @@ async function execRipgrep(rgPath: string, args: string[], limit: number): Promi
console.warn(`ripgrep process exited with code ${code}, returning partial results`)
}
resolve(results.slice(0, limit))
resolve(results) // Don't slice here - let balanced sampling handle the limit
})
// Handle process errors
@ -396,12 +485,10 @@ async function execRipgrep(rgPath: string, args: string[], limit: number): Promi
output = ""
}
// Process each complete line
// Process each complete line - don't limit here, let balanced sampling handle it
for (const line of lines) {
if (line.trim() && results.length < limit) {
if (line.trim()) {
results.push(line)
} else if (results.length >= limit) {
break
}
}
}

1
test-issue-5301/a/file001.txt Normal file
View file

@ -0,0 +1 @@
Content of file 1

1
test-issue-5301/a/file002.txt Normal file
View file

@ -0,0 +1 @@
Content of file 2

1
test-issue-5301/a/file003.txt Normal file
View file

@ -0,0 +1 @@
Content of file 3

1
test-issue-5301/a/file004.txt Normal file
View file

@ -0,0 +1 @@
Content of file 4

1
test-issue-5301/a/file005.txt Normal file
View file

@ -0,0 +1 @@
Content of file 5

1
test-issue-5301/a/file006.txt Normal file
View file

@ -0,0 +1 @@
Content of file 6

1
test-issue-5301/a/file007.txt Normal file
View file

@ -0,0 +1 @@
Content of file 7

1
test-issue-5301/a/file008.txt Normal file
View file

@ -0,0 +1 @@
Content of file 8

1
test-issue-5301/a/file009.txt Normal file
View file

@ -0,0 +1 @@
Content of file 9

1
test-issue-5301/a/file010.txt Normal file
View file

@ -0,0 +1 @@
Content of file 10

1
test-issue-5301/a/file011.txt Normal file
View file

@ -0,0 +1 @@
Content of file 11

1
test-issue-5301/a/file012.txt Normal file
View file

@ -0,0 +1 @@
Content of file 12

1
test-issue-5301/a/file013.txt Normal file
View file

@ -0,0 +1 @@
Content of file 13

1
test-issue-5301/a/file014.txt Normal file
View file

@ -0,0 +1 @@
Content of file 14

1
test-issue-5301/a/file015.txt Normal file
View file

@ -0,0 +1 @@
Content of file 15

1
test-issue-5301/a/file016.txt Normal file
View file

@ -0,0 +1 @@
Content of file 16

1
test-issue-5301/a/file017.txt Normal file
View file

@ -0,0 +1 @@
Content of file 17

1
test-issue-5301/a/file018.txt Normal file
View file

@ -0,0 +1 @@
Content of file 18

1
test-issue-5301/a/file019.txt Normal file
View file

@ -0,0 +1 @@
Content of file 19

1
test-issue-5301/a/file020.txt Normal file
View file

@ -0,0 +1 @@
Content of file 20

1
test-issue-5301/a/file021.txt Normal file
View file

@ -0,0 +1 @@
Content of file 21

1
test-issue-5301/a/file022.txt Normal file
View file

@ -0,0 +1 @@
Content of file 22

1
test-issue-5301/a/file023.txt Normal file
View file

@ -0,0 +1 @@
Content of file 23

1
test-issue-5301/a/file024.txt Normal file
View file

@ -0,0 +1 @@
Content of file 24

1
test-issue-5301/a/file025.txt Normal file
View file

@ -0,0 +1 @@
Content of file 25

1
test-issue-5301/a/file026.txt Normal file
View file

@ -0,0 +1 @@
Content of file 26

1
test-issue-5301/a/file027.txt Normal file
View file

@ -0,0 +1 @@
Content of file 27

1
test-issue-5301/a/file028.txt Normal file
View file

@ -0,0 +1 @@
Content of file 28

1
test-issue-5301/a/file029.txt Normal file
View file

@ -0,0 +1 @@
Content of file 29

1
test-issue-5301/a/file030.txt Normal file
View file

@ -0,0 +1 @@
Content of file 30

1
test-issue-5301/a/file031.txt Normal file
View file

@ -0,0 +1 @@
Content of file 31

1
test-issue-5301/a/file032.txt Normal file
View file

@ -0,0 +1 @@
Content of file 32

1
test-issue-5301/a/file033.txt Normal file
View file

@ -0,0 +1 @@
Content of file 33

1
test-issue-5301/a/file034.txt Normal file
View file

@ -0,0 +1 @@
Content of file 34

1
test-issue-5301/a/file035.txt Normal file
View file

@ -0,0 +1 @@
Content of file 35

1
test-issue-5301/a/file036.txt Normal file
View file

@ -0,0 +1 @@
Content of file 36

1
test-issue-5301/a/file037.txt Normal file
View file

@ -0,0 +1 @@
Content of file 37

1
test-issue-5301/a/file038.txt Normal file
View file

@ -0,0 +1 @@
Content of file 38

1
test-issue-5301/a/file039.txt Normal file
View file

@ -0,0 +1 @@
Content of file 39

1
test-issue-5301/a/file040.txt Normal file
View file

@ -0,0 +1 @@
Content of file 40

1
test-issue-5301/a/file041.txt Normal file
View file

@ -0,0 +1 @@
Content of file 41

1
test-issue-5301/a/file042.txt Normal file
View file

@ -0,0 +1 @@
Content of file 42

1
test-issue-5301/a/file043.txt Normal file
View file

@ -0,0 +1 @@
Content of file 43

1
test-issue-5301/a/file044.txt Normal file
View file

@ -0,0 +1 @@
Content of file 44

1
test-issue-5301/a/file045.txt Normal file
View file

@ -0,0 +1 @@
Content of file 45

1
test-issue-5301/a/file046.txt Normal file
View file

@ -0,0 +1 @@
Content of file 46

1
test-issue-5301/a/file047.txt Normal file
View file

@ -0,0 +1 @@
Content of file 47

1
test-issue-5301/a/file048.txt Normal file
View file

@ -0,0 +1 @@
Content of file 48

1
test-issue-5301/a/file049.txt Normal file
View file

@ -0,0 +1 @@
Content of file 49

1
test-issue-5301/a/file050.txt Normal file
View file

@ -0,0 +1 @@
Content of file 50

1
test-issue-5301/a/file051.txt Normal file
View file

@ -0,0 +1 @@
Content of file 51

1
test-issue-5301/a/file052.txt Normal file
View file

@ -0,0 +1 @@
Content of file 52

1
test-issue-5301/a/file053.txt Normal file
View file

@ -0,0 +1 @@
Content of file 53

1
test-issue-5301/a/file054.txt Normal file
View file

@ -0,0 +1 @@
Content of file 54

1
test-issue-5301/a/file055.txt Normal file
View file

@ -0,0 +1 @@
Content of file 55

1
test-issue-5301/a/file056.txt Normal file
View file

@ -0,0 +1 @@
Content of file 56

1
test-issue-5301/a/file057.txt Normal file
View file

@ -0,0 +1 @@
Content of file 57

1
test-issue-5301/a/file058.txt Normal file
View file

@ -0,0 +1 @@
Content of file 58

1
test-issue-5301/a/file059.txt Normal file
View file

@ -0,0 +1 @@
Content of file 59

1
test-issue-5301/a/file060.txt Normal file
View file

@ -0,0 +1 @@
Content of file 60

1
test-issue-5301/a/file061.txt Normal file
View file

@ -0,0 +1 @@
Content of file 61

1
test-issue-5301/a/file062.txt Normal file
View file

@ -0,0 +1 @@
Content of file 62

1
test-issue-5301/a/file063.txt Normal file
View file

@ -0,0 +1 @@
Content of file 63

1
test-issue-5301/a/file064.txt Normal file
View file

@ -0,0 +1 @@
Content of file 64

1
test-issue-5301/a/file065.txt Normal file
View file

@ -0,0 +1 @@
Content of file 65

1
test-issue-5301/a/file066.txt Normal file
View file

@ -0,0 +1 @@
Content of file 66

1
test-issue-5301/a/file067.txt Normal file
View file

@ -0,0 +1 @@
Content of file 67

1
test-issue-5301/a/file068.txt Normal file
View file

@ -0,0 +1 @@
Content of file 68

1
test-issue-5301/a/file069.txt Normal file
View file

@ -0,0 +1 @@
Content of file 69

1
test-issue-5301/a/file070.txt Normal file
View file

@ -0,0 +1 @@
Content of file 70

1
test-issue-5301/a/file071.txt Normal file
View file

@ -0,0 +1 @@
Content of file 71

1
test-issue-5301/a/file072.txt Normal file
View file

@ -0,0 +1 @@
Content of file 72

1
test-issue-5301/a/file073.txt Normal file
View file

@ -0,0 +1 @@
Content of file 73

1
test-issue-5301/a/file074.txt Normal file
View file

@ -0,0 +1 @@
Content of file 74

1
test-issue-5301/a/file075.txt Normal file
View file

@ -0,0 +1 @@
Content of file 75

1
test-issue-5301/a/file076.txt Normal file
View file

@ -0,0 +1 @@
Content of file 76

1
test-issue-5301/a/file077.txt Normal file
View file

@ -0,0 +1 @@
Content of file 77

1
test-issue-5301/a/file078.txt Normal file
View file

@ -0,0 +1 @@
Content of file 78

1
test-issue-5301/a/file079.txt Normal file
View file

@ -0,0 +1 @@
Content of file 79

1
test-issue-5301/a/file080.txt Normal file
View file

@ -0,0 +1 @@
Content of file 80

1
test-issue-5301/a/file081.txt Normal file
View file

@ -0,0 +1 @@
Content of file 81

1
test-issue-5301/a/file082.txt Normal file
View file

@ -0,0 +1 @@
Content of file 82

1
test-issue-5301/a/file083.txt Normal file
View file

@ -0,0 +1 @@
Content of file 83

1
test-issue-5301/a/file084.txt Normal file
View file

@ -0,0 +1 @@
Content of file 84

1
test-issue-5301/a/file085.txt Normal file
View file

@ -0,0 +1 @@
Content of file 85

1
test-issue-5301/a/file086.txt Normal file
View file

@ -0,0 +1 @@
Content of file 86

1
test-issue-5301/a/file087.txt Normal file
View file

@ -0,0 +1 @@
Content of file 87

1
test-issue-5301/a/file088.txt Normal file
View file

@ -0,0 +1 @@
Content of file 88

1
test-issue-5301/a/file089.txt Normal file
View file

@ -0,0 +1 @@
Content of file 89

1
test-issue-5301/a/file090.txt Normal file
View file

@ -0,0 +1 @@
Content of file 90

1
test-issue-5301/a/file091.txt Normal file
View file

@ -0,0 +1 @@
Content of file 91

1
test-issue-5301/a/file092.txt Normal file
View file

@ -0,0 +1 @@
Content of file 92

1
test-issue-5301/a/file093.txt Normal file
View file

@ -0,0 +1 @@
Content of file 93

1
test-issue-5301/a/file094.txt Normal file
View file

@ -0,0 +1 @@
Content of file 94

1
test-issue-5301/a/file095.txt Normal file
View file

@ -0,0 +1 @@
Content of file 95

1
test-issue-5301/a/file096.txt Normal file
View file

@ -0,0 +1 @@
Content of file 96

1
test-issue-5301/a/file097.txt Normal file
View file

@ -0,0 +1 @@
Content of file 97

1
test-issue-5301/a/file098.txt Normal file
View file

@ -0,0 +1 @@
Content of file 98

1
test-issue-5301/a/file099.txt Normal file
View file

@ -0,0 +1 @@
Content of file 99

Some files were not shown because too many files have changed in this diff Show more