mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-08-28 05:27:24 +00:00
Added more exlcusions to checkpoints
Added CheckpointExclusions.ts to separate file filtering for checkpoints. This excludes default a default list of extensions, build artifacts, binary files, large files (>10MB), and various development-specific files from being tracked in checkpoints. Includes a caching system to improve filtering performance and handles OS-specific binary detection. TODO - Make exclusions user-configurable.
This commit is contained in:
canvrno
parent
397e8bda1f
commit
353df9afd8
2 changed files with 369 additions and 119 deletions
363
src/integrations/checkpoints/CheckpointExclusions.ts
Normal file
363
src/integrations/checkpoints/CheckpointExclusions.ts
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
import fs from "fs/promises"
|
||||
import * as path from "path"
|
||||
import { fileExistsAtPath } from "../../utils/fs"
|
||||
import { execa } from "execa"
|
||||
|
||||
const GIT_DISABLED_SUFFIX = "_disabled"
|
||||
|
||||
// Type definition for the file filtering cache system
|
||||
// Tracks directory/extension patterns and binary file results for performance optimization
|
||||
interface FileFilterCache {
|
||||
directoryPatterns: Set<string>
|
||||
extensionPatterns: Set<string>
|
||||
binaryResults: Map<string, boolean>
|
||||
}
|
||||
|
||||
// Singleton cache instance for application-wide file filtering
|
||||
// Used to avoid redundant pattern matching and binary checks
|
||||
const filterCache: FileFilterCache = {
|
||||
directoryPatterns: new Set(),
|
||||
extensionPatterns: new Set(),
|
||||
binaryResults: new Map(),
|
||||
}
|
||||
|
||||
// Updates cache with new pattern sets and clears stale entries
|
||||
// Processes directory patterns (ending with '/') and extension patterns (starting with '*.')
|
||||
function initializeCache(patterns: string[]): void {
|
||||
filterCache.directoryPatterns.clear()
|
||||
filterCache.extensionPatterns.clear()
|
||||
|
||||
patterns.forEach((pattern) => {
|
||||
if (pattern.endsWith("/")) {
|
||||
filterCache.directoryPatterns.add(pattern.slice(0, -1))
|
||||
} else if (pattern.startsWith("*.")) {
|
||||
filterCache.extensionPatterns.add(pattern.slice(1))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Helper function to check if path matches directory exclusions
|
||||
function isExcludedDirectory(filePath: string): boolean {
|
||||
const normalizedPath = filePath.replace(/\\/g, "/")
|
||||
return Array.from(filterCache.directoryPatterns).some(
|
||||
(dir) => normalizedPath.includes(`/${dir}/`) || normalizedPath.endsWith(`/${dir}`),
|
||||
)
|
||||
}
|
||||
|
||||
// Helper function to check if path matches extension exclusions
|
||||
function isExcludedExtension(filePath: string): boolean {
|
||||
const ext = path.extname(filePath)
|
||||
return filterCache.extensionPatterns.has(ext)
|
||||
}
|
||||
|
||||
// Helper function to check if file exceeds size limit (10MB)
|
||||
async function isOverSizeLimit(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
const stats = await fs.stat(filePath)
|
||||
return stats.size > 10 * 1024 * 1024 // 10MB limit
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// TODO Make this configurable by the user
|
||||
export const getDefaultExclusions = (lfsPatterns: string[] = []): string[] => [
|
||||
".git/", // ignore the user's .git
|
||||
`.git${GIT_DISABLED_SUFFIX}/`, // ignore the disabled nested git repos
|
||||
//Build and Development Artifacts
|
||||
"*.log",
|
||||
".DS_Store",
|
||||
".gradle/",
|
||||
".idea/",
|
||||
".parcel-cache/",
|
||||
".pytest_cache/",
|
||||
".next/",
|
||||
".nuxt/",
|
||||
".sass-cache/",
|
||||
".vs/",
|
||||
".vscode/",
|
||||
"Pods/",
|
||||
"__pycache__/",
|
||||
"bin/",
|
||||
"build/",
|
||||
"build/dependencies/",
|
||||
"bundle/",
|
||||
"coverage/",
|
||||
"deps/",
|
||||
"dist/",
|
||||
"env/",
|
||||
"node_modules/",
|
||||
"obj/",
|
||||
"out/",
|
||||
"pkg/",
|
||||
"pycache/",
|
||||
"target/dependency/",
|
||||
"temp/",
|
||||
"tmp/",
|
||||
"vendor/",
|
||||
"venv/",
|
||||
|
||||
// Image files
|
||||
"*.jpg",
|
||||
"*.jpeg",
|
||||
"*.png",
|
||||
"*.gif",
|
||||
"*.bmp",
|
||||
"*.ico",
|
||||
"*.webp",
|
||||
"*.tiff",
|
||||
"*.tif",
|
||||
"*.svg",
|
||||
"*.raw",
|
||||
"*.heic",
|
||||
"*.avif",
|
||||
"*.eps",
|
||||
"*.psd",
|
||||
// ".ai", // Adobe Illustrator, commented out as some users may use this extension in AI projects
|
||||
// "*.svg", // SVG files were commented out in the original exclusion implementation
|
||||
|
||||
// Audio & Video files
|
||||
".3gp",
|
||||
".aac",
|
||||
".aiff",
|
||||
".asf",
|
||||
".avi",
|
||||
".divx",
|
||||
".flac",
|
||||
".m4a",
|
||||
".m4v",
|
||||
".mkv",
|
||||
".mov",
|
||||
".mp3",
|
||||
".mp4",
|
||||
".mpeg",
|
||||
".mpg",
|
||||
".ogg",
|
||||
".opus",
|
||||
".rm",
|
||||
".rmvb",
|
||||
".ts",
|
||||
".vob",
|
||||
".wav",
|
||||
".webm",
|
||||
".webp",
|
||||
".wma",
|
||||
".wmv",
|
||||
|
||||
// Cache and temporary files
|
||||
".DS_Store",
|
||||
".bak",
|
||||
".cache",
|
||||
".crdownload",
|
||||
".dmp",
|
||||
".dump",
|
||||
".eslintcache",
|
||||
".lock",
|
||||
".log",
|
||||
".old",
|
||||
".part",
|
||||
".partial",
|
||||
".pyc",
|
||||
".pyo",
|
||||
".stackdump",
|
||||
".swo",
|
||||
".swp",
|
||||
".temp",
|
||||
".tmp",
|
||||
"Thumbs.db",
|
||||
|
||||
// Environment and config files
|
||||
".env*",
|
||||
"*.local",
|
||||
"*.development",
|
||||
"*.production",
|
||||
|
||||
// Large data files
|
||||
"*.zip",
|
||||
"*.tar",
|
||||
"*.gz",
|
||||
"*.rar",
|
||||
"*.7z",
|
||||
"*.iso",
|
||||
"*.bin",
|
||||
"*.exe",
|
||||
"*.dll",
|
||||
"*.so",
|
||||
"*.dylib",
|
||||
"*.dat",
|
||||
"*.dmg",
|
||||
"*.msi",
|
||||
|
||||
// Database files
|
||||
"*.arrow",
|
||||
"*.accdb",
|
||||
".aof",
|
||||
"*.avro",
|
||||
".bak",
|
||||
"*.bson",
|
||||
".csv",
|
||||
".db",
|
||||
".dbf",
|
||||
".dmp",
|
||||
"*.frm",
|
||||
"*.ibd",
|
||||
".mdb",
|
||||
"*.myd",
|
||||
"*.myi",
|
||||
".orc",
|
||||
".parquet",
|
||||
".pdb",
|
||||
".rdb",
|
||||
".sql",
|
||||
".sqlite",
|
||||
|
||||
// Geospatial datasets
|
||||
".shp",
|
||||
".shx",
|
||||
".dbf",
|
||||
".prj",
|
||||
".sbn",
|
||||
".sbx",
|
||||
".shp.xml",
|
||||
".cpg",
|
||||
".gdb",
|
||||
".mdb",
|
||||
".gpkg",
|
||||
".kml",
|
||||
".kmz",
|
||||
".gml",
|
||||
".geojson",
|
||||
".dem",
|
||||
".asc",
|
||||
".img",
|
||||
".ecw",
|
||||
".las",
|
||||
".laz",
|
||||
".mxd",
|
||||
".qgs",
|
||||
".grd",
|
||||
".csv",
|
||||
".dwg",
|
||||
".dxf",
|
||||
|
||||
// Log files
|
||||
"*.error",
|
||||
"*.log",
|
||||
"*.logs",
|
||||
"npm-debug.log*",
|
||||
"*.out",
|
||||
"*.stdout",
|
||||
"yarn-debug.log*",
|
||||
"yarn-error.log*",
|
||||
...lfsPatterns,
|
||||
]
|
||||
|
||||
export const writeExcludesFile = async (gitPath: string, lfsPatterns: string[] = []): Promise<void> => {
|
||||
const excludesPath = path.join(gitPath, "info", "exclude")
|
||||
await fs.mkdir(path.join(gitPath, "info"), { recursive: true })
|
||||
const patterns = getDefaultExclusions(lfsPatterns)
|
||||
await fs.writeFile(excludesPath, patterns.join("\n"))
|
||||
|
||||
// Reinitialize cache with new patterns
|
||||
initializeCache(patterns)
|
||||
// Clear binary results cache as patterns have changed
|
||||
filterCache.binaryResults.clear()
|
||||
}
|
||||
// Get LFS patterns from workspace if they exist
|
||||
export const getLfsPatterns = async (workspacePath: string): Promise<string[]> => {
|
||||
try {
|
||||
const attributesPath = path.join(workspacePath, ".gitattributes")
|
||||
if (await fileExistsAtPath(attributesPath)) {
|
||||
const attributesContent = await fs.readFile(attributesPath, "utf8")
|
||||
return attributesContent
|
||||
.split("\n")
|
||||
.filter((line) => line.includes("filter=lfs"))
|
||||
.map((line) => line.split(" ")[0].trim())
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("Failed to read .gitattributes:", error)
|
||||
}
|
||||
return []
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a file is binary based on the operating system.
|
||||
* Uses different approaches for Windows vs Unix-like systems.
|
||||
* Implements caching and optimized buffer reading.
|
||||
* @param filePath - Path to the file to check
|
||||
* @returns Promise<boolean> - True if the file is binary, false otherwise
|
||||
*/
|
||||
export const isBinaryFile = async (filePath: string): Promise<boolean> => {
|
||||
// Windows-specific implementation
|
||||
if (process.platform === "win32") {
|
||||
const cachedResult = filterCache.binaryResults.get(filePath)
|
||||
if (cachedResult !== undefined) {
|
||||
return cachedResult
|
||||
}
|
||||
|
||||
let fileHandle: fs.FileHandle | null = null
|
||||
try {
|
||||
fileHandle = await fs.open(filePath, "r")
|
||||
const buffer = new Uint8Array(512) // May need to adjust buffer size if this is too slow
|
||||
const { bytesRead } = await fileHandle.read(buffer, 0, buffer.length, 0)
|
||||
|
||||
// Using includes() is faster than some() for small arrays
|
||||
const isBinary = buffer.subarray(0, bytesRead).includes(0)
|
||||
filterCache.binaryResults.set(filePath, isBinary)
|
||||
return isBinary
|
||||
} catch (error) {
|
||||
console.warn("Failed to check if file is binary (win32):", error)
|
||||
return false
|
||||
} finally {
|
||||
if (fileHandle) {
|
||||
try {
|
||||
await fileHandle.close()
|
||||
} catch (err) {
|
||||
console.warn("Error closing file handle:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Unix-like systems implementation using 'file' command
|
||||
try {
|
||||
const { stdout } = await execa(`file --mime-type "${filePath}"`)
|
||||
const isBinary = stdout.toLowerCase().includes("binary")
|
||||
filterCache.binaryResults.set(filePath, isBinary)
|
||||
return isBinary
|
||||
} catch (error) {
|
||||
console.warn("Failed to check if file is binary using 'file' command:", error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main function to determine if a file should be excluded based on
|
||||
* multiple criteria, ordered from fastest to most expensive checks.
|
||||
* @param filePath - Path to the file to check
|
||||
* @returns Promise<boolean> - True if the file should be excluded
|
||||
*/
|
||||
export const shouldExcludeFile = async (filePath: string): Promise<boolean> => {
|
||||
try {
|
||||
// 1. Check directory exclusions (fastest)
|
||||
if (isExcludedDirectory(filePath)) {
|
||||
return true
|
||||
}
|
||||
|
||||
// 2. Check extension exclusions
|
||||
if (isExcludedExtension(filePath)) {
|
||||
return true
|
||||
}
|
||||
|
||||
// 3 & 4. Check size and binary in parallel (most expensive operations)
|
||||
const [sizeResult, binaryResult] = await Promise.all([isOverSizeLimit(filePath), isBinaryFile(filePath)])
|
||||
|
||||
return sizeResult || binaryResult
|
||||
} catch (error) {
|
||||
console.warn("Error in shouldExcludeFile:", error)
|
||||
return false // Default to not excluding on error
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize cache when module loads
|
||||
initializeCache(getDefaultExclusions())
|
||||
|
|
@ -6,6 +6,7 @@ import * as vscode from "vscode"
|
|||
import { ClineProvider } from "../../core/webview/ClineProvider"
|
||||
import { fileExistsAtPath } from "../../utils/fs"
|
||||
import { globby } from "globby"
|
||||
import { getLfsPatterns, writeExcludesFile } from "./CheckpointExclusions"
|
||||
|
||||
class CheckpointTracker {
|
||||
private providerRef: WeakRef<ClineProvider>
|
||||
|
|
@ -21,18 +22,12 @@ class CheckpointTracker {
|
|||
this.cwd = cwd
|
||||
}
|
||||
|
||||
public static async create(taskId: string, provider?: ClineProvider): Promise<CheckpointTracker | undefined> {
|
||||
public static async create(taskId: string, provider?: ClineProvider): Promise<CheckpointTracker> {
|
||||
try {
|
||||
if (!provider) {
|
||||
throw new Error("Provider is required to create a checkpoint tracker")
|
||||
}
|
||||
|
||||
// Check if checkpoints are disabled in VS Code settings
|
||||
const enableCheckpoints = vscode.workspace.getConfiguration("cline").get<boolean>("enableCheckpoints") ?? true
|
||||
if (!enableCheckpoints) {
|
||||
return undefined // Don't create tracker when disabled
|
||||
}
|
||||
|
||||
// Check if git is installed by attempting to get version
|
||||
try {
|
||||
await simpleGit().version()
|
||||
|
|
@ -114,117 +109,9 @@ class CheckpointTracker {
|
|||
// Disable commit signing for shadow repo
|
||||
await git.addConfig("commit.gpgSign", "false")
|
||||
|
||||
// Get LFS patterns from workspace if they exist
|
||||
let lfsPatterns: string[] = []
|
||||
try {
|
||||
const attributesPath = path.join(this.cwd, ".gitattributes")
|
||||
if (await fileExistsAtPath(attributesPath)) {
|
||||
const attributesContent = await fs.readFile(attributesPath, "utf8")
|
||||
lfsPatterns = attributesContent
|
||||
.split("\n")
|
||||
.filter((line) => line.includes("filter=lfs"))
|
||||
.map((line) => line.split(" ")[0].trim())
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("Failed to read .gitattributes:", error)
|
||||
}
|
||||
|
||||
// Add basic excludes directly in git config, while respecting any .gitignore in the workspace
|
||||
// .git/info/exclude is local to the shadow git repo, so it's not shared with the main repo - and won't conflict with user's .gitignore
|
||||
// TODO: let user customize these
|
||||
const excludesPath = path.join(gitPath, "info", "exclude")
|
||||
await fs.mkdir(path.join(gitPath, "info"), { recursive: true })
|
||||
await fs.writeFile(
|
||||
excludesPath,
|
||||
[
|
||||
".git/", // ignore the user's .git
|
||||
`.git${GIT_DISABLED_SUFFIX}/`, // ignore the disabled nested git repos
|
||||
".DS_Store",
|
||||
"*.log",
|
||||
"node_modules/",
|
||||
"__pycache__/",
|
||||
"env/",
|
||||
"venv/",
|
||||
"target/dependency/",
|
||||
"build/dependencies/",
|
||||
"dist/",
|
||||
"out/",
|
||||
"bundle/",
|
||||
"vendor/",
|
||||
"tmp/",
|
||||
"temp/",
|
||||
"deps/",
|
||||
"pkg/",
|
||||
"Pods/",
|
||||
// Media files
|
||||
"*.jpg",
|
||||
"*.jpeg",
|
||||
"*.png",
|
||||
"*.gif",
|
||||
"*.bmp",
|
||||
"*.ico",
|
||||
// "*.svg",
|
||||
"*.mp3",
|
||||
"*.mp4",
|
||||
"*.wav",
|
||||
"*.avi",
|
||||
"*.mov",
|
||||
"*.wmv",
|
||||
"*.webm",
|
||||
"*.webp",
|
||||
"*.m4a",
|
||||
"*.flac",
|
||||
// Build and dependency directories
|
||||
"build/",
|
||||
"bin/",
|
||||
"obj/",
|
||||
".gradle/",
|
||||
".idea/",
|
||||
".vscode/",
|
||||
".vs/",
|
||||
"coverage/",
|
||||
".next/",
|
||||
".nuxt/",
|
||||
// Cache and temporary files
|
||||
"*.cache",
|
||||
"*.tmp",
|
||||
"*.temp",
|
||||
"*.swp",
|
||||
"*.swo",
|
||||
"*.pyc",
|
||||
"*.pyo",
|
||||
".pytest_cache/",
|
||||
".eslintcache",
|
||||
// Environment and config files
|
||||
".env*",
|
||||
"*.local",
|
||||
"*.development",
|
||||
"*.production",
|
||||
// Large data files
|
||||
"*.zip",
|
||||
"*.tar",
|
||||
"*.gz",
|
||||
"*.rar",
|
||||
"*.7z",
|
||||
"*.iso",
|
||||
"*.bin",
|
||||
"*.exe",
|
||||
"*.dll",
|
||||
"*.so",
|
||||
"*.dylib",
|
||||
// Database files
|
||||
"*.sqlite",
|
||||
"*.db",
|
||||
"*.sql",
|
||||
// Log files
|
||||
"*.logs",
|
||||
"*.error",
|
||||
"npm-debug.log*",
|
||||
"yarn-debug.log*",
|
||||
"yarn-error.log*",
|
||||
...lfsPatterns,
|
||||
].join("\n"),
|
||||
)
|
||||
// Get LFS patterns and write excludes file
|
||||
const lfsPatterns = await getLfsPatterns(this.cwd)
|
||||
await writeExcludesFile(gitPath, lfsPatterns)
|
||||
|
||||
// Set up git identity (git throws an error if user.name or user.email is not set)
|
||||
await git.addConfig("user.name", "Cline Checkpoint")
|
||||
|
|
@ -415,6 +302,6 @@ class CheckpointTracker {
|
|||
}
|
||||
}
|
||||
|
||||
const GIT_DISABLED_SUFFIX = "_disabled"
|
||||
export const GIT_DISABLED_SUFFIX = "_disabled"
|
||||
|
||||
export default CheckpointTracker
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue