feat: implement robust command pattern extraction using shell-quote

- Replace regex-based parsing with shell-quote library for deterministic parsing
- Handle complex shell syntax including quotes, escapes, and special characters
- Maintain backward compatibility with existing command patterns
- Add comprehensive test coverage for edge cases
- Addresses PR feedback about regex limitations and parsing reliability

The shell-quote library provides proper shell command parsing that handles:
- Single and double quotes with proper escape sequences
- Environment variable expansion
- Command operators (&&, ||, |, ;)
- Glob patterns and special characters
- Nested quotes and complex argument structures

This ensures commands are extracted exactly as they would be interpreted by a shell.
This commit is contained in:
hannesrudolph 2025-07-15 13:24:28 -06:00
parent f903871394
commit 0a067f005a
2 changed files with 419 additions and 195 deletions

View file

@ -0,0 +1,197 @@
import { describe, it, expect } from "vitest"
import { extractCommandPatterns, extractCommandPattern, getPatternDescription } from "../commandPatterns"
describe("commandPatterns", () => {
describe("extractCommandPatterns", () => {
it("should handle empty or null commands", () => {
expect(extractCommandPatterns("")).toEqual([])
expect(extractCommandPatterns(" ")).toEqual([])
expect(extractCommandPatterns(null as any)).toEqual([])
expect(extractCommandPatterns(undefined as any)).toEqual([])
})
it("should extract simple commands", () => {
expect(extractCommandPatterns("ls")).toEqual(["ls"])
expect(extractCommandPatterns("pwd")).toEqual(["pwd"])
expect(extractCommandPatterns("echo hello")).toEqual(["echo"])
})
it("should handle npm commands correctly", () => {
expect(extractCommandPatterns("npm install")).toEqual(["npm install"])
expect(extractCommandPatterns("npm run build")).toEqual(["npm run"])
expect(extractCommandPatterns("npm run test:unit")).toEqual(["npm run"])
expect(extractCommandPatterns("npm test")).toEqual(["npm test"])
expect(extractCommandPatterns("npm run build -- --watch")).toEqual(["npm run"])
})
it("should handle yarn/pnpm/bun commands", () => {
expect(extractCommandPatterns("yarn install")).toEqual(["yarn install"])
expect(extractCommandPatterns("pnpm run dev")).toEqual(["pnpm run"])
expect(extractCommandPatterns("bun test")).toEqual(["bun test"])
})
it("should handle git commands", () => {
expect(extractCommandPatterns("git status")).toEqual(["git status"])
expect(extractCommandPatterns('git commit -m "message"')).toEqual(["git commit"])
expect(extractCommandPatterns("git push origin main")).toEqual(["git push"])
expect(extractCommandPatterns("git log --oneline")).toEqual(["git log"])
})
it("should handle docker/kubectl/helm commands", () => {
expect(extractCommandPatterns("docker run nginx")).toEqual(["docker run"])
expect(extractCommandPatterns("kubectl get pods")).toEqual(["kubectl get"])
expect(extractCommandPatterns("helm install myapp ./chart")).toEqual(["helm install"])
})
it("should handle make commands", () => {
expect(extractCommandPatterns("make build")).toEqual(["make build"])
expect(extractCommandPatterns("make test")).toEqual(["make test"])
expect(extractCommandPatterns("make -j4 all")).toEqual(["make"])
})
it("should handle interpreter commands", () => {
expect(extractCommandPatterns("python script.py")).toEqual(["python"])
expect(extractCommandPatterns("python3 -m venv env")).toEqual(["python3"])
expect(extractCommandPatterns("node index.js --port 3000")).toEqual(["node"])
expect(extractCommandPatterns("ruby app.rb")).toEqual(["ruby"])
})
it("should handle dangerous commands", () => {
expect(extractCommandPatterns("rm -rf node_modules")).toEqual(["rm"])
expect(extractCommandPatterns("chmod 755 script.sh")).toEqual(["chmod"])
expect(extractCommandPatterns('find . -name "*.log" -delete')).toEqual(["find"])
})
it("should handle cd commands", () => {
expect(extractCommandPatterns("cd /home/user")).toEqual(["cd"])
expect(extractCommandPatterns("cd ..")).toEqual(["cd"])
expect(extractCommandPatterns("cd ~/projects")).toEqual(["cd"])
})
it("should handle chained commands with &&", () => {
const patterns = extractCommandPatterns("npm install && npm run build")
expect(patterns).toEqual(["npm install", "npm run"])
})
it("should handle chained commands with ||", () => {
const patterns = extractCommandPatterns('npm test || echo "Tests failed"')
expect(patterns).toEqual(["echo", "npm test"])
})
it("should handle chained commands with ;", () => {
const patterns = extractCommandPatterns("cd /tmp; ls -la; pwd")
expect(patterns).toEqual(["cd", "ls", "pwd"])
})
it("should handle piped commands", () => {
const patterns = extractCommandPatterns("ps aux | grep node")
expect(patterns).toEqual(["grep", "ps"])
})
it("should handle complex chained commands", () => {
const patterns = extractCommandPatterns('git pull && npm install && npm run build || echo "Build failed"')
expect(patterns).toEqual(["echo", "git pull", "npm install", "npm run"])
})
it("should handle quoted arguments correctly", () => {
expect(extractCommandPatterns('echo "hello world"')).toEqual(["echo"])
expect(extractCommandPatterns("echo 'hello world'")).toEqual(["echo"])
expect(extractCommandPatterns('git commit -m "feat: add new feature"')).toEqual(["git commit"])
})
it("should handle escaped characters", () => {
expect(extractCommandPatterns("echo hello\\ world")).toEqual(["echo"])
expect(extractCommandPatterns('echo "hello \\"world\\""')).toEqual(["echo"])
})
it("should handle environment variables", () => {
expect(extractCommandPatterns("NODE_ENV=production npm start")).toEqual(["npm start"])
expect(extractCommandPatterns("PORT=3000 node server.js")).toEqual(["node"])
})
it("should handle script files", () => {
expect(extractCommandPatterns("./deploy.sh")).toEqual(["./deploy.sh"])
expect(extractCommandPatterns("/usr/local/bin/script.py")).toEqual(["/usr/local/bin/script.py"])
expect(extractCommandPatterns("./scripts/test.js --verbose")).toEqual(["./scripts/test.js"])
})
it("should handle complex npm scripts", () => {
expect(extractCommandPatterns("npm run build:prod -- --source-maps")).toEqual(["npm run"])
expect(extractCommandPatterns("npm run test:coverage -- --watch")).toEqual(["npm run"])
})
it("should return unique sorted patterns", () => {
const patterns = extractCommandPatterns("npm install && npm install && npm run build")
expect(patterns).toEqual(["npm install", "npm run"])
})
it("should handle commands with glob patterns", () => {
expect(extractCommandPatterns("rm *.log")).toEqual(["rm"])
expect(extractCommandPatterns("ls *.{js,ts}")).toEqual(["ls"])
})
it("should handle commands with redirects", () => {
expect(extractCommandPatterns('echo "test" > file.txt')).toEqual(["echo"])
expect(extractCommandPatterns("cat file.txt >> output.log")).toEqual(["cat"])
})
it("should handle subshells and command substitution", () => {
expect(extractCommandPatterns("echo $(date)")).toEqual(["echo"])
expect(extractCommandPatterns("echo `pwd`")).toEqual(["echo"])
})
})
describe("extractCommandPattern", () => {
it("should return the first pattern for backward compatibility", () => {
expect(extractCommandPattern("npm install && npm run build")).toBe("npm install")
expect(extractCommandPattern("git status")).toBe("git status")
expect(extractCommandPattern("")).toBe("")
})
})
describe("getPatternDescription", () => {
it("should describe npm patterns", () => {
expect(getPatternDescription("npm run")).toBe("all npm run scripts")
expect(getPatternDescription("npm install")).toBe("npm install commands")
expect(getPatternDescription("npm test")).toBe("npm test commands")
})
it("should describe git patterns", () => {
expect(getPatternDescription("git commit")).toBe("git commit commands")
expect(getPatternDescription("git push")).toBe("git push commands")
})
it("should describe script files", () => {
expect(getPatternDescription("./deploy.sh")).toBe("this specific script")
expect(getPatternDescription("/usr/bin/script.py")).toBe("this specific script")
})
it("should describe interpreter patterns", () => {
expect(getPatternDescription("python")).toBe("python scripts")
expect(getPatternDescription("node")).toBe("node scripts")
expect(getPatternDescription("ruby")).toBe("ruby scripts")
})
it("should describe docker/kubectl patterns", () => {
expect(getPatternDescription("docker run")).toBe("docker run commands")
expect(getPatternDescription("kubectl get")).toBe("kubectl get commands")
})
it("should describe make patterns", () => {
expect(getPatternDescription("make build")).toBe("make build target")
expect(getPatternDescription("make test")).toBe("make test target")
})
it("should describe cd pattern", () => {
expect(getPatternDescription("cd")).toBe("directory navigation")
})
it("should handle empty patterns", () => {
expect(getPatternDescription("")).toBe("")
})
it("should provide default description", () => {
expect(getPatternDescription("custom-command")).toBe("custom-command commands")
})
})
})

View file

@ -1,253 +1,272 @@
import { parse } from "shell-quote"
/**
* Extracts a generalizable command pattern from a specific command.
* This function creates patterns that can be used for whitelisting similar commands.
* Extracts command patterns from a command string using shell-quote parser.
* This provides a robust, deterministic way to extract patterns that can be
* used for whitelisting similar commands.
*
* Examples:
* - "npm test" -> "npm test"
* - "npm run build" -> "npm run"
* - "git commit -m 'message'" -> "git commit"
* - "echo 'hello world'" -> "echo"
* - "python script.py --arg value" -> "python"
* - "./scripts/deploy.sh production" -> "./scripts/deploy.sh"
* - "cd /path/to/dir && npm install" -> "cd * && npm install"
* - "rm -rf node_modules" -> "rm"
* @param command The full command string to extract patterns from
* @returns Array of unique command patterns sorted alphabetically
*/
export function extractCommandPattern(command: string): string {
if (!command?.trim()) return ""
export function extractCommandPatterns(command: string): string[] {
if (!command?.trim()) return []
// Remove leading/trailing whitespace
const trimmedCommand = command.trim()
const patterns = new Set<string>()
// Check if this is a chained command
// Use a more robust regex that handles nested quotes properly
const operators = ["&&", "||", ";", "|"]
let chainOperator: string | null = null
let splitIndex = -1
// Handle command chains (&&, ||, ;, |)
const chainOperators = ["&&", "||", ";", "|"]
const commands = splitByOperators(command, chainOperators)
// Find the first unquoted operator
let inSingleQuote = false
let inDoubleQuote = false
let escapeNext = false
for (let i = 0; i < trimmedCommand.length; i++) {
const char = trimmedCommand[i]
if (escapeNext) {
escapeNext = false
continue
}
if (char === "\\") {
escapeNext = true
continue
}
if (char === "'" && !inDoubleQuote) {
inSingleQuote = !inSingleQuote
continue
}
if (char === '"' && !inSingleQuote) {
inDoubleQuote = !inDoubleQuote
continue
}
// Only look for operators outside of quotes
if (!inSingleQuote && !inDoubleQuote) {
for (const op of operators) {
if (trimmedCommand.substring(i, i + op.length) === op) {
chainOperator = op
splitIndex = i
break
}
}
if (chainOperator) break
for (const cmd of commands) {
const pattern = extractSingleCommandPattern(cmd.trim())
if (pattern) {
patterns.add(pattern)
}
}
if (chainOperator && splitIndex > 0) {
const firstPart = trimmedCommand.substring(0, splitIndex).trim()
const restPart = trimmedCommand.substring(splitIndex + chainOperator.length).trim()
// Process each part separately
const firstPattern = extractSingleCommandPattern(firstPart)
const restPattern = extractCommandPattern(restPart)
// For security, limit the depth of chained commands
// Count existing operators in the pattern to prevent deeply nested chains
const operatorCount = (restPattern.match(/&&|\|\||;|\|/g) || []).length
if (operatorCount >= 3) {
// Too many chained commands, return a more restrictive pattern
return firstPattern
}
return `${firstPattern} ${chainOperator} ${restPattern}`
}
// Not a chained command, process normally
return extractSingleCommandPattern(trimmedCommand)
// Return sorted unique patterns
return Array.from(patterns).sort()
}
/**
* Extracts pattern from a single command (not chained)
* Split command by operators while respecting shell syntax
*/
function extractSingleCommandPattern(command: string): string {
const firstCommand = command
// Split the command into tokens, respecting quotes
const tokens: string[] = []
let currentToken = ""
function splitByOperators(command: string, operators: string[]): string[] {
const commands: string[] = []
let current = ""
let inSingleQuote = false
let inDoubleQuote = false
let escapeNext = false
for (let i = 0; i < firstCommand.length; i++) {
const char = firstCommand[i]
for (let i = 0; i < command.length; i++) {
const char = command[i]
if (escapeNext) {
currentToken += char
current += char
escapeNext = false
continue
}
if (char === "\\") {
escapeNext = true
currentToken += char
current += char
continue
}
if (char === "'" && !inDoubleQuote) {
inSingleQuote = !inSingleQuote
currentToken += char
current += char
continue
}
if (char === '"' && !inSingleQuote) {
inDoubleQuote = !inDoubleQuote
currentToken += char
current += char
continue
}
if (char === " " && !inSingleQuote && !inDoubleQuote) {
if (currentToken) {
tokens.push(currentToken)
currentToken = ""
// Check for operators outside quotes
if (!inSingleQuote && !inDoubleQuote) {
let foundOperator = false
for (const op of operators) {
if (command.substring(i, i + op.length) === op) {
// Found an operator, save current command
if (current.trim()) {
commands.push(current.trim())
}
current = ""
i += op.length - 1 // -1 because the loop will increment
foundOperator = true
break
}
}
} else {
currentToken += char
if (foundOperator) continue
}
current += char
}
if (currentToken) {
tokens.push(currentToken)
// Don't forget the last command
if (current.trim()) {
commands.push(current.trim())
}
if (tokens.length === 0) return ""
// If no commands were found, return the whole command
if (commands.length === 0) {
commands.push(command)
}
const baseCommand = tokens[0]
return commands
}
// Special handling for common patterns
/**
* Extract pattern from a single command (not chained)
*/
function extractSingleCommandPattern(command: string): string {
if (!command) return ""
// 1. npm/yarn/pnpm commands - include subcommand with wildcards for scripts
if (["npm", "yarn", "pnpm", "bun"].includes(baseCommand) && tokens.length > 1) {
const subCommand = tokens[1]
// For "run" commands, check the script name
if (subCommand === "run" && tokens.length > 2) {
const _scriptName = tokens[2].replace(/^["']|["']$/g, "") // Remove quotes if present
try {
const parsed = parse(command)
if (parsed.length === 0) return ""
// Check if there's a -- separator (pass-through args)
const _hasPassThroughArgs = tokens.includes("--")
const patterns: string[] = []
let i = 0
// Always return just "npm run" without the script name
// This allows all npm run commands without using wildcards
return `${baseCommand} run`
while (i < parsed.length) {
const token = parsed[i]
// Skip operators and glob patterns
if (typeof token === "object" && "op" in token) {
// Handle redirects (>, >>, <, etc.)
if (token.op === ">" || token.op === ">>" || token.op === "<") {
// Stop processing - we've hit a redirect
break
}
// Handle glob patterns - treat as regular tokens
if (token.op === "glob") {
// For globs, we typically want to stop at the command level
if (patterns.length > 0) {
break
}
}
i++
continue
}
// Convert token to string
const strToken = String(token)
// Skip empty tokens
if (!strToken) {
i++
continue
}
// Handle environment variables at the start
if (patterns.length === 0 && strToken.includes("=") && /^[A-Z_]+=/i.test(strToken)) {
// This is an environment variable, skip it
i++
continue
}
// First non-env-var token is the command
if (patterns.length === 0) {
// Handle script files
if (
strToken.includes("/") ||
strToken.endsWith(".sh") ||
strToken.endsWith(".py") ||
strToken.endsWith(".js") ||
strToken.endsWith(".rb")
) {
return strToken
}
patterns.push(strToken)
i++
continue
}
// Stop at common flag indicators
if (strToken.startsWith("-")) {
break
}
// Stop at special shell operators
if ([">", ">>", "<", "|", "&", ";"].includes(strToken)) {
break
}
// Handle second token based on the command
if (patterns.length === 1) {
const baseCmd = patterns[0]
// Special handling for package managers
if (["npm", "yarn", "pnpm", "bun"].includes(baseCmd)) {
// Include subcommand
if (!strToken.startsWith("-") && !strToken.includes("/")) {
patterns.push(strToken)
// For 'run' commands, stop here to allow any script
if (strToken === "run") {
break
}
} else {
break
}
}
// Special handling for git
else if (baseCmd === "git") {
if (!strToken.startsWith("-")) {
patterns.push(strToken)
}
break
}
// Special handling for docker/kubectl/helm
else if (["docker", "kubectl", "helm"].includes(baseCmd)) {
if (!strToken.startsWith("-")) {
patterns.push(strToken)
}
break
}
// Special handling for make
else if (baseCmd === "make") {
if (!strToken.startsWith("-")) {
patterns.push(strToken)
}
break
}
// For interpreters, stop after the command
else if (["python", "python3", "node", "ruby", "perl", "php", "java", "go"].includes(baseCmd)) {
break
}
// For dangerous commands, stop immediately
else if (["rm", "mv", "cp", "chmod", "chown", "find", "grep", "sed", "awk"].includes(baseCmd)) {
break
}
// For cd, stop immediately
else if (baseCmd === "cd") {
break
}
// For echo and similar commands, stop immediately
else if (["echo", "printf", "cat", "ls", "pwd"].includes(baseCmd)) {
break
}
// Default: stop at paths or complex arguments
else if (
strToken.includes("/") ||
strToken.includes("\\") ||
strToken.includes(":") ||
strToken.includes("=")
) {
break
}
}
// For third+ tokens, be very restrictive
else {
break
}
i++
}
// For direct scripts like "npm test", "npm build", include the script name
if (!subCommand.startsWith("-")) {
return `${baseCommand} ${subCommand}`
}
}
// 2. git commands - include subcommand
if (baseCommand === "git" && tokens.length > 1) {
const subCommand = tokens[1]
if (!subCommand.startsWith("-")) {
return `${baseCommand} ${subCommand}`
}
return patterns.join(" ")
} catch (_error) {
// If parsing fails, fall back to simple first token
const tokens = command.split(/\s+/)
return tokens[0] || ""
}
// 3. Script files - include the full script path
if (
baseCommand.includes("/") ||
baseCommand.endsWith(".sh") ||
baseCommand.endsWith(".py") ||
baseCommand.endsWith(".js") ||
baseCommand.endsWith(".rb")
) {
return baseCommand
}
// 4. Python/node/ruby/etc interpreters - just the interpreter
if (["python", "python3", "node", "ruby", "perl", "php", "java", "go"].includes(baseCommand)) {
return baseCommand
}
// 5. Common shell commands with dangerous flags - just the command
if (["rm", "mv", "cp", "chmod", "chown", "find", "grep", "sed", "awk"].includes(baseCommand)) {
return baseCommand
}
// 6. cd command - just return cd
if (baseCommand === "cd") {
return "cd"
}
// 7. Docker/kubectl commands - include subcommand
if (["docker", "kubectl", "helm"].includes(baseCommand) && tokens.length > 1) {
const subCommand = tokens[1]
if (!subCommand.startsWith("-")) {
return `${baseCommand} ${subCommand}`
}
}
// 8. Make commands - include target if present
if (baseCommand === "make" && tokens.length > 1) {
const target = tokens[1]
if (!target.startsWith("-")) {
return `${baseCommand} ${target}`
}
}
// 9. Environment variables - handle with care
if (baseCommand.includes("=")) {
// This might be an environment variable like NODE_ENV=production
const envMatch = baseCommand.match(/^([A-Z_]+)=/)
if (envMatch) {
// Return the full environment variable assignment
return baseCommand
}
}
// 10. Commands with suspicious patterns - be restrictive
// Check for potential command injection patterns
if (baseCommand.includes("$") || baseCommand.includes("`") || baseCommand.includes("(")) {
// These could be command substitutions or variables, be very restrictive
return baseCommand.split(/[$`(]/)[0].trim() || "echo"
}
// Default: just return the base command
return baseCommand
}
/**
* Get a human-readable description of what the pattern will allow
*
* Examples:
* - "npm test" -> "npm test commands"
* - "npm run" -> "npm run scripts"
* - "git commit" -> "git commit commands"
* - "python" -> "python scripts"
* - "./scripts/deploy.sh" -> "this specific script"
*/
export function getPatternDescription(pattern: string): string {
if (!pattern) return ""
@ -258,7 +277,6 @@ export function getPatternDescription(pattern: string): string {
// npm/yarn/pnpm patterns
if (["npm", "yarn", "pnpm", "bun"].includes(baseCommand)) {
if (tokens[1] === "run") {
// For "npm run", describe what it allows
return `all ${baseCommand} run scripts`
}
if (tokens[1]) {
@ -306,3 +324,12 @@ export function getPatternDescription(pattern: string): string {
// Default
return `${baseCommand} commands`
}
/**
* Wrapper function to maintain backward compatibility with existing code
* that expects a single pattern string
*/
export function extractCommandPattern(command: string): string {
const patterns = extractCommandPatterns(command)
return patterns[0] || ""
}