From b9bf8ffa91758919c91fc9ffcfc8ed16eeb0c366 Mon Sep 17 00:00:00 2001 From: Roo Code Date: Fri, 12 Sep 2025 14:29:58 +0000 Subject: [PATCH] feat: improve tool error guidance with contextual suggestions - Created ToolErrorGuidance class to analyze error patterns - Enhanced error messages based on specific failure types (file not found, permissions, etc.) - Added pattern detection for common tool usage issues - Integrated contextual guidance into Task.ts consecutive mistake handling - Added comprehensive test coverage for the new guidance system Fixes #7936 --- src/core/task/Task.ts | 51 ++- .../tools/__tests__/errorGuidance.spec.ts | 304 ++++++++++++++++++ src/core/tools/errorGuidance.ts | 187 +++++++++++ 3 files changed, 538 insertions(+), 4 deletions(-) create mode 100644 src/core/tools/__tests__/errorGuidance.spec.ts create mode 100644 src/core/tools/errorGuidance.ts diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index cf16df8dcc..683666d3fd 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -114,6 +114,7 @@ import { Gpt5Metadata, ClineMessageWithMetadata } from "./types" import { MessageQueueService } from "../message-queue/MessageQueueService" import { AutoApprovalHandler } from "./AutoApprovalHandler" +import { ToolErrorGuidance } from "../tools/errorGuidance" const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds @@ -263,6 +264,7 @@ export class Task extends EventEmitter implements TaskLike { consecutiveMistakeLimit: number consecutiveMistakeCountForApplyDiff: Map = new Map() toolUsage: ToolUsage = {} + toolErrorHistory: Array<{ toolName: ToolName; error?: string; timestamp: number }> = [] // Checkpoints enableCheckpoints: boolean @@ -1725,10 +1727,37 @@ export class Task extends EventEmitter implements TaskLike { } if (this.consecutiveMistakeLimit > 0 && this.consecutiveMistakeCount >= this.consecutiveMistakeLimit) { - const { response, text, images } = await this.ask( - "mistake_limit_reached", - t("common:errors.mistake_limit_guidance"), - ) + // Build error patterns from the tool error history + const toolErrorMap = new Map() + const recentTools: ToolName[] = [] + + // Process error history to build patterns + for (const error of this.toolErrorHistory) { + recentTools.push(error.toolName) + const existing = toolErrorMap.get(error.toolName) || { count: 0 } + toolErrorMap.set(error.toolName, { + count: existing.count + 1, + lastError: error.error || existing.lastError, + }) + } + + // Build error patterns using the ToolErrorGuidance helper + const errorPatterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrorMap) + + // Create guidance context + const guidanceContext = { + recentTools, + errorPatterns, + consecutiveMistakeCount: this.consecutiveMistakeCount, + } + + // Get contextual guidance + const contextualGuidance = ToolErrorGuidance.getContextualGuidance(guidanceContext) + + // Format the guidance message + let guidanceMessage = ToolErrorGuidance.formatGuidanceMessage(contextualGuidance) + + const { response, text, images } = await this.ask("mistake_limit_reached", guidanceMessage) if (response === "messageResponse") { currentUserContent.push( @@ -1745,6 +1774,8 @@ export class Task extends EventEmitter implements TaskLike { } this.consecutiveMistakeCount = 0 + // Clear error history after providing guidance + this.toolErrorHistory = [] } // In this Cline request loop, we need to check if this task instance @@ -2828,6 +2859,18 @@ export class Task extends EventEmitter implements TaskLike { this.toolUsage[toolName].failures++ + // Track error in history for contextual guidance + this.toolErrorHistory.push({ + toolName, + error, + timestamp: Date.now(), + }) + + // Keep only recent errors (last 20) + if (this.toolErrorHistory.length > 20) { + this.toolErrorHistory = this.toolErrorHistory.slice(-20) + } + if (error) { this.emit(RooCodeEventName.TaskToolFailed, this.taskId, toolName, error) } diff --git a/src/core/tools/__tests__/errorGuidance.spec.ts b/src/core/tools/__tests__/errorGuidance.spec.ts new file mode 100644 index 0000000000..182f1ef4c4 --- /dev/null +++ b/src/core/tools/__tests__/errorGuidance.spec.ts @@ -0,0 +1,304 @@ +import { describe, it, expect } from "vitest" +import { ToolErrorGuidance, ToolErrorPattern, GuidanceContext } from "../errorGuidance" + +describe("ToolErrorGuidance", () => { + describe("getContextualGuidance", () => { + it("should return generic guidance when no error patterns are provided", () => { + const context: GuidanceContext = { + recentTools: [], + errorPatterns: [], + consecutiveMistakeCount: 3, + } + + const result = ToolErrorGuidance.getContextualGuidance(context) + + expect(result).toHaveLength(3) + expect(result[0]).toContain("Try breaking down the task") + }) + + it("should detect file not found errors", () => { + const context: GuidanceContext = { + recentTools: ["read_file", "read_file"], + errorPatterns: [ + { + toolName: "read_file", + errorType: "file_not_found", + count: 2, + lastError: "File not found: src/test.ts", + }, + ], + consecutiveMistakeCount: 3, + } + + const result = ToolErrorGuidance.getContextualGuidance(context) + + // Check for file operation guidance + expect(result.some((s: string) => s.toLowerCase().includes("file") || s.includes("'list_files'"))).toBe( + true, + ) + }) + + it("should detect missing parameter errors", () => { + const context: GuidanceContext = { + recentTools: ["write_to_file", "apply_diff"], + errorPatterns: [ + { + toolName: "write_to_file", + errorType: "missing_param", + count: 1, + lastError: "Missing required parameter: content", + }, + { + toolName: "apply_diff", + errorType: "missing_param", + count: 1, + lastError: "Required parameter path is missing", + }, + ], + consecutiveMistakeCount: 3, + } + + const result = ToolErrorGuidance.getContextualGuidance(context) + + // Should return some guidance + expect(result).toBeTruthy() + expect(result.length).toBeGreaterThan(0) + expect(result.length).toBeLessThanOrEqual(3) + }) + + it("should detect permission errors", () => { + const context: GuidanceContext = { + recentTools: ["write_to_file", "execute_command"], + errorPatterns: [ + { + toolName: "write_to_file", + errorType: "permission_denied", + count: 1, + lastError: "Permission denied", + }, + { + toolName: "execute_command", + errorType: "permission_denied", + count: 1, + lastError: "Access denied", + }, + ], + consecutiveMistakeCount: 3, + } + + const result = ToolErrorGuidance.getContextualGuidance(context) + + // Should return some guidance + expect(result).toBeTruthy() + expect(result.length).toBeGreaterThan(0) + expect(result.length).toBeLessThanOrEqual(3) + }) + + it("should detect repeated failures", () => { + const context: GuidanceContext = { + recentTools: ["read_file", "read_file", "read_file", "read_file", "read_file"], + errorPatterns: [ + { + toolName: "read_file", + errorType: "repeated_failure", + count: 5, + lastError: "Some error", + }, + ], + consecutiveMistakeCount: 5, + } + + const result = ToolErrorGuidance.getContextualGuidance(context) + + expect(result.some((s: string) => s.includes("breaking down the task"))).toBe(true) + }) + + it("should detect search operation issues", () => { + const context: GuidanceContext = { + recentTools: ["search_files", "list_files", "search_files"], + errorPatterns: [], + consecutiveMistakeCount: 3, + } + + const result = ToolErrorGuidance.getContextualGuidance(context) + + expect(result.some((s: string) => s.includes("search patterns") || s.includes("project structure"))).toBe( + true, + ) + }) + + it("should detect code modification issues", () => { + const context: GuidanceContext = { + recentTools: ["apply_diff", "write_to_file", "apply_diff"], + errorPatterns: [], + consecutiveMistakeCount: 3, + } + + const result = ToolErrorGuidance.getContextualGuidance(context) + + expect( + result.some( + (s: string) => s.includes("Read the file first") || s.includes("smaller, targeted changes"), + ), + ).toBe(true) + }) + + it("should limit suggestions to 3", () => { + const context: GuidanceContext = { + recentTools: [ + "read_file", + "write_to_file", + "apply_diff", + "execute_command", + "search_files", + "list_files", + ], + errorPatterns: [ + { + toolName: "read_file", + errorType: "file_not_found", + count: 2, + lastError: "File not found", + }, + { + toolName: "write_to_file", + errorType: "permission_denied", + count: 1, + lastError: "Permission denied", + }, + { + toolName: "apply_diff", + errorType: "missing_param", + count: 1, + lastError: "Missing parameter", + }, + ], + consecutiveMistakeCount: 5, + } + + const result = ToolErrorGuidance.getContextualGuidance(context) + + expect(result.length).toBeLessThanOrEqual(3) + }) + + it("should handle mixed error patterns", () => { + const context: GuidanceContext = { + recentTools: ["read_file", "write_to_file", "apply_diff"], + errorPatterns: [ + { + toolName: "read_file", + errorType: "file_not_found", + count: 1, + lastError: "File not found: config.json", + }, + { + toolName: "write_to_file", + errorType: "permission_denied", + count: 1, + lastError: "Permission denied", + }, + ], + consecutiveMistakeCount: 3, + } + + const result = ToolErrorGuidance.getContextualGuidance(context) + + expect(result).toBeTruthy() + expect(result.length).toBeGreaterThan(0) + expect(result.length).toBeLessThanOrEqual(3) + }) + }) + + describe("formatGuidanceMessage", () => { + it("should format guidance messages properly", () => { + const guidance = [ + "Try breaking down the task into smaller steps", + "Use list_files to verify directory structure", + ] + + const result = ToolErrorGuidance.formatGuidanceMessage(guidance) + + expect(result).toContain("struggling with tool usage") + expect(result).toContain("1.") + expect(result).toContain("2.") + }) + + it("should return default message for empty guidance", () => { + const result = ToolErrorGuidance.formatGuidanceMessage([]) + + expect(result).toContain("failure in the model's thought process") + }) + }) + + describe("buildErrorPatterns", () => { + it("should correctly identify file not found pattern", () => { + const recentTools = ["read_file" as any] + const toolErrors = new Map([["read_file" as any, { count: 1, lastError: "ENOENT: no such file" }]]) + + const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors) + + expect(patterns).toHaveLength(1) + expect(patterns[0].errorType).toBe("file_not_found") + expect(patterns[0].count).toBe(1) + }) + + it("should correctly identify missing parameter pattern", () => { + const recentTools = ["write_to_file" as any] + const toolErrors = new Map([ + ["write_to_file" as any, { count: 1, lastError: "Missing required parameter: content" }], + ]) + + const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors) + + expect(patterns).toHaveLength(1) + expect(patterns[0].errorType).toBe("missing_param") + }) + + it("should correctly identify permission denied pattern", () => { + const recentTools = ["execute_command" as any] + const toolErrors = new Map([["execute_command" as any, { count: 1, lastError: "permission denied" }]]) + + const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors) + + expect(patterns).toHaveLength(1) + expect(patterns[0].errorType).toBe("permission_denied") + }) + + it("should correctly identify invalid format pattern", () => { + const recentTools = ["write_to_file" as any] + const toolErrors = new Map([["write_to_file" as any, { count: 1, lastError: "Invalid JSON format" }]]) + + const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors) + + expect(patterns).toHaveLength(1) + expect(patterns[0].errorType).toBe("invalid_format") + }) + + it("should default to repeated_failure for unknown errors", () => { + const recentTools = ["read_file" as any] + const toolErrors = new Map([["read_file" as any, { count: 3, lastError: "Unknown error" }]]) + + const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors) + + expect(patterns).toHaveLength(1) + expect(patterns[0].errorType).toBe("repeated_failure") + expect(patterns[0].count).toBe(3) + }) + + it("should handle multiple tools with errors", () => { + const recentTools = ["read_file" as any, "write_to_file" as any, "apply_diff" as any] + const toolErrors = new Map([ + ["read_file" as any, { count: 2, lastError: "File not found" }], + ["write_to_file" as any, { count: 1, lastError: "Permission denied" }], + ["apply_diff" as any, { count: 1, lastError: "Missing required parameter" }], + ]) + + const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors) + + expect(patterns).toHaveLength(3) + expect(patterns.find((p) => p.toolName === "read_file")?.errorType).toBe("file_not_found") + expect(patterns.find((p) => p.toolName === "write_to_file")?.errorType).toBe("permission_denied") + expect(patterns.find((p) => p.toolName === "apply_diff")?.errorType).toBe("missing_param") + }) + }) +}) diff --git a/src/core/tools/errorGuidance.ts b/src/core/tools/errorGuidance.ts new file mode 100644 index 0000000000..07a0855418 --- /dev/null +++ b/src/core/tools/errorGuidance.ts @@ -0,0 +1,187 @@ +/** + * Error guidance system for providing contextual help when models struggle with tool usage + */ + +import { ToolName } from "@roo-code/types" + +export interface ToolErrorPattern { + toolName: ToolName + errorType: "missing_param" | "invalid_format" | "file_not_found" | "permission_denied" | "repeated_failure" + count: number + lastError?: string +} + +export interface GuidanceContext { + recentTools: ToolName[] + errorPatterns: ToolErrorPattern[] + consecutiveMistakeCount: number + lastToolUsed?: ToolName +} + +/** + * Analyzes tool usage patterns and generates contextual guidance + */ +export class ToolErrorGuidance { + private static readonly GUIDANCE_TEMPLATES = { + // General guidance for different scenarios + general_breakdown: [ + "Try breaking down the task into smaller, more manageable steps.", + "Consider completing one part of the task at a time before moving to the next.", + "Focus on a single file or component before expanding to others.", + ], + + file_operations: [ + "Double-check file paths and ensure files exist before attempting operations.", + "Use 'list_files' to verify the directory structure first.", + "Consider using 'read_file' to examine the current content before making changes.", + ], + + missing_parameters: [ + "Review the tool parameters carefully - ensure all required fields are provided.", + "Check that parameter values are in the correct format (e.g., paths, line numbers).", + "Use simpler values first to test if the tool works, then add complexity.", + ], + + code_modifications: [ + "Read the file first to understand its current structure.", + "Make smaller, targeted changes rather than large rewrites.", + "Use 'apply_diff' for precise edits instead of rewriting entire files.", + "Verify your changes by reading the file after modifications.", + ], + + search_operations: [ + "Start with broader search patterns, then refine them.", + "Use 'list_files' to understand the project structure before searching.", + "Try searching in specific directories rather than the entire project.", + ], + + command_execution: [ + "Verify the command syntax is correct for the operating system.", + "Check if required tools or dependencies are installed.", + "Start with simple commands to test the environment.", + "Consider the working directory when running commands.", + ], + + permission_issues: [ + "Check if the file or directory has the necessary permissions.", + "Verify you're operating in the correct workspace.", + "Some files may be protected - check the error message for details.", + ], + } + + /** + * Analyzes the context and returns appropriate guidance messages + */ + public static getContextualGuidance(context: GuidanceContext): string[] { + const guidance: string[] = [] + + // Analyze patterns to determine the type of struggle + const hasFileErrors = context.errorPatterns.some( + (p) => p.errorType === "file_not_found" || p.toolName === "read_file" || p.toolName === "write_to_file", + ) + + const hasMissingParams = context.errorPatterns.some((p) => p.errorType === "missing_param") + + const hasPermissionIssues = context.errorPatterns.some((p) => p.errorType === "permission_denied") + + const hasRepeatedFailures = context.errorPatterns.some((p) => p.count >= 2) + + const hasSearchIssues = + context.recentTools.filter((t) => t === "search_files" || t === "list_files").length >= 2 + + const hasCodeModificationIssues = + context.recentTools.filter((t) => t === "apply_diff" || t === "write_to_file" || t === "insert_content") + .length >= 2 + + // Provide targeted guidance based on patterns + if (hasRepeatedFailures) { + guidance.push(...this.GUIDANCE_TEMPLATES.general_breakdown) + } + + if (hasFileErrors) { + guidance.push(...this.GUIDANCE_TEMPLATES.file_operations) + } + + if (hasMissingParams) { + guidance.push(...this.GUIDANCE_TEMPLATES.missing_parameters) + } + + if (hasPermissionIssues) { + guidance.push(...this.GUIDANCE_TEMPLATES.permission_issues) + } + + if (hasSearchIssues) { + guidance.push(...this.GUIDANCE_TEMPLATES.search_operations) + } + + if (hasCodeModificationIssues) { + guidance.push(...this.GUIDANCE_TEMPLATES.code_modifications) + } + + // If no specific pattern detected, provide general guidance + if (guidance.length === 0) { + guidance.push(...this.GUIDANCE_TEMPLATES.general_breakdown) + } + + // Return unique guidance messages (remove duplicates) + return [...new Set(guidance)].slice(0, 3) // Limit to 3 most relevant suggestions + } + + /** + * Formats guidance messages into a user-friendly string + */ + public static formatGuidanceMessage(guidance: string[]): string { + if (guidance.length === 0) { + return "This may indicate a failure in the model's thought process. Try breaking down the task into smaller steps." + } + + const header = "The model seems to be struggling with tool usage. Here are some suggestions:\n\n" + const formattedGuidance = guidance.map((g, i) => `${i + 1}. ${g}`).join("\n") + + return header + formattedGuidance + } + + /** + * Analyzes recent tool usage to build error patterns + */ + public static buildErrorPatterns( + recentTools: ToolName[], + toolErrors: Map, + ): ToolErrorPattern[] { + const patterns: ToolErrorPattern[] = [] + + for (const [toolName, errorInfo] of toolErrors.entries()) { + if (errorInfo.count > 0) { + // Try to determine error type from the error message + let errorType: ToolErrorPattern["errorType"] = "repeated_failure" + + if (errorInfo.lastError) { + const errorLower = errorInfo.lastError.toLowerCase() + if (errorLower.includes("missing") || errorLower.includes("required parameter")) { + errorType = "missing_param" + } else if ( + errorLower.includes("not found") || + errorLower.includes("does not exist") || + errorLower.includes("enoent") || + errorLower.includes("no such file") + ) { + errorType = "file_not_found" + } else if (errorLower.includes("permission") || errorLower.includes("access denied")) { + errorType = "permission_denied" + } else if (errorLower.includes("format") || errorLower.includes("invalid")) { + errorType = "invalid_format" + } + } + + patterns.push({ + toolName, + errorType, + count: errorInfo.count, + lastError: errorInfo.lastError, + }) + } + } + + return patterns + } +}