fix: improve error messages for OpenAI Compatible providers

- Add specific guidance when OpenAI Compatible models fail to use tools
- Provide clearer instructions about XML tool format requirements
- Help users understand common issues with tool usage formatting
- Add tests for the new error message variations

Fixes #7226

src/core/prompts/__tests__/responses-openai-compatible.spec.ts

@@ -0,0 +1,52 @@
+import { describe, it, expect } from "vitest"
+import { formatResponse } from "../responses"
+
+describe("formatResponse.noToolsUsed", () => {
+	it("should return standard message when no apiProvider is specified", () => {
+		const result = formatResponse.noToolsUsed()
+
+		expect(result).toContain("[ERROR] You did not use a tool in your previous response!")
+		expect(result).toContain("# Reminder: Instructions for Tool Use")
+		expect(result).not.toContain("OpenAI Compatible")
+	})
+
+	it("should return standard message for non-OpenAI-compatible providers", () => {
+		const result = formatResponse.noToolsUsed("anthropic")
+
+		expect(result).toContain("[ERROR] You did not use a tool in your previous response!")
+		expect(result).toContain("# Reminder: Instructions for Tool Use")
+		expect(result).not.toContain("OpenAI Compatible")
+	})
+
+	it("should include OpenAI Compatible specific hints when apiProvider is openai-compatible", () => {
+		const result = formatResponse.noToolsUsed("openai-compatible")
+
+		expect(result).toContain("[ERROR] You did not use a tool in your previous response!")
+		expect(result).toContain("# Important Note for OpenAI Compatible Models")
+		expect(result).toContain("Your model appears to not be using the required XML tool format")
+		expect(result).toContain("Use XML tags for ALL tool invocations")
+		expect(result).toContain("Place tool uses at the END of your message")
+		expect(result).toContain("Use only ONE tool per message")
+		expect(result).toContain("Follow the exact XML format shown below")
+		expect(result).toContain("# Reminder: Instructions for Tool Use")
+	})
+
+	it("should maintain the same structure with Next Steps section", () => {
+		const resultStandard = formatResponse.noToolsUsed()
+		const resultOpenAI = formatResponse.noToolsUsed("openai-compatible")
+
+		// Both should have the Next Steps section
+		expect(resultStandard).toContain("# Next Steps")
+		expect(resultOpenAI).toContain("# Next Steps")
+
+		// Both should mention attempt_completion and ask_followup_question
+		expect(resultStandard).toContain("attempt_completion")
+		expect(resultStandard).toContain("ask_followup_question")
+		expect(resultOpenAI).toContain("attempt_completion")
+		expect(resultOpenAI).toContain("ask_followup_question")
+
+		// Both should end with the automated message note
+		expect(resultStandard).toContain("(This is an automated message, so do not respond to it conversationally.)")
+		expect(resultOpenAI).toContain("(This is an automated message, so do not respond to it conversationally.)")
+	})
+})

src/core/prompts/responses.ts

@@ -18,17 +18,33 @@ export const formatResponse = {
 	rooIgnoreError: (path: string) =>
 		`Access to ${path} is blocked by the .rooignore file settings. You must try to continue in the task without using this file, or ask the user to update the .rooignore file.`,
 
-	noToolsUsed: () =>
-		`[ERROR] You did not use a tool in your previous response! Please retry with a tool use.
+	noToolsUsed: (apiProvider?: string) => {
+		const baseMessage = `[ERROR] You did not use a tool in your previous response! Please retry with a tool use.`
+
+		let providerSpecificHint = ""
+		if (apiProvider === "openai-compatible") {
+			providerSpecificHint = `
+
+# Important Note for OpenAI Compatible Models
+
+Your model appears to not be using the required XML tool format. Make sure to:
+1. Use XML tags for ALL tool invocations (not JSON or function calls)
+2. Place tool uses at the END of your message
+3. Use only ONE tool per message
+4. Follow the exact XML format shown below`
+		}
+
+		return `${baseMessage}${providerSpecificHint}
 
 ${toolUseInstructionsReminder}
 
 # Next Steps
 
-If you have completed the user's task, use the attempt_completion tool. 
-If you require additional information from the user, use the ask_followup_question tool. 
-Otherwise, if you have not completed the task and do not need additional information, then proceed with the next step of the task. 
-(This is an automated message, so do not respond to it conversationally.)`,
+If you have completed the user's task, use the attempt_completion tool.
+If you require additional information from the user, use the ask_followup_question tool.
+Otherwise, if you have not completed the task and do not need additional information, then proceed with the next step of the task.
+(This is an automated message, so do not respond to it conversationally.)`
+	},
 
 	tooManyMistakes: (feedback?: string) =>
 		`You seem to be having trouble proceeding. The user has provided the following feedback to help guide you:\n<feedback>\n${feedback}\n</feedback>`,

src/core/task/Task.ts

@@ -1510,7 +1510,9 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 				// the user hits max requests and denies resetting the count.
 				break
 			} else {
-				nextUserContent = [{ type: "text", text: formatResponse.noToolsUsed() }]
+				nextUserContent = [
+					{ type: "text", text: formatResponse.noToolsUsed(this.apiConfiguration.apiProvider) },
+				]
 				this.consecutiveMistakeCount++
 			}
 		}
@@ -1537,10 +1539,28 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 			}
 
 			if (this.consecutiveMistakeLimit > 0 && this.consecutiveMistakeCount >= this.consecutiveMistakeLimit) {
-				const { response, text, images } = await this.ask(
-					"mistake_limit_reached",
-					t("common:errors.mistake_limit_guidance"),
-				)
+				// Provide more specific guidance for OpenAI Compatible providers
+				const isOpenAICompatible = this.apiConfiguration.apiProvider === "openai-compatible"
+				const modelId = getModelId(this.apiConfiguration)
+
+				let guidanceMessage = t("common:errors.mistake_limit_guidance")
+
+				if (isOpenAICompatible) {
+					guidanceMessage = `The model appears to be having difficulty with tool usage. This often happens with OpenAI Compatible providers when the model doesn't properly format tool calls using XML tags.
+
+Common issues with ${modelId || "this model"}:
+1. The model may not be following the XML tool format correctly
+2. The model might be responding conversationally instead of using tools
+3. The model's output format may be incompatible with Roo Code's expectations
+
+Try these solutions:
+• Break down your request into smaller, more specific steps
+• Be more explicit about what you want to accomplish
+• Try a different model that better supports tool usage
+• Ensure your OpenAI Compatible endpoint is properly configured`
+				}
+
+				const { response, text, images } = await this.ask("mistake_limit_reached", guidanceMessage)
 
 				if (response === "messageResponse") {
 					currentUserContent.push(
@@ -2108,7 +2128,10 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 					const didToolUse = this.assistantMessageContent.some((block) => block.type === "tool_use")
 
 					if (!didToolUse) {
-						this.userMessageContent.push({ type: "text", text: formatResponse.noToolsUsed() })
+						this.userMessageContent.push({
+							type: "text",
+							text: formatResponse.noToolsUsed(this.apiConfiguration.apiProvider),
+						})
 						this.consecutiveMistakeCount++
 					}