From 3927d66b43d9f488e24fd616cda17a62ed77ee73 Mon Sep 17 00:00:00 2001 From: Dennise Bartlett Date: Sat, 20 Dec 2025 02:13:02 +0000 Subject: [PATCH] chore: Remove non-native e2e test for apply-diff tool --- .../src/suite/tools/apply-diff.test.ts | 793 ------------------ 1 file changed, 793 deletions(-) delete mode 100644 apps/vscode-e2e/src/suite/tools/apply-diff.test.ts diff --git a/apps/vscode-e2e/src/suite/tools/apply-diff.test.ts b/apps/vscode-e2e/src/suite/tools/apply-diff.test.ts deleted file mode 100644 index 3b55d09490..0000000000 --- a/apps/vscode-e2e/src/suite/tools/apply-diff.test.ts +++ /dev/null @@ -1,793 +0,0 @@ -import * as assert from "assert" -import * as fs from "fs/promises" -import * as path from "path" -import * as vscode from "vscode" - -import { RooCodeEventName, type ClineMessage } from "@roo-code/types" - -import { waitFor, sleep } from "../utils" -import { setDefaultSuiteTimeout } from "../test-utils" - -suite("Roo Code apply_diff Tool", function () { - setDefaultSuiteTimeout(this) - - let workspaceDir: string - - // Pre-created test files that will be used across tests - const testFiles = { - simpleModify: { - name: `test-file-simple-${Date.now()}.txt`, - content: "Hello World\nThis is a test file\nWith multiple lines", - path: "", - }, - multipleReplace: { - name: `test-func-multiple-${Date.now()}.js`, - content: `function calculate(x, y) { - const sum = x + y - const product = x * y - return { sum: sum, product: product } -}`, - path: "", - }, - lineNumbers: { - name: `test-lines-${Date.now()}.js`, - content: `// Header comment -function oldFunction() { - console.log("Old implementation") -} - -// Another function -function keepThis() { - console.log("Keep this") -} - -// Footer comment`, - path: "", - }, - errorHandling: { - name: `test-error-${Date.now()}.txt`, - content: "Original content", - path: "", - }, - multiSearchReplace: { - name: `test-multi-search-${Date.now()}.js`, - content: `function processData(data) { - console.log("Processing data") - return data.map(item => item * 2) -} - -// Some other code in between -const config = { - timeout: 5000, - retries: 3 -} - -function validateInput(input) { - console.log("Validating input") - if (!input) { - throw new Error("Invalid input") - } - return true -}`, - path: "", - }, - } - - // Get the actual workspace directory that VSCode is using and create all test files - suiteSetup(async function () { - // Get the workspace folder from VSCode - const workspaceFolders = vscode.workspace.workspaceFolders - if (!workspaceFolders || workspaceFolders.length === 0) { - throw new Error("No workspace folder found") - } - workspaceDir = workspaceFolders[0]!.uri.fsPath - console.log("Using workspace directory:", workspaceDir) - - // Create all test files before any tests run - console.log("Creating test files in workspace...") - for (const [key, file] of Object.entries(testFiles)) { - file.path = path.join(workspaceDir, file.name) - await fs.writeFile(file.path, file.content) - console.log(`Created ${key} test file at:`, file.path) - } - - // Verify all files exist - for (const [key, file] of Object.entries(testFiles)) { - const exists = await fs - .access(file.path) - .then(() => true) - .catch(() => false) - if (!exists) { - throw new Error(`Failed to create ${key} test file at ${file.path}`) - } - } - }) - - // Clean up after all tests - suiteTeardown(async () => { - // Cancel any running tasks before cleanup - try { - await globalThis.api.cancelCurrentTask() - } catch { - // Task might not be running - } - - // Clean up all test files - console.log("Cleaning up test files...") - for (const [key, file] of Object.entries(testFiles)) { - try { - await fs.unlink(file.path) - console.log(`Cleaned up ${key} test file`) - } catch (error) { - console.log(`Failed to clean up ${key} test file:`, error) - } - } - }) - - // Clean up before each test - setup(async () => { - // Cancel any previous task - try { - await globalThis.api.cancelCurrentTask() - } catch { - // Task might not be running - } - - // Reset all test files to their original content before each test - // This ensures each test starts with a known clean state, even if a previous - // test or run modified the file content - for (const [key, file] of Object.entries(testFiles)) { - if (file.path) { - try { - await fs.writeFile(file.path, file.content) - console.log(`Reset ${key} test file to original content`) - } catch (error) { - console.log(`Failed to reset ${key} test file:`, error) - } - } - } - - // Small delay to ensure clean state - await sleep(100) - }) - - // Clean up after each test - teardown(async () => { - // Cancel the current task - try { - await globalThis.api.cancelCurrentTask() - } catch { - // Task might not be running - } - - // Small delay to ensure clean state - await sleep(100) - }) - - test("Should apply diff to modify existing file content", async function () { - // Increase timeout for this specific test - - const api = globalThis.api - const messages: ClineMessage[] = [] - const testFile = testFiles.simpleModify - const expectedContent = "Hello Universe\nThis is a test file\nWith multiple lines" - let taskStarted = false - let taskCompleted = false - let errorOccurred: string | null = null - let applyDiffExecuted = false - - // Listen for messages - const messageHandler = ({ message }: { message: ClineMessage }) => { - messages.push(message) - - // Log important messages for debugging - if (message.type === "say" && message.say === "error") { - errorOccurred = message.text || "Unknown error" - console.error("Error:", message.text) - } - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool request:", message.text?.substring(0, 200)) - // Check for appliedDiff tool execution - try { - const toolData = JSON.parse(message.text || "{}") - if (toolData.tool === "appliedDiff") { - applyDiffExecuted = true - console.log("apply_diff tool executed!") - } - } catch (_e) { - // Not JSON or parsing failed - } - } - if (message.type === "say" && (message.say === "completion_result" || message.say === "text")) { - console.log("AI response:", message.text?.substring(0, 200)) - } - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - } - } - api.on(RooCodeEventName.Message, messageHandler) - - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string - try { - // Start task with apply_diff instruction - file already exists - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - apiProvider: "openrouter", // Use OpenRouter provider - openRouterModelId: "openai/gpt-5.1", // GPT-5.1 supports native tools - }, - text: `Use apply_diff on the file ${testFile.name} to change "Hello World" to "Hello Universe". The file already exists with this content: -${testFile.content}\nAssume the file exists and you can modify it directly.`, - }) //Temporary measure since list_files ignores all the files inside a tmp workspace - - console.log("Task ID:", taskId) - console.log("Test filename:", testFile.name) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 60_000 }) - - // Check for early errors - if (errorOccurred) { - console.error("Early error detected:", errorOccurred) - } - - // Wait for task completion - await waitFor(() => taskCompleted, { timeout: 60_000 }) - - // Give extra time for file system operations - await sleep(2000) - - // Check if the file was modified correctly - const actualContent = await fs.readFile(testFile.path, "utf-8") - console.log("File content after modification:", actualContent) - - // Verify tool was executed - assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed") - - // Verify file content - assert.strictEqual( - actualContent.trim(), - expectedContent.trim(), - "File content should be modified correctly", - ) - - console.log("Test passed! apply_diff tool executed and file modified successfully") - } finally { - // Clean up - api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) - } - }) - - test("Should apply multiple search/replace blocks in single diff", async function () { - // Increase timeout for this specific test - - const api = globalThis.api - const messages: ClineMessage[] = [] - const testFile = testFiles.multipleReplace - const expectedContent = `function compute(a, b) { - const total = a + b - const result = a * b - return { total: total, result: result } -}` - let taskStarted = false - let taskCompleted = false - let applyDiffExecuted = false - - // Listen for messages - const messageHandler = ({ message }: { message: ClineMessage }) => { - messages.push(message) - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool request:", message.text?.substring(0, 200)) - // Check for appliedDiff tool execution - try { - const toolData = JSON.parse(message.text || "{}") - if (toolData.tool === "appliedDiff") { - applyDiffExecuted = true - console.log("apply_diff tool executed!") - } - } catch (_e) { - // Not JSON or parsing failed - } - } - if (message.type === "say" && message.text) { - console.log("AI response:", message.text.substring(0, 200)) - } - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - } - } - api.on(RooCodeEventName.Message, messageHandler) - - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string - try { - // Start task with multiple replacements - file already exists - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - apiProvider: "openrouter", // Use OpenRouter provider - openRouterModelId: "openai/gpt-5.1", // GPT-5.1 supports native tools - }, - text: `Use apply_diff on the file ${testFile.name} to make ALL of these changes: -1. Rename function "calculate" to "compute" -2. Rename parameters "x, y" to "a, b" -3. Rename variable "sum" to "total" (including in the return statement) -4. Rename variable "product" to "result" (including in the return statement) -5. In the return statement, change { sum: sum, product: product } to { total: total, result: result } - -The file already exists with this content: -${testFile.content}\nAssume the file exists and you can modify it directly.`, - }) - - console.log("Task ID:", taskId) - console.log("Test filename:", testFile.name) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 60_000 }) - - // Wait for task completion - await waitFor(() => taskCompleted, { timeout: 60_000 }) - - // Give extra time for file system operations - await sleep(2000) - - // Check the file was modified correctly - const actualContent = await fs.readFile(testFile.path, "utf-8") - console.log("File content after modification:", actualContent) - - // Verify tool was executed - assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed") - - // Verify file content - assert.strictEqual( - actualContent.trim(), - expectedContent.trim(), - "All replacements should be applied correctly", - ) - - console.log("Test passed! apply_diff tool executed and multiple replacements applied successfully") - } finally { - // Clean up - api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) - } - }) - - test("Should handle apply_diff with line number hints", async function () { - // Increase timeout for this specific test - - const api = globalThis.api - const messages: ClineMessage[] = [] - const testFile = testFiles.lineNumbers - const expectedContent = `// Header comment -function newFunction() { - console.log("New implementation") -} - -// Another function -function keepThis() { - console.log("Keep this") -} - -// Footer comment` - - let taskStarted = false - let taskCompleted = false - let applyDiffExecuted = false - - // Listen for messages - const messageHandler = ({ message }: { message: ClineMessage }) => { - messages.push(message) - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool request:", message.text?.substring(0, 200)) - // Check for appliedDiff tool execution - try { - const toolData = JSON.parse(message.text || "{}") - if (toolData.tool === "appliedDiff") { - applyDiffExecuted = true - console.log("apply_diff tool executed!") - } - } catch (_e) { - // Not JSON or parsing failed - } - } - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - } - } - api.on(RooCodeEventName.Message, messageHandler) - - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string - try { - // Start task with line number context - file already exists - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - apiProvider: "openrouter", // Use OpenRouter provider - openRouterModelId: "openai/gpt-5.1", // GPT-5.1 supports native tools - }, - text: `Use apply_diff on the file ${testFile.name} to change "oldFunction" to "newFunction" and update its console.log to "New implementation". Keep the rest of the file unchanged. - -The file already exists with this content: -${testFile.content}\nAssume the file exists and you can modify it directly.`, - }) - - console.log("Task ID:", taskId) - console.log("Test filename:", testFile.name) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 60_000 }) - - // Wait for task completion - await waitFor(() => taskCompleted, { timeout: 60_000 }) - - // Give extra time for file system operations - await sleep(2000) - - // Check the file was modified correctly - const actualContent = await fs.readFile(testFile.path, "utf-8") - console.log("File content after modification:", actualContent) - - // Verify tool was executed - assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed") - - // Verify file content - assert.strictEqual( - actualContent.trim(), - expectedContent.trim(), - "Only specified function should be modified", - ) - - console.log("Test passed! apply_diff tool executed and targeted modification successful") - } finally { - // Clean up - api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) - } - }) - - test("Should handle apply_diff errors gracefully", async function () { - const api = globalThis.api - const messages: ClineMessage[] = [] - const testFile = testFiles.errorHandling - let taskStarted = false - let taskCompleted = false - let errorDetected = false - let applyDiffAttempted = false - let writeToFileUsed = false - - // Listen for messages - const messageHandler = ({ message }: { message: ClineMessage }) => { - messages.push(message) - - // Check for error messages - if (message.type === "say" && message.say === "error") { - errorDetected = true - console.log("Error detected:", message.text) - } - - // Check for tool execution attempt - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool ASK request:", message.text?.substring(0, 500)) - try { - const toolData = JSON.parse(message.text || "{}") - if (toolData.tool === "appliedDiff") { - applyDiffAttempted = true - console.log("apply_diff tool attempted via ASK!") - } - // Detect if write_to_file was used (shows as editedExistingFile or newFileCreated) - if (toolData.tool === "editedExistingFile" || toolData.tool === "newFileCreated") { - writeToFileUsed = true - console.log("write_to_file tool used!") - } - } catch (e) { - console.error(e) - } - } - - // Check for diff_error which indicates apply_diff was attempted but failed - if (message.type === "say" && message.say === "diff_error") { - applyDiffAttempted = true - } - - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - } - } - api.on(RooCodeEventName.Message, messageHandler) - - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string - try { - // Start task with invalid search content - file already exists - // NOTE: We ask the AI to search for a pattern that doesn't exist, without telling it - // what the replacement should be. This prevents the AI from working around by - // replacing the actual content with the desired replacement. - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - apiProvider: "openrouter", - openRouterModelId: "anthropic/claude-sonnet-4.5", - }, - text: `Use apply_diff on the file ${testFile.name} to find and replace the text "PATTERN_THAT_DOES_NOT_EXIST_xyz123" with "REPLACEMENT_xyz123". - -The file already exists with this content: -${testFile.content} - -CRITICAL INSTRUCTIONS: -1. You MUST attempt the apply_diff with EXACTLY the search pattern "PATTERN_THAT_DOES_NOT_EXIST_xyz123" -2. If apply_diff cannot find this exact search pattern, report that the pattern was not found -3. Do NOT use write_to_file under any circumstances -4. Do NOT use a different search pattern - use EXACTLY "PATTERN_THAT_DOES_NOT_EXIST_xyz123" -5. Do NOT modify the file in any other way -6. Simply report that the replacement could not be made because the search text was not found`, - }) - - console.log("Task ID:", taskId) - console.log("Test filename:", testFile.name) - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 90_000 }) - - // Wait for task completion or error - await waitFor(() => taskCompleted || errorDetected, { timeout: 90_000 }) - - // Give time for any final operations - await sleep(2000) - - // Read the file content - const actualContent = await fs.readFile(testFile.path, "utf-8") - console.log("File content after task:", actualContent) - - // The AI should have attempted to use apply_diff - assert.strictEqual(applyDiffAttempted, true, "apply_diff tool should have been attempted") - - // The AI should NOT have used write_to_file as a fallback - assert.strictEqual( - writeToFileUsed, - false, - "write_to_file should NOT be used when apply_diff fails - the AI should report the error instead", - ) - - // The content should remain unchanged since the search pattern wasn't found - assert.strictEqual( - actualContent.trim(), - testFile.content.trim(), - "File content should remain unchanged when search pattern not found", - ) - - console.log("Test passed! apply_diff attempted and error handled gracefully") - } finally { - // Clean up - api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) - } - }) - - test("Should apply multiple search/replace blocks to edit two separate functions", async function () { - const api = globalThis.api - const messages: ClineMessage[] = [] - const testFile = testFiles.multiSearchReplace - const expectedContent = `function transformData(data) { - console.log("Transforming data") - return data.map(item => item * 2) -} - -// Some other code in between -const config = { - timeout: 5000, - retries: 3 -} - -function checkInput(input) { - console.log("Checking input") - if (!input) { - throw new Error("Invalid input") - } - return true -}` - let taskStarted = false - let taskCompleted = false - let errorOccurred: string | null = null - let applyDiffExecuted = false - let applyDiffCount = 0 - - // Listen for messages - const messageHandler = ({ message }: { message: ClineMessage }) => { - messages.push(message) - - // Log important messages for debugging - if (message.type === "say" && message.say === "error") { - errorOccurred = message.text || "Unknown error" - console.error("Error:", message.text) - } - if (message.type === "ask" && message.ask === "tool") { - console.log("Tool request:", message.text?.substring(0, 200)) - // Check for appliedDiff tool execution - try { - const toolData = JSON.parse(message.text || "{}") - if (toolData.tool === "appliedDiff") { - applyDiffExecuted = true - applyDiffCount++ - console.log(`apply_diff tool executed! (count: ${applyDiffCount})`) - } - } catch (_e) { - // Not JSON or parsing failed - } - } - if (message.type === "say" && (message.say === "completion_result" || message.say === "text")) { - console.log("AI response:", message.text?.substring(0, 200)) - } - if (message.type === "say" && message.say === "api_req_started" && message.text) { - console.log("API request started:", message.text.substring(0, 200)) - } - } - api.on(RooCodeEventName.Message, messageHandler) - - // Listen for task events - const taskStartedHandler = (id: string) => { - if (id === taskId) { - taskStarted = true - console.log("Task started:", id) - } - } - api.on(RooCodeEventName.TaskStarted, taskStartedHandler) - - const taskCompletedHandler = (id: string) => { - if (id === taskId) { - taskCompleted = true - console.log("Task completed:", id) - } - } - api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler) - - let taskId: string - try { - // Start task with instruction to edit two separate functions using multiple search/replace blocks - taskId = await api.startNewTask({ - configuration: { - mode: "code", - autoApprovalEnabled: true, - alwaysAllowWrite: true, - alwaysAllowReadOnly: true, - alwaysAllowReadOnlyOutsideWorkspace: true, - apiProvider: "openrouter", // Use OpenRouter provider - openRouterModelId: "openai/gpt-5.1", // GPT-5.1 supports native tools - }, - text: `Use apply_diff on the file ${testFile.name} to make these changes. You MUST use TWO SEPARATE search/replace blocks within a SINGLE apply_diff call: - -FIRST search/replace block: Edit the processData function to rename it to "transformData" and change "Processing data" to "Transforming data" - -SECOND search/replace block: Edit the validateInput function to rename it to "checkInput" and change "Validating input" to "Checking input" - -Important: Use multiple SEARCH/REPLACE blocks in one apply_diff call, NOT multiple apply_diff calls. Each function should have its own search/replace block. - -The file already exists with this content: -${testFile.content} - -Assume the file exists and you can modify it directly.`, - }) - - console.log("Task ID:", taskId) - console.log("Test filename:", testFile.name) - - // Wait for task to start - await waitFor(() => taskStarted, { timeout: 60_000 }) - - // Check for early errors - if (errorOccurred) { - console.error("Early error detected:", errorOccurred) - } - - // Wait for task completion - await waitFor(() => taskCompleted, { timeout: 60_000 }) - - // Give extra time for file system operations - await sleep(2000) - - // Check if the file was modified correctly - const actualContent = await fs.readFile(testFile.path, "utf-8") - console.log("File content after modification:", actualContent) - - // Verify tool was executed - assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed") - console.log(`apply_diff was executed ${applyDiffCount} time(s)`) - - // Verify file content - assert.strictEqual( - actualContent.trim(), - expectedContent.trim(), - "Both functions should be modified with separate search/replace blocks", - ) - - console.log("Test passed! apply_diff tool executed and multiple search/replace blocks applied successfully") - } finally { - // Clean up - api.off(RooCodeEventName.Message, messageHandler) - api.off(RooCodeEventName.TaskStarted, taskStartedHandler) - api.off(RooCodeEventName.TaskCompleted, taskCompletedHandler) - } - }) -})