From 1d1f5c9c3b4e5523f483ef16187c92df48bdb254 Mon Sep 17 00:00:00 2001 From: cte Date: Mon, 24 Feb 2025 13:43:47 -0800 Subject: [PATCH] Integration test cleanup --- src/test/suite/index.ts | 17 ++-- src/test/suite/modes.test.ts | 170 ++++++++++++++++++----------------- src/test/suite/task.test.ts | 74 +++++++-------- 3 files changed, 131 insertions(+), 130 deletions(-) diff --git a/src/test/suite/index.ts b/src/test/suite/index.ts index 540be7cef8..cc487b0bf7 100644 --- a/src/test/suite/index.ts +++ b/src/test/suite/index.ts @@ -13,23 +13,23 @@ declare global { } export async function run(): Promise { - // Create the mocha test const mocha = new Mocha({ ui: "tdd", - timeout: 600000, // 10 minutes to compensate for time communicating with LLM while running in GHA + timeout: 600000, // 10 minutes to compensate for time communicating with LLM while running in GHA. }) const testsRoot = path.resolve(__dirname, "..") try { - // Find all test files + // Find all test files. const files = await glob("**/**.test.js", { cwd: testsRoot }) - // Add files to the test suite + // Add files to the test suite. files.forEach((f: string) => mocha.addFile(path.resolve(testsRoot, f))) - //Set up global extension, api, provider, and panel + // Set up global extension, api, provider, and panel. globalThis.extension = vscode.extensions.getExtension("RooVeterinaryInc.roo-cline") + if (!globalThis.extension) { throw new Error("Extension not found") } @@ -37,9 +37,12 @@ export async function run(): Promise { globalThis.api = globalThis.extension.isActive ? globalThis.extension.exports : await globalThis.extension.activate() + globalThis.provider = globalThis.api.sidebarProvider + await globalThis.provider.updateGlobalState("apiProvider", "openrouter") - await globalThis.provider.updateGlobalState("openRouterModelId", "anthropic/claude-3.7-sonnet") + await globalThis.provider.updateGlobalState("openRouterModelId", "anthropic/claude-3.5-sonnet") + await globalThis.provider.storeSecret( "openRouterApiKey", process.env.OPENROUTER_API_KEY || "sk-or-v1-fake-api-key", @@ -71,7 +74,7 @@ export async function run(): Promise { await new Promise((resolve) => setTimeout(resolve, interval)) } - // Run the mocha test + // Run the mocha test. return new Promise((resolve, reject) => { try { mocha.run((failures: number) => { diff --git a/src/test/suite/modes.test.ts b/src/test/suite/modes.test.ts index 2fe0eaa597..b94e71d110 100644 --- a/src/test/suite/modes.test.ts +++ b/src/test/suite/modes.test.ts @@ -1,101 +1,105 @@ import * as assert from "assert" -import * as vscode from "vscode" suite("Roo Code Modes", () => { test("Should handle switching modes correctly", async function () { const timeout = 30000 const interval = 1000 + const testPrompt = "For each mode (Code, Architect, Ask) respond with the mode name and what it specializes in after switching to that mode, do not start with the current mode, be sure to say 'I AM DONE' after the task is complete" + if (!globalThis.extension) { assert.fail("Extension not found") } - try { - let startTime = Date.now() + let startTime = Date.now() - // Ensure the webview is launched. - while (Date.now() - startTime < timeout) { - if (globalThis.provider.viewLaunched) { - break - } - - await new Promise((resolve) => setTimeout(resolve, interval)) + // Ensure the webview is launched. + while (Date.now() - startTime < timeout) { + if (globalThis.provider.viewLaunched) { + break } - await globalThis.provider.updateGlobalState("mode", "Ask") - await globalThis.provider.updateGlobalState("alwaysAllowModeSwitch", true) - await globalThis.provider.updateGlobalState("autoApprovalEnabled", true) - - // Start a new task. - await globalThis.api.startNewTask(testPrompt) - - // Wait for task to appear in history with tokens. - startTime = Date.now() - - while (Date.now() - startTime < timeout) { - const messages = globalThis.provider.messages - - if ( - messages.some( - ({ type, text }) => - type === "say" && text?.includes("I AM DONE") && !text?.includes("be sure to say"), - ) - ) { - break - } - - await new Promise((resolve) => setTimeout(resolve, interval)) - } - if (globalThis.provider.messages.length === 0) { - assert.fail("No messages received") - } - - //Log the messages to the console - globalThis.provider.messages.forEach(({ type, text }) => { - if (type === "say") { - console.log(text) - } - }) - - //Start Grading Portion of test to grade the response from 1 to 10 - await globalThis.provider.updateGlobalState("mode", "Ask") - let output = globalThis.provider.messages.map(({ type, text }) => (type === "say" ? text : "")).join("\n") - await globalThis.api.startNewTask( - `Given this prompt: ${testPrompt} grade the response from 1 to 10 in the format of "Grade: (1-10)": ${output} \n Be sure to say 'I AM DONE GRADING' after the task is complete`, - ) - - startTime = Date.now() - - while (Date.now() - startTime < timeout) { - const messages = globalThis.provider.messages - - if ( - messages.some( - ({ type, text }) => - type === "say" && text?.includes("I AM DONE GRADING") && !text?.includes("be sure to say"), - ) - ) { - break - } - - await new Promise((resolve) => setTimeout(resolve, interval)) - } - if (globalThis.provider.messages.length === 0) { - assert.fail("No messages received") - } - globalThis.provider.messages.forEach(({ type, text }) => { - if (type === "say" && text?.includes("Grade:")) { - console.log(text) - } - }) - const gradeMessage = globalThis.provider.messages.find( - ({ type, text }) => type === "say" && !text?.includes("Grade: (1-10)") && text?.includes("Grade:"), - )?.text - const gradeMatch = gradeMessage?.match(/Grade: (\d+)/) - const gradeNum = gradeMatch ? parseInt(gradeMatch[1]) : undefined - assert.ok(gradeNum !== undefined && gradeNum >= 7 && gradeNum <= 10, "Grade must be between 7 and 10") - } finally { + await new Promise((resolve) => setTimeout(resolve, interval)) } + + await globalThis.provider.updateGlobalState("mode", "Ask") + await globalThis.provider.updateGlobalState("alwaysAllowModeSwitch", true) + await globalThis.provider.updateGlobalState("autoApprovalEnabled", true) + + // Start a new task. + await globalThis.api.startNewTask(testPrompt) + + // Wait for task to appear in history with tokens. + startTime = Date.now() + + while (Date.now() - startTime < timeout) { + const messages = globalThis.provider.messages + + if ( + messages.some( + ({ type, text }) => + type === "say" && text?.includes("I AM DONE") && !text?.includes("be sure to say"), + ) + ) { + break + } + + await new Promise((resolve) => setTimeout(resolve, interval)) + } + + if (globalThis.provider.messages.length === 0) { + assert.fail("No messages received") + } + + // Log the messages to the console. + globalThis.provider.messages.forEach(({ type, text }) => { + if (type === "say") { + console.log(text) + } + }) + + // Start Grading Portion of test to grade the response from 1 to 10. + await globalThis.provider.updateGlobalState("mode", "Ask") + let output = globalThis.provider.messages.map(({ type, text }) => (type === "say" ? text : "")).join("\n") + + await globalThis.api.startNewTask( + `Given this prompt: ${testPrompt} grade the response from 1 to 10 in the format of "Grade: (1-10)": ${output} \n Be sure to say 'I AM DONE GRADING' after the task is complete`, + ) + + startTime = Date.now() + + while (Date.now() - startTime < timeout) { + const messages = globalThis.provider.messages + + if ( + messages.some( + ({ type, text }) => + type === "say" && text?.includes("I AM DONE GRADING") && !text?.includes("be sure to say"), + ) + ) { + break + } + + await new Promise((resolve) => setTimeout(resolve, interval)) + } + + if (globalThis.provider.messages.length === 0) { + assert.fail("No messages received") + } + + globalThis.provider.messages.forEach(({ type, text }) => { + if (type === "say" && text?.includes("Grade:")) { + console.log(text) + } + }) + + const gradeMessage = globalThis.provider.messages.find( + ({ type, text }) => type === "say" && !text?.includes("Grade: (1-10)") && text?.includes("Grade:"), + )?.text + + const gradeMatch = gradeMessage?.match(/Grade: (\d+)/) + const gradeNum = gradeMatch ? parseInt(gradeMatch[1]) : undefined + assert.ok(gradeNum !== undefined && gradeNum >= 7 && gradeNum <= 10, "Grade must be between 7 and 10") }) }) diff --git a/src/test/suite/task.test.ts b/src/test/suite/task.test.ts index 2d34bc78ff..6bdedcde00 100644 --- a/src/test/suite/task.test.ts +++ b/src/test/suite/task.test.ts @@ -1,5 +1,4 @@ import * as assert from "assert" -import * as vscode from "vscode" suite("Roo Code Task", () => { test("Should handle prompt and response correctly", async function () { @@ -10,48 +9,43 @@ suite("Roo Code Task", () => { assert.fail("Extension not found") } - try { - // Ensure the webview is launched. - let startTime = Date.now() + // Ensure the webview is launched. + let startTime = Date.now() - while (Date.now() - startTime < timeout) { - if (globalThis.provider.viewLaunched) { - break - } - - await new Promise((resolve) => setTimeout(resolve, interval)) + while (Date.now() - startTime < timeout) { + if (globalThis.provider.viewLaunched) { + break } - await globalThis.provider.updateGlobalState("mode", "Code") - await globalThis.provider.updateGlobalState("alwaysAllowModeSwitch", true) - await globalThis.provider.updateGlobalState("autoApprovalEnabled", true) - - await globalThis.api.startNewTask("Hello world, what is your name? Respond with 'My name is ...'") - - // Wait for task to appear in history with tokens. - startTime = Date.now() - - while (Date.now() - startTime < timeout) { - const messages = globalThis.provider.messages - - if (messages.some(({ type, text }) => type === "say" && text?.includes("My name is Roo"))) { - break - } - - await new Promise((resolve) => setTimeout(resolve, interval)) - } - - if (globalThis.provider.messages.length === 0) { - assert.fail("No messages received") - } - - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => type === "say" && text?.includes("My name is Roo"), - ), - "Did not receive expected response containing 'My name is Roo'", - ) - } finally { + await new Promise((resolve) => setTimeout(resolve, interval)) } + + await globalThis.provider.updateGlobalState("mode", "Code") + await globalThis.provider.updateGlobalState("alwaysAllowModeSwitch", true) + await globalThis.provider.updateGlobalState("autoApprovalEnabled", true) + + await globalThis.api.startNewTask("Hello world, what is your name? Respond with 'My name is ...'") + + // Wait for task to appear in history with tokens. + startTime = Date.now() + + while (Date.now() - startTime < timeout) { + const messages = globalThis.provider.messages + + if (messages.some(({ type, text }) => type === "say" && text?.includes("My name is Roo"))) { + break + } + + await new Promise((resolve) => setTimeout(resolve, interval)) + } + + if (globalThis.provider.messages.length === 0) { + assert.fail("No messages received") + } + + assert.ok( + globalThis.provider.messages.some(({ type, text }) => type === "say" && text?.includes("My name is Roo")), + "Did not receive expected response containing 'My name is Roo'", + ) }) })