From d7b44a2828bd7df85b837c7c8970cb1660f0430c Mon Sep 17 00:00:00 2001 From: ColemanRoo Date: Tue, 11 Feb 2025 10:55:04 -0600 Subject: [PATCH] Converting modes.test.ts to use grading to measure if the test passes Add documentation for VSCode Integration Tests Use explicit list of tests instead of searching ot make it easier to run 1 test at time locally. --- src/test/VSCODE_INTEGRATION_TESTS.md | 160 +++++++++++++++++++++++++++ src/test/suite/index.ts | 3 +- src/test/suite/modes.test.ts | 79 +++++++------ 3 files changed, 201 insertions(+), 41 deletions(-) create mode 100644 src/test/VSCODE_INTEGRATION_TESTS.md diff --git a/src/test/VSCODE_INTEGRATION_TESTS.md b/src/test/VSCODE_INTEGRATION_TESTS.md new file mode 100644 index 0000000000..6192c4488f --- /dev/null +++ b/src/test/VSCODE_INTEGRATION_TESTS.md @@ -0,0 +1,160 @@ +# VSCode Integration Tests + +This document describes the integration test setup for the Roo Code VSCode extension. + +## Overview + +The integration tests use the `@vscode/test-electron` package to run tests in a real VSCode environment. These tests verify that the extension works correctly within VSCode, including features like mode switching, webview interactions, and API communication. + +## Test Setup + +### Directory Structure + +``` +src/test/ +├── runTest.ts # Main test runner +├── suite/ +│ ├── index.ts # Test suite configuration +│ ├── modes.test.ts # Mode switching tests +│ ├── tasks.test.ts # Task execution tests +│ └── extension.test.ts # Extension activation tests +``` + +### Test Runner Configuration + +The test runner (`runTest.ts`) is responsible for: + +- Setting up the extension development path +- Configuring the test environment +- Running the integration tests using `@vscode/test-electron` + +### Environment Setup + +1. Create a `.env.integration` file in the root directory with required environment variables: + +``` +OPENROUTER_API_KEY=sk-or-v1-... +``` + +2. The test suite (`suite/index.ts`) configures: + +- Mocha test framework with TDD interface +- 10-minute timeout for LLM communication +- Global extension API access +- WebView panel setup +- OpenRouter API configuration + +## Test Suite Structure + +Tests are organized using Mocha's TDD interface (`suite` and `test` functions). The main test files are: + +- `modes.test.ts`: Tests mode switching functionality +- `tasks.test.ts`: Tests task execution +- `extension.test.ts`: Tests extension activation + +### Global Objects + +The following global objects are available in tests: + +```typescript +declare global { + var api: ClineAPI + var provider: ClineProvider + var extension: vscode.Extension + var panel: vscode.WebviewPanel +} +``` + +## Running Tests + +1. Ensure you have the required environment variables set in `.env.integration` + +2. Run the integration tests: + +```bash +npm run test:integration +``` + +The tests will: + +- Download and launch a clean VSCode instance +- Install the extension +- Execute the test suite +- Report results + +## Writing New Tests + +When writing new integration tests: + +1. Create a new test file in `src/test/suite/` with the `.test.ts` extension + +2. Add the test file to the `files` array in `suite/index.ts` (you can temporarily comment out the other tests to run just the new test): + +```typescript +const files = ["suite/modes.test.js", "suite/tasks.test.js", "suite/extension.test.js", "suite/your-new-test.test.js"] +``` + +3. Structure your tests using the TDD interface: + +```typescript +import * as assert from "assert" +import * as vscode from "vscode" + +suite("Your Test Suite Name", () => { + test("Should do something specific", async function () { + // Your test code here + }) +}) +``` + +4. Use the global objects (`api`, `provider`, `extension`, `panel`) to interact with the extension + +### Best Practices + +1. **Timeouts**: Use appropriate timeouts for async operations: + +```typescript +const timeout = 30000 +const interval = 1000 +``` + +2. **State Management**: Reset extension state before/after tests: + +```typescript +await globalThis.provider.updateGlobalState("mode", "Ask") +await globalThis.provider.updateGlobalState("alwaysAllowModeSwitch", true) +``` + +3. **Assertions**: Use clear assertions with meaningful messages: + +```typescript +assert.ok(condition, "Descriptive message about what failed") +``` + +4. **Error Handling**: Wrap test code in try/catch blocks and clean up resources: + +```typescript +try { + // Test code +} finally { + // Cleanup code +} +``` + +5. **Wait for Operations**: Use polling when waiting for async operations: + +```typescript +let startTime = Date.now() +while (Date.now() - startTime < timeout) { + if (condition) break + await new Promise((resolve) => setTimeout(resolve, interval)) +} +``` + +6. **Grading**: When grading tests, use the `Grade:` format to ensure the test is graded correctly (See modes.test.ts for an example). + +```typescript +await globalThis.api.startNewTask( + `Given this prompt: ${testPrompt} grade the response from 1 to 10 in the format of "Grade: (1-10)": ${output} \n Be sure to say 'I AM DONE GRADING' after the task is complete`, +) +``` diff --git a/src/test/suite/index.ts b/src/test/suite/index.ts index ffb8de7473..c538541a4b 100644 --- a/src/test/suite/index.ts +++ b/src/test/suite/index.ts @@ -23,7 +23,8 @@ export async function run(): Promise { try { // Find all test files - const files = await glob("**/**.test.js", { cwd: testsRoot }) + //const files = await glob("**/**.test.js", { cwd: testsRoot } leaving this commented out for now since we only have three tests + const files = ["suite/modes.test.js", "suite/tasks.test.js", "suite/extension.test.js"] // Add files to the test suite files.forEach((f: string) => mocha.addFile(path.resolve(testsRoot, f))) diff --git a/src/test/suite/modes.test.ts b/src/test/suite/modes.test.ts index edd1a6a4ae..6f20ee3b9e 100644 --- a/src/test/suite/modes.test.ts +++ b/src/test/suite/modes.test.ts @@ -5,7 +5,8 @@ suite("Roo Code Modes", () => { test("Should handle switching modes correctly", async function () { const timeout = 30000 const interval = 1000 - + const testPrompt = + "For each mode (Code, Architect, Ask) respond with the mode name and what it specializes in after switching to that mode, do not start with the current mode, be sure to say 'I AM DONE' after the task is complete" if (!globalThis.extension) { assert.fail("Extension not found") } @@ -27,9 +28,7 @@ suite("Roo Code Modes", () => { await globalThis.provider.updateGlobalState("autoApprovalEnabled", true) // Start a new task. - await globalThis.api.startNewTask( - "For each mode (Code, Architect, Ask) respond with the mode name and what it specializes in after switching to that mode, do not start with the current mode, be sure to say 'I AM DONE' after the task is complete", - ) + await globalThis.api.startNewTask(testPrompt) // Wait for task to appear in history with tokens. startTime = Date.now() @@ -52,46 +51,46 @@ suite("Roo Code Modes", () => { assert.fail("No messages received") } - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => type === "say" && text?.includes(`"request":"[switch_mode to 'code' because:`), - ), - "Did not receive expected response containing 'Roo wants to switch to code mode'", - ) - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => type === "say" && text?.includes("software engineer"), - ), - "Did not receive expected response containing 'I am Roo in Code mode, specializing in software engineering'", + await globalThis.provider.updateGlobalState("mode", "Ask") + let output = globalThis.provider.messages.map(({ type, text }) => (type === "say" ? text : "")).join("\n") + await globalThis.api.startNewTask( + `Given this prompt: ${testPrompt} grade the response from 1 to 10 in the format of "Grade: (1-10)": ${output} \n Be sure to say 'I AM DONE GRADING' after the task is complete`, ) - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => - type === "say" && text?.includes(`"request":"[switch_mode to 'architect' because:`), - ), - "Did not receive expected response containing 'Roo wants to switch to architect mode'", - ) - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => - type === "say" && (text?.includes("technical planning") || text?.includes("technical leader")), - ), - "Did not receive expected response containing 'I am Roo in Architect mode, specializing in analyzing codebases'", - ) + startTime = Date.now() + while (Date.now() - startTime < timeout) { + const messages = globalThis.provider.messages + + if ( + messages.some( + ({ type, text }) => + type === "say" && text?.includes("I AM DONE GRADING") && !text?.includes("be sure to say"), + ) + ) { + break + } + + await new Promise((resolve) => setTimeout(resolve, interval)) + } + if (globalThis.provider.messages.length === 0) { + assert.fail("No messages received") + } + globalThis.provider.messages.forEach(({ type, text }) => { + if (type === "say" && text?.includes("Grade:")) { + console.log(text) + } + }) + const grade = globalThis.provider.messages.find( + ({ type, text }) => type === "say" && !text?.includes("Grade: (1-10)") && text?.includes("Grade:"), + )?.text + console.log("THIS IS THE GRADE", grade) assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => type === "say" && text?.includes(`"request":"[switch_mode to 'ask' because:`), - ), - "Did not receive expected response containing 'Roo wants to switch to ask mode'", - ) - assert.ok( - globalThis.provider.messages.some( - ({ type, text }) => - type === "say" && (text?.includes("technical knowledge") || text?.includes("technical assist")), - ), - "Did not receive expected response containing 'I am Roo in Ask mode, specializing in answering questions'", + grade?.includes("Grade: 10") || + grade?.includes("Grade: 9") || + grade?.includes("Grade: 8") || + grade?.includes("Grade: 7"), + "Did not receive expected response containing 'Grade: 10' or 'Grade: 9' or 'Grade: 8' or 'Grade: 7'", ) } finally { }