Merge pull request #689 from RooVetGit/cte/benchmarks

Aider-inspired polyglot benchmarks

.changeset/little-parents-shake.md

@@ -0,0 +1,5 @@
+---
+"roo-cline": patch
+---
+
+Aider-inspired polyglot benchmarks

.dockerignore

@@ -0,0 +1,45 @@
+# Version control
+# .git/
+# .gitignore
+# .gitattributes
+# .git-blame-ignore-revs
+# .gitconfig
+
+# Build artifacts
+bin/
+dist/
+**/dist/
+out/
+**/out/
+
+# Dependencies
+node_modules/
+**/node_modules/
+
+# Test and development files
+coverage/
+**/.vscode-test/
+
+# Configuration files
+# .env*
+knip.json
+.husky/
+
+# CI/CD
+# .changeset/
+# .github/
+# ellipsis.yaml
+
+# OS specific
+.DS_Store
+
+# Logs
+logs/
+*.log
+
+# Nix
+# flake.lock
+# flake.nix
+
+# Monorepo
+benchmark/exercises/

.vscodeignore

@@ -5,6 +5,7 @@
 .vscode-test/**
 out/**
 out-integration/**
+benchmark/**
 e2e/**
 node_modules/**
 src/**

benchmark/.env.local.sample

@@ -0,0 +1,2 @@
+OPENROUTER_API_KEY=sk-or-v1-...
+POSTHOG_API_KEY=phc_...

benchmark/Dockerfile

@@ -0,0 +1,89 @@
+# docker build -f Dockerfile.base -t roo-code-benchmark-base ..
+# docker build -f Dockerfile -t roo-code-benchmark ..
+# docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark
+# docker exec -it $(docker ps --filter "ancestor=roo-code-benchmark" -q) /bin/bash
+
+FROM ubuntu:latest
+
+# Install dependencies
+RUN apt update && apt install -y sudo curl git vim jq
+
+# Create a `vscode` user
+RUN useradd -m vscode -s /bin/bash && \
+  echo "vscode ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/vscode && \
+  chmod 0440 /etc/sudoers.d/vscode
+
+# Install VS Code
+# https://code.visualstudio.com/docs/setup/linux
+RUN apt install -y wget gpg apt-transport-https
+RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg
+RUN install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg
+RUN echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" | tee /etc/apt/sources.list.d/vscode.list > /dev/null
+RUN rm -f packages.microsoft.gpg
+RUN apt update && apt install -y code
+
+# Install Xvfb
+RUN apt install -y xvfb
+
+# [cpp] Install cmake 3.28.3
+RUN apt install -y cmake
+
+# [go] Install Go 1.22.2
+RUN apt install -y golang-go
+
+# [java] Install Java 21
+RUN apt install -y default-jre
+
+# [javascript] Install Node.js v18.20.6
+RUN curl -sL https://deb.nodesource.com/setup_18.x | bash -
+RUN apt update && apt install -y nodejs
+RUN npm install -g corepack@latest
+
+# [python] Install Python 3.12.3 and uv 0.6.6
+RUN apt install -y python3 python3-venv python3-dev python3-pip
+
+# [rust] Install Rust 1.85
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc
+
+WORKDIR /home/vscode
+USER vscode
+
+# Enable corepack and install pnpm for the vscode user
+RUN corepack enable
+RUN yes y | pnpm --version
+
+COPY benchmark/entrypoint.sh /usr/local/bin/entrypoint.sh
+
+# Copy and build dependencies
+COPY --chown=vscode:vscode package*.json /home/vscode/repo/
+COPY --chown=vscode:vscode webview-ui/package*.json /home/vscode/repo/webview-ui/
+COPY --chown=vscode:vscode e2e/package*.json /home/vscode/repo/e2e/
+COPY --chown=vscode:vscode benchmark/package*.json /home/vscode/repo/benchmark/
+WORKDIR /home/vscode/repo
+RUN npm run install:all
+
+# Copy and build benchmark runner
+COPY --chown=vscode:vscode . /home/vscode/repo
+WORKDIR /home/vscode/repo/benchmark
+RUN npm run build
+
+# Copy exercises
+WORKDIR /home/vscode
+RUN git clone https://github.com/cte/Roo-Code-Benchmark.git exercises
+
+# Prepare exercises
+WORKDIR /home/vscode/exercises/python
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+RUN /home/vscode/.local/bin/uv sync
+
+# Build web-ui
+WORKDIR /home/vscode/exercises/web-ui
+RUN echo "DB_FILE_NAME=file:/tmp/benchmarks.db" > .env
+RUN pnpm install
+RUN npx drizzle-kit push
+
+# Run web-ui
+EXPOSE 3000
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+CMD ["/usr/bin/pnpm", "dev"]

benchmark/README.md

@@ -0,0 +1,51 @@
+# Benchmark Harness
+
+Configure ENV vars (OpenRouter, PostHog, etc):
+
+```sh
+cp .env.local.sample .env.local
+# Update ENV vars as needed.
+```
+
+Build and run a Docker image with the development environment needed to run the
+benchmarks (C++, Go, Java, Node.js, Python & Rust):
+
+```sh
+npm run docker:start
+```
+
+Run an exercise:
+
+```sh
+npm run docker:benchmark -- -e exercises/javascript/binary
+```
+
+Select and run an exercise:
+
+```sh
+npm run cli
+```
+
+Select and run an exercise for a specific language:
+
+```sh
+npm run cli -- run rust
+```
+
+Run all exercises for a language:
+
+```sh
+npm run cli -- run rust all
+```
+
+Run all exercises:
+
+```sh
+npm run cli -- run all
+```
+
+Run all exercises using a specific runId (useful for re-trying when an unexpected error occurs):
+
+```sh
+npm run cli -- run all --runId 1
+```

benchmark/entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+npx drizzle-kit push
+exec "$@"

benchmark/package.json

@@ -0,0 +1,30 @@
+{
+	"name": "benchmark",
+	"version": "0.1.0",
+	"private": true,
+	"main": "out/run.js",
+	"scripts": {
+		"build": "npm run compile && cd .. && npm run compile && npm run build:webview",
+		"lint": "eslint src --ext ts",
+		"check-types": "tsc --noEmit",
+		"compile": "rm -rf out && tsc -p tsconfig.json",
+		"cli": "npm run compile && npx dotenvx run -f .env.local -- tsx src/cli.ts",
+		"clean": "rimraf out",
+		"clean:exercises": "cd exercises && git checkout -f && git clean -fd",
+		"docker:build": "docker build -f Dockerfile -t roo-code-benchmark ..",
+		"docker:run": "touch /tmp/benchmarks.db && docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark",
+		"docker:start": "npm run docker:build && npm run docker:run",
+		"docker:shell": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) /bin/bash",
+		"docker:cli": "docker exec -it -w /home/vscode/repo/benchmark $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) xvfb-run npm run cli --",
+		"docker:stop": "docker stop $(docker ps --filter \"ancestor=roo-code-benchmark\" -q)",
+		"docker:rm": "docker rm $(docker ps -a --filter \"ancestor=roo-code-benchmark\" -q)",
+		"docker:clean": "npm run docker:stop && npm run docker:rm"
+	},
+	"devDependencies": {
+		"@vscode/test-electron": "^2.4.0",
+		"gluegun": "^5.1.2",
+		"tsx": "^4.19.3",
+		"typescript": "^5.4.5",
+		"yargs": "^17.7.2"
+	}
+}

benchmark/prompts/cpp.md

@@ -0,0 +1,17 @@
+Your job is to complete a coding exercise described by `.docs/instructions.md`.
+
+A file with the implementation stubbed out has been created for you, along with a test file.
+
+To successfully complete the exercise, you must pass all the tests in the test file.
+
+To confirm that your solution is correct, you can compile your code and run the tests with:
+
+```
+mkdir -p build && cd build
+cmake -G "Unix Makefiles" -DEXERCISM_RUN_ALL_TESTS=1 ..
+make
+```
+
+Note that running `make` will compile the tests and generate compile time errors. Once the errors are fixed, running `make` will build and run the tests.
+
+Do not alter the test file; it should be run as-is.

benchmark/prompts/go.md

@@ -0,0 +1,7 @@
+Your job is to complete a coding exercise described by `.docs/instructions.md`.
+
+A file with the implementation stubbed out has been created for you, along with a test file.
+
+To successfully complete the exercise, you must pass all the tests in the test file.
+
+To confirm that your solution is correct, run the tests with `go test`. Do not alter the test file; it should be run as-is.

benchmark/prompts/java.md

@@ -0,0 +1,7 @@
+Your job is to complete a coding exercise described by `.docs/instructions.md`.
+
+A file with the implementation stubbed out has been created for you, along with a test file.
+
+To successfully complete the exercise, you must pass all the tests in the test file.
+
+To confirm that your solution is correct, run the tests with `./gradlew test`. Do not alter the test file; it should be run as-is.

benchmark/prompts/javascript.md

@@ -0,0 +1,9 @@
+Your job is to complete a coding exercise described by `.docs/instructions.md`.
+
+A file with the implementation stubbed out has been created for you, along with a test file.
+
+To successfully complete the exercise, you must pass all the tests in the test file.
+
+To confirm that your solution is correct, run the tests with `pnpm test`. Do not alter the test file; it should be run as-is.
+
+Before running the tests make sure your environment is set up by running `pnpm install` to install the dependencies.

benchmark/prompts/python.md

@@ -0,0 +1,7 @@
+Your job is to complete a coding exercise described by `.docs/instructions.md`.
+
+A file with the implementation stubbed out has been created for you, along with a test file.
+
+To successfully complete the exercise, you must pass all the tests in the test file.
+
+To confirm that your solution is correct, run the tests with `uv run python3 -m pytest -o markers=task [name]_test.py`. Do not alter the test file; it should be run as-is.

benchmark/prompts/rust.md

@@ -0,0 +1,7 @@
+Your job is to complete a coding exercise described by `.docs/instructions.md`.
+
+A file with the implementation stubbed out has been created for you, along with a test file.
+
+To successfully complete the exercise, you must pass all the tests in the test file.
+
+To confirm that your solution is correct, run the tests with `cargo test`. Do not alter the test file; it should be run as-is.

benchmark/src/cli.ts

@@ -0,0 +1,171 @@
+import * as fs from "fs"
+import * as path from "path"
+
+import { build, filesystem, GluegunPrompt } from "gluegun"
+import { runTests } from "@vscode/test-electron"
+
+// console.log(__dirname)
+// <...>/Roo-Code/benchmark/src
+
+const extensionDevelopmentPath = path.resolve(__dirname, "../../")
+const extensionTestsPath = path.resolve(__dirname, "../out/runExercise")
+const promptsPath = path.resolve(__dirname, "../prompts")
+const exercisesPath = path.resolve(__dirname, "../../../exercises")
+const languages = ["cpp", "go", "java", "javascript", "python", "rust"]
+
+async function runAll({ runId, model }: { runId: number; model: string }) {
+	for (const language of languages) {
+		await runLanguage({ runId, model, language })
+	}
+}
+
+async function runLanguage({ runId, model, language }: { runId: number; model: string; language: string }) {
+	const languagePath = path.resolve(exercisesPath, language)
+
+	if (!fs.existsSync(languagePath)) {
+		console.error(`Language directory ${languagePath} does not exist`)
+		process.exit(1)
+	}
+
+	const exercises = filesystem
+		.subdirectories(languagePath)
+		.map((exercise) => path.basename(exercise))
+		.filter((exercise) => !exercise.startsWith("."))
+
+	for (const exercise of exercises) {
+		await runExercise({ runId, model, language, exercise })
+	}
+}
+
+async function runExercise({
+	runId,
+	model,
+	language,
+	exercise,
+}: {
+	runId: number
+	model: string
+	language: string
+	exercise: string
+}) {
+	const workspacePath = path.resolve(exercisesPath, language, exercise)
+	const promptPath = path.resolve(promptsPath, `${language}.md`)
+
+	const extensionTestsEnv = {
+		PROMPT_PATH: promptPath,
+		WORKSPACE_PATH: workspacePath,
+		OPENROUTER_MODEL_ID: model,
+		RUN_ID: runId.toString(),
+	}
+
+	if (fs.existsSync(path.resolve(workspacePath, "usage.json"))) {
+		console.log(`Test result exists for ${language} / ${exercise}, skipping`)
+		return
+	}
+
+	console.log(`Running ${language} / ${exercise}`)
+
+	await runTests({
+		extensionDevelopmentPath,
+		extensionTestsPath,
+		launchArgs: [workspacePath, "--disable-extensions"],
+		extensionTestsEnv,
+	})
+}
+
+async function askLanguage(prompt: GluegunPrompt) {
+	const languages = filesystem.subdirectories(exercisesPath)
+
+	if (languages.length === 0) {
+		throw new Error(`No languages found in ${exercisesPath}`)
+	}
+
+	const { language } = await prompt.ask<{ language: string }>({
+		type: "select",
+		name: "language",
+		message: "Which language?",
+		choices: languages.map((language) => path.basename(language)).filter((language) => !language.startsWith(".")),
+	})
+
+	return language
+}
+
+async function askExercise(prompt: GluegunPrompt, language: string) {
+	const exercises = filesystem.subdirectories(path.join(exercisesPath, language))
+
+	if (exercises.length === 0) {
+		throw new Error(`No exercises found for ${language}`)
+	}
+
+	const { exercise } = await prompt.ask<{ exercise: string }>({
+		type: "select",
+		name: "exercise",
+		message: "Which exercise?",
+		choices: exercises.map((exercise) => path.basename(exercise)),
+	})
+
+	return exercise
+}
+
+async function createRun({ model }: { model: string }): Promise<{ id: number; model: string }> {
+	const response = await fetch("http://localhost:3000/api/runs", {
+		method: "POST",
+		body: JSON.stringify({ model }),
+	})
+
+	if (!response.ok) {
+		throw new Error(`Failed to create run: ${response.statusText}`)
+	}
+
+	const {
+		run: [run],
+	} = await response.json()
+	return run
+}
+
+async function main() {
+	const cli = build()
+		.brand("benchmark-runner")
+		.src(__dirname)
+		.help()
+		.version()
+		.command({
+			name: "run",
+			run: ({ config, parameters }) => {
+				config.language = parameters.first
+				config.exercise = parameters.second
+
+				if (parameters.options["runId"]) {
+					config.runId = parameters.options["runId"]
+				}
+			},
+		})
+		.defaultCommand() // Use the default command if no args.
+		.create()
+
+	const { print, prompt, config } = await cli.run(process.argv)
+
+	try {
+		const model = "anthropic/claude-3.7-sonnet"
+		const runId = config.runId ? Number(config.runId) : (await createRun({ model })).id
+
+		if (config.language === "all") {
+			console.log("Running all exercises for all languages")
+			await runAll({ runId, model })
+		} else if (config.exercise === "all") {
+			console.log(`Running all exercises for ${config.language}`)
+			await runLanguage({ runId, model, language: config.language })
+		} else {
+			const language = config.language || (await askLanguage(prompt))
+			const exercise = config.exercise || (await askExercise(prompt, language))
+			await runExercise({ runId, model, language, exercise })
+		}
+
+		process.exit(0)
+	} catch (error) {
+		print.error(error)
+		process.exit(1)
+	}
+}
+
+main()

benchmark/src/runExercise.ts

@@ -0,0 +1,94 @@
+import * as fs from "fs/promises"
+import * as path from "path"
+
+import * as vscode from "vscode"
+
+import { RooCodeAPI, TokenUsage } from "../../src/exports/roo-code"
+
+import { waitUntilReady, waitUntilCompleted, sleep } from "./utils"
+
+export async function run() {
+	/**
+	 * Validate environment variables.
+	 */
+
+	const runId = process.env.RUN_ID
+	const openRouterApiKey = process.env.OPENROUTER_API_KEY
+	const openRouterModelId = process.env.OPENROUTER_MODEL_ID
+	const promptPath = process.env.PROMPT_PATH
+	const workspacePath = process.env.WORKSPACE_PATH
+
+	if (!runId || !openRouterApiKey || !openRouterModelId || !promptPath || !workspacePath) {
+		throw new Error("ENV not configured.")
+	}
+
+	const prompt = await fs.readFile(promptPath, "utf-8")
+
+	/**
+	 * Activate the extension.
+	 */
+
+	const extension = vscode.extensions.getExtension<RooCodeAPI>("RooVeterinaryInc.roo-cline")
+
+	if (!extension) {
+		throw new Error("Extension not found.")
+	}
+
+	const api = extension.isActive ? extension.exports : await extension.activate()
+
+	/**
+	 * Wait for the Roo Code to be ready to accept tasks.
+	 */
+
+	await waitUntilReady({ api })
+
+	/**
+	 * Configure Roo Code as needed.
+	 *
+	 * Use Claude 3.7 Sonnet via OpenRouter.
+	 * Don't require approval for anything.
+	 * Run any command without approval.
+	 * Disable checkpoints (for performance).
+	 */
+
+	await api.setConfiguration({
+		apiProvider: "openrouter",
+		openRouterApiKey,
+		openRouterModelId,
+		autoApprovalEnabled: true,
+		alwaysAllowReadOnly: true,
+		alwaysAllowWrite: true,
+		alwaysAllowExecute: true,
+		alwaysAllowBrowser: true,
+		alwaysApproveResubmit: true,
+		alwaysAllowMcp: true,
+		alwaysAllowModeSwitch: true,
+		enableCheckpoints: false,
+	})
+
+	await vscode.workspace
+		.getConfiguration("roo-cline")
+		.update("allowedCommands", ["*"], vscode.ConfigurationTarget.Global)
+
+	await sleep(2_000)
+
+	/**
+	 * Run the task and wait up to 10 minutes for it to complete.
+	 */
+
+	const startTime = Date.now()
+	const taskId = await api.startNewTask(prompt)
+
+	let usage: TokenUsage | undefined = undefined
+
+	try {
+		usage = await waitUntilCompleted({ api, taskId, timeout: 5 * 60 * 1_000 }) // 5m
+	} catch (e) {
+		usage = api.getTokenUsage(taskId)
+	}
+
+	if (usage) {
+		const content = JSON.stringify({ runId: parseInt(runId), ...usage, duration: Date.now() - startTime }, null, 2)
+		await fs.writeFile(path.resolve(workspacePath, "usage.json"), content)
+	}
+}

benchmark/src/utils.ts

@@ -0,0 +1,111 @@
+import * as vscode from "vscode"
+
+import { RooCodeAPI, TokenUsage } from "../../src/exports/roo-code"
+
+type WaitForOptions = {
+	timeout?: number
+	interval?: number
+}
+
+export const waitFor = (
+	condition: (() => Promise<boolean>) | (() => boolean),
+	{ timeout = 30_000, interval = 250 }: WaitForOptions = {},
+) => {
+	let timeoutId: NodeJS.Timeout | undefined = undefined
+
+	return Promise.race([
+		new Promise<void>((resolve) => {
+			const check = async () => {
+				const result = condition()
+				const isSatisfied = result instanceof Promise ? await result : result
+
+				if (isSatisfied) {
+					if (timeoutId) {
+						clearTimeout(timeoutId)
+						timeoutId = undefined
+					}
+
+					resolve()
+				} else {
+					setTimeout(check, interval)
+				}
+			}
+
+			check()
+		}),
+		new Promise((_, reject) => {
+			timeoutId = setTimeout(() => {
+				reject(new Error(`Timeout after ${Math.floor(timeout / 1000)}s`))
+			}, timeout)
+		}),
+	])
+}
+
+type WaitUntilReadyOptions = WaitForOptions & {
+	api: RooCodeAPI
+}
+
+export const waitUntilReady = async ({ api, ...options }: WaitUntilReadyOptions) => {
+	await vscode.commands.executeCommand("roo-cline.SidebarProvider.focus")
+	await waitFor(() => api.isReady(), options)
+}
+
+type WaitUntilAbortedOptions = WaitForOptions & {
+	api: RooCodeAPI
+	taskId: string
+}
+
+export const waitUntilAborted = async ({ api, taskId, ...options }: WaitUntilAbortedOptions) => {
+	const set = new Set<string>()
+	api.on("taskAborted", (taskId) => set.add(taskId))
+	await waitFor(() => set.has(taskId), options)
+}
+
+type WaitUntilCompletedOptions = WaitForOptions & {
+	api: RooCodeAPI
+	taskId: string
+}
+
+export const waitUntilCompleted = async ({ api, taskId, ...options }: WaitUntilCompletedOptions) => {
+	const map = new Map<string, TokenUsage>()
+	api.on("taskCompleted", (taskId, usage) => map.set(taskId, usage))
+	await waitFor(() => map.has(taskId), options)
+	return map.get(taskId)
+}
+
+export const waitForCompletion = async ({
+	api,
+	taskId,
+	...options
+}: WaitUntilReadyOptions & {
+	taskId: string
+}) => waitFor(() => !!getCompletion({ api, taskId }), options)
+
+export const getCompletion = ({ api, taskId }: { api: RooCodeAPI; taskId: string }) =>
+	api.getMessages(taskId).find(({ say, partial }) => say === "completion_result" && partial === false)
+
+type WaitForMessageOptions = WaitUntilReadyOptions & {
+	taskId: string
+	include: string
+	exclude?: string
+}
+
+export const waitForMessage = async ({ api, taskId, include, exclude, ...options }: WaitForMessageOptions) =>
+	waitFor(() => !!getMessage({ api, taskId, include, exclude }), options)
+
+type GetMessageOptions = {
+	api: RooCodeAPI
+	taskId: string
+	include: string
+	exclude?: string
+}
+
+export const getMessage = ({ api, taskId, include, exclude }: GetMessageOptions) =>
+	api
+		.getMessages(taskId)
+		.find(
+			({ type, text }) =>
+				type === "say" && text && text.includes(include) && (!exclude || !text.includes(exclude)),
+		)
+
+export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))

benchmark/tsconfig.json

@@ -0,0 +1,16 @@
+{
+	"compilerOptions": {
+		"module": "CommonJS",
+		"moduleResolution": "Node",
+		"esModuleInterop": true,
+		"target": "ES2022",
+		"lib": ["ES2022", "ESNext.Disposable", "DOM"],
+		"sourceMap": true,
+		"strict": true,
+		"skipLibCheck": true,
+		"useUnknownInCatchVariables": false,
+		"outDir": "out"
+	},
+	"include": ["src", "../src/exports/roo-code.d.ts"],
+	"exclude": ["**/node_modules/**", "out"]
+}

knip.json

@@ -14,6 +14,7 @@
 		"out/**",
 		"bin/**",
 		"e2e/**",
+		"benchmark/**",
 		"src/activate/**",
 		"src/exports/**",
 		"src/extension.ts",

package.json

@@ -280,16 +280,19 @@
 		"install:all": "npm install npm-run-all && npm run install:_all",
 		"install:_all": "npm-run-all -p install-*",
 		"install-extension": "npm install",
-		"install-webview-ui": "cd webview-ui && npm install",
+		"install-webview": "cd webview-ui && npm install",
 		"install-e2e": "cd e2e && npm install",
+		"install-benchmark": "cd benchmark && npm install",
 		"lint": "npm-run-all -p lint:*",
 		"lint:extension": "eslint src --ext ts",
-		"lint:webview-ui": "cd webview-ui && npm run lint",
+		"lint:webview": "cd webview-ui && npm run lint",
 		"lint:e2e": "cd e2e && npm run lint",
+		"lint:benchmark": "cd benchmark && npm run lint",
 		"check-types": "npm-run-all -p check-types:*",
 		"check-types:extension": "tsc --noEmit",
-		"check-types:webview-ui": "cd webview-ui && npm run check-types",
+		"check-types:webview": "cd webview-ui && npm run check-types",
 		"check-types:e2e": "cd e2e && npm run check-types",
+		"check-types:benchmark": "cd benchmark && npm run check-types",
 		"package": "npm run build:webview && npm run check-types && npm run lint && node esbuild.js --production",
 		"pretest": "npm run compile",
 		"dev": "cd webview-ui && npm run dev",
@@ -308,6 +311,11 @@
 		"watch-tests": "tsc -p . -w --outDir out",
 		"changeset": "changeset",
 		"knip": "knip --include files",
+		"clean": "npm-run-all -p clean:*",
+		"clean:extension": "rimraf bin dist out",
+		"clean:webview": "cd webview-ui && npm run clean",
+		"clean:e2e": "cd e2e && npm run clean",
+		"clean:benchmark": "cd benchmark && npm run clean",
 		"update-contributors": "node scripts/update-contributors.js"
 	},
 	"dependencies": {

src/api/providers/openrouter.ts

@@ -177,7 +177,7 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 
 		if (lastError) {
 			console.error(
-				`Failed to fetch OpenRouter generation details after ${Date.now() - startTime}ms (${genId})`,
+				`Failed to fetch OpenRouter generation details after attempt #${attempt} (${Date.now() - startTime}ms) [${genId}]`,
 				lastError,
 			)
 		}

src/core/Cline.ts

@@ -12,6 +12,7 @@ import getFolderSize from "get-folder-size"
 import { serializeError } from "serialize-error"
 import * as vscode from "vscode"
 
+import { TokenUsage } from "../exports/roo-code"
 import { ApiHandler, buildApiHandler } from "../api"
 import { ApiStream } from "../api/transform/stream"
 import { DIFF_VIEW_URI_SCHEME, DiffViewProvider } from "../integrations/editor/DiffViewProvider"
@@ -92,6 +93,8 @@ export type ClineEvents = {
 	taskAskResponded: []
 	taskAborted: []
 	taskSpawned: [taskId: string]
+	taskCompleted: [taskId: string, usage: TokenUsage]
+	taskTokenUsageUpdated: [taskId: string, usage: TokenUsage]
 }
 
 export type ClineOptions = {
@@ -351,13 +354,19 @@ export class Cline extends EventEmitter<ClineEvents> {
 		this.emit("message", { action: "updated", message: partialMessage })
 	}
 
+	private getTokenUsage() {
+		const usage = getApiMetrics(combineApiRequests(combineCommandSequences(this.clineMessages.slice(1))))
+		this.emit("taskTokenUsageUpdated", this.taskId, usage)
+		return usage
+	}
+
 	private async saveClineMessages() {
 		try {
 			const taskDir = await this.ensureTaskDirectoryExists()
 			const filePath = path.join(taskDir, GlobalFileNames.uiMessages)
 			await fs.writeFile(filePath, JSON.stringify(this.clineMessages))
 			// combined as they are in ChatView
-			const apiMetrics = getApiMetrics(combineApiRequests(combineCommandSequences(this.clineMessages.slice(1))))
+			const apiMetrics = this.getTokenUsage()
 			const taskMessage = this.clineMessages[0] // first message is always the task say
 			const lastRelevantMessage =
 				this.clineMessages[
@@ -2925,26 +2934,6 @@ export class Cline extends EventEmitter<ClineEvents> {
 					}
 
 					case "attempt_completion": {
-						/*
-						this.consecutiveMistakeCount = 0
-						let resultToSend = result
-						if (command) {
-							await this.say("completion_result", resultToSend)
-							// TODO: currently we don't handle if this command fails, it could be useful to let cline know and retry
-							const [didUserReject, commandResult] = await this.executeCommand(command, true)
-							// if we received non-empty string, the command was rejected or failed
-							if (commandResult) {
-								return [didUserReject, commandResult]
-							}
-							resultToSend = ""
-						}
-						const { response, text, images } = await this.ask("completion_result", resultToSend) // this prompts webview to show 'new task' button, and enable text input (which would be the 'text' here)
-						if (response === "yesButtonClicked") {
-							return [false, ""] // signals to recursive loop to stop (for now this never happens since yesButtonClicked will trigger a new task)
-						}
-						await this.say("user_feedback", text ?? "", images)
-						return [
-						*/
 						const result: string | undefined = block.params.result
 						const command: string | undefined = block.params.command
 						try {
@@ -2996,34 +2985,42 @@ export class Cline extends EventEmitter<ClineEvents> {
 									)
 									break
 								}
+
 								this.consecutiveMistakeCount = 0
 
 								let commandResult: ToolResponse | undefined
+
 								if (command) {
 									if (lastMessage && lastMessage.ask !== "command") {
-										// havent sent a command message yet so first send completion_result then command
+										// Haven't sent a command message yet so
+										// first send completion_result then command.
 										await this.say("completion_result", result, undefined, false)
-										telemetryService.captureTaskCompleted(this.taskId)
 									}
 
-									// complete command message
+									// Complete command message.
 									const didApprove = await askApproval("command", command)
+
 									if (!didApprove) {
 										break
 									}
+
 									const [userRejected, execCommandResult] = await this.executeCommandTool(command!)
+
 									if (userRejected) {
 										this.didRejectTool = true
 										pushToolResult(execCommandResult)
 										break
 									}
-									// user didn't reject, but the command may have output
+
+									// User didn't reject, but the command may have output.
 									commandResult = execCommandResult
 								} else {
 									await this.say("completion_result", result, undefined, false)
-									telemetryService.captureTaskCompleted(this.taskId)
 								}
 
+								telemetryService.captureTaskCompleted(this.taskId)
+								this.emit("taskCompleted", this.taskId, this.getTokenUsage())
+
 								if (this.parentTask) {
 									const didApprove = await askFinishSubTaskApproval()
 
@@ -3036,15 +3033,22 @@ export class Cline extends EventEmitter<ClineEvents> {
 									break
 								}
 
-								// we already sent completion_result says, an empty string asks relinquishes control over button and field
+								// We already sent completion_result says, an
+								// empty string asks relinquishes control over
+								// button and field.
 								const { response, text, images } = await this.ask("completion_result", "", false)
+
+								// Signals to recursive loop to stop (for now
+								// this never happens since yesButtonClicked
+								// will trigger a new task).
 								if (response === "yesButtonClicked") {
-									pushToolResult("") // signals to recursive loop to stop (for now this never happens since yesButtonClicked will trigger a new task)
+									pushToolResult("")
 									break
 								}
-								await this.say("user_feedback", text ?? "", images)
 
+								await this.say("user_feedback", text ?? "", images)
 								const toolResults: (Anthropic.TextBlockParam | Anthropic.ImageBlockParam)[] = []
+
 								if (commandResult) {
 									if (typeof commandResult === "string") {
 										toolResults.push({ type: "text", text: commandResult })
@@ -3052,17 +3056,20 @@ export class Cline extends EventEmitter<ClineEvents> {
 										toolResults.push(...commandResult)
 									}
 								}
+
 								toolResults.push({
 									type: "text",
 									text: `The user has provided feedback on the results. Consider their input to continue the task, and then attempt completion again.\n<feedback>\n${text}\n</feedback>`,
 								})
+
 								toolResults.push(...formatResponse.imageBlocks(images))
+
 								this.userMessageContent.push({
 									type: "text",
 									text: `${toolDescription()} Result:`,
 								})
-								this.userMessageContent.push(...toolResults)
 
+								this.userMessageContent.push(...toolResults)
 								break
 							}
 						} catch (error) {
@@ -3071,6 +3078,7 @@ export class Cline extends EventEmitter<ClineEvents> {
 						}
 					}
 				}
+
 				break
 		}
 

src/exports/api.ts

@@ -3,13 +3,14 @@ import * as vscode from "vscode"
 
 import { ClineProvider } from "../core/webview/ClineProvider"
 
-import { RooCodeAPI, RooCodeEvents, ConfigurationValues } from "./roo-code"
+import { RooCodeAPI, RooCodeEvents, ConfigurationValues, TokenUsage } from "./roo-code"
 import { MessageHistory } from "./message-history"
 
 export class API extends EventEmitter<RooCodeEvents> implements RooCodeAPI {
 	private readonly outputChannel: vscode.OutputChannel
 	private readonly provider: ClineProvider
 	private readonly history: MessageHistory
+	private readonly tokenUsage: Record<string, TokenUsage>
 
 	constructor(outputChannel: vscode.OutputChannel, provider: ClineProvider) {
 		super()
@@ -17,6 +18,7 @@ export class API extends EventEmitter<RooCodeEvents> implements RooCodeAPI {
 		this.outputChannel = outputChannel
 		this.provider = provider
 		this.history = new MessageHistory()
+		this.tokenUsage = {}
 
 		this.provider.on("clineAdded", (cline) => {
 			cline.on("message", (message) => this.emit("message", { taskId: cline.taskId, ...message }))
@@ -39,6 +41,8 @@ export class API extends EventEmitter<RooCodeEvents> implements RooCodeAPI {
 				this.history.update(taskId, message)
 			}
 		})
+
+		this.on("taskTokenUsageUpdated", (taskId, usage) => (this.tokenUsage[taskId] = usage))
 	}
 
 	public async startNewTask(text?: string, images?: string[]) {
@@ -51,6 +55,10 @@ export class API extends EventEmitter<RooCodeEvents> implements RooCodeAPI {
 		return cline.taskId
 	}
 
+	public getCurrentTaskStack() {
+		return this.provider.getCurrentTaskStack()
+	}
+
 	public async clearCurrentTask(lastMessage?: string) {
 		await this.provider.finishSubTask(lastMessage)
 	}
@@ -84,7 +92,11 @@ export class API extends EventEmitter<RooCodeEvents> implements RooCodeAPI {
 		return this.history.getMessages(taskId)
 	}
 
-	public getCurrentTaskStack(): string[] {
-		return this.provider.getCurrentTaskStack()
+	public getTokenUsage(taskId: string) {
+		return this.tokenUsage[taskId]
+	}
+
+	public log(message: string) {
+		this.outputChannel.appendLine(message)
 	}
 }

src/exports/roo-code.d.ts

@@ -1,5 +1,14 @@
 import { EventEmitter } from "events"
 
+export interface TokenUsage {
+	totalTokensIn: number
+	totalTokensOut: number
+	totalCacheWrites?: number
+	totalCacheReads?: number
+	totalCost: number
+	contextTokens: number
+}
+
 export interface RooCodeEvents {
 	message: [{ taskId: string; action: "created" | "updated"; message: ClineMessage }]
 	taskStarted: [taskId: string]
@@ -8,6 +17,8 @@ export interface RooCodeEvents {
 	taskAskResponded: [taskId: string]
 	taskAborted: [taskId: string]
 	taskSpawned: [taskId: string, childTaskId: string]
+	taskCompleted: [taskId: string, usage: TokenUsage]
+	taskTokenUsageUpdated: [taskId: string, usage: TokenUsage]
 }
 
 export interface RooCodeAPI extends EventEmitter<RooCodeEvents> {
@@ -19,6 +30,12 @@ export interface RooCodeAPI extends EventEmitter<RooCodeEvents> {
 	 */
 	startNewTask(task?: string, images?: string[]): Promise<string>
 
+	/**
+	 * Returns the current task stack.
+	 * @returns An array of task IDs.
+	 */
+	getCurrentTaskStack(): string[]
+
 	/**
 	 * Clears the current task.
 	 */
@@ -65,10 +82,17 @@ export interface RooCodeAPI extends EventEmitter<RooCodeEvents> {
 	getMessages(taskId: string): ClineMessage[]
 
 	/**
-	 * Returns the current task stack.
-	 * @returns An array of task IDs.
+	 * Returns the token usage for a given task.
+	 * @param taskId The ID of the task.
+	 * @returns A TokenUsage object.
 	 */
-	getCurrentTaskStack(): string[]
+	getTokenUsage(taskId: string): TokenUsage
+
+	/**
+	 * Logs a message to the output channel.
+	 * @param message The message to log.
+	 */
+	log(message: string): void
 }
 
 export type ClineAsk =

src/shared/getApiMetrics.ts

@@ -1,13 +1,6 @@
-import { ClineMessage } from "./ExtensionMessage"
+import { TokenUsage } from "../exports/roo-code"
 
-interface ApiMetrics {
-	totalTokensIn: number
-	totalTokensOut: number
-	totalCacheWrites?: number
-	totalCacheReads?: number
-	totalCost: number
-	contextTokens: number // Total tokens in conversation (last message's tokensIn + tokensOut + cacheWrites + cacheReads)
-}
+import { ClineMessage } from "./ExtensionMessage"
 
 /**
  * Calculates API metrics from an array of ClineMessages.
@@ -26,8 +19,8 @@ interface ApiMetrics {
  * const { totalTokensIn, totalTokensOut, totalCost } = getApiMetrics(messages);
  * // Result: { totalTokensIn: 10, totalTokensOut: 20, totalCost: 0.005 }
  */
-export function getApiMetrics(messages: ClineMessage[]): ApiMetrics {
-	const result: ApiMetrics = {
+export function getApiMetrics(messages: ClineMessage[]) {
+	const result: TokenUsage = {
 		totalTokensIn: 0,
 		totalTokensOut: 0,
 		totalCacheWrites: undefined,

webview-ui/package.json

@@ -6,13 +6,14 @@
 	"scripts": {
 		"lint": "eslint src --ext ts,tsx",
 		"lint-fix": "eslint src --ext ts,tsx --fix",
-		"check-types": "tsc --noEmit",
+		"check-types": "tsc",
 		"test": "jest",
 		"dev": "vite",
 		"build": "tsc -b && vite build",
 		"preview": "vite preview",
 		"storybook": "storybook dev -p 6006",
-		"build-storybook": "storybook build"
+		"build-storybook": "storybook build",
+		"clean": "rimraf build"
 	},
 	"dependencies": {
 		"@radix-ui/react-alert-dialog": "^1.1.6",