mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-07 08:26:51 +00:00
Turn benchmarks into a monorepo
This commit is contained in:
cte
parent
70a5d2b48c
commit
2f6f016f3b
72 changed files with 7634 additions and 2583 deletions
|
|
@ -1,2 +0,0 @@
|
|||
OPENROUTER_API_KEY=sk-or-v1-...
|
||||
POSTHOG_API_KEY=phc_...
|
||||
38
benchmark/.gitignore
vendored
Normal file
38
benchmark/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# Dependencies
|
||||
node_modules
|
||||
.pnp
|
||||
.pnp.js
|
||||
|
||||
# Local env files
|
||||
.env
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
|
||||
# Testing
|
||||
coverage
|
||||
|
||||
# Turbo
|
||||
.turbo
|
||||
|
||||
# Vercel
|
||||
.vercel
|
||||
|
||||
# Build Outputs
|
||||
.next/
|
||||
out/
|
||||
build
|
||||
dist
|
||||
|
||||
|
||||
# Debug
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# Misc
|
||||
.DS_Store
|
||||
*.pem
|
||||
2
benchmark/.npmrc
Normal file
2
benchmark/.npmrc
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
public-hoist-pattern[]=*libsql*
|
||||
public-hoist-pattern[]=*libsql*
|
||||
|
|
@ -1,5 +1,43 @@
|
|||
# Benchmark Harness
|
||||
|
||||
Install nvm:
|
||||
|
||||
```sh
|
||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash
|
||||
# Reload shell.
|
||||
nvm install
|
||||
```
|
||||
|
||||
Install pnpm:
|
||||
|
||||
```sh
|
||||
npm install --global corepack@latest
|
||||
corepack enable pnpm
|
||||
corepack use pnpm@latest-10
|
||||
```
|
||||
|
||||
Install dependencies:
|
||||
|
||||
```sh
|
||||
pnpm install
|
||||
```
|
||||
|
||||
Configure database:
|
||||
|
||||
```sh
|
||||
cp packages/server/.env.sample packages/server/.env
|
||||
# Update BENCHMARKS_DB_PATH as needed in `packages/server/.env`.
|
||||
pnpm --filter @benchmark/server db:migrate
|
||||
```
|
||||
|
||||
Run the benchmark server:
|
||||
|
||||
```sh
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
################################################################################
|
||||
|
||||
Configure ENV vars (OpenRouter, PostHog, etc):
|
||||
|
||||
```sh
|
||||
|
|
|
|||
2493
benchmark/package-lock.json
generated
2493
benchmark/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,30 +1,22 @@
|
|||
{
|
||||
"name": "benchmark",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"main": "out/run.js",
|
||||
"packageManager": "pnpm@10.6.5",
|
||||
"scripts": {
|
||||
"build": "npm run compile && cd .. && npm run compile && npm run build:webview",
|
||||
"lint": "eslint src --ext ts",
|
||||
"check-types": "tsc --noEmit",
|
||||
"compile": "rm -rf out && tsc -p tsconfig.json",
|
||||
"cli": "npm run compile && npx dotenvx run -f .env.local -- tsx src/cli.ts",
|
||||
"clean": "rimraf out",
|
||||
"clean:exercises": "cd exercises && git checkout -f && git clean -fd",
|
||||
"docker:build": "docker build -f Dockerfile -t roo-code-benchmark ..",
|
||||
"docker:run": "touch /tmp/benchmarks.db && docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark",
|
||||
"docker:start": "npm run docker:build && npm run docker:run",
|
||||
"docker:shell": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) /bin/bash",
|
||||
"docker:cli": "docker exec -it -w /home/vscode/repo/benchmark $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) xvfb-run npm run cli --",
|
||||
"docker:stop": "docker stop $(docker ps --filter \"ancestor=roo-code-benchmark\" -q)",
|
||||
"docker:rm": "docker rm $(docker ps -a --filter \"ancestor=roo-code-benchmark\" -q)",
|
||||
"docker:clean": "npm run docker:stop && npm run docker:rm"
|
||||
"lint": "turbo lint",
|
||||
"check-types": "turbo check-types",
|
||||
"format": "turbo format",
|
||||
"dev": "pnpm --filter @benchmark/server dev"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@vscode/test-electron": "^2.4.0",
|
||||
"gluegun": "^5.1.2",
|
||||
"@dotenvx/dotenvx": "^1.39.0",
|
||||
"@eslint/js": "^9.22.0",
|
||||
"eslint": "^9.22.0",
|
||||
"globals": "^16.0.0",
|
||||
"prettier": "^3.5.3",
|
||||
"rimraf": "^6.0.1",
|
||||
"tsx": "^4.19.3",
|
||||
"typescript": "^5.4.5",
|
||||
"yargs": "^17.7.2"
|
||||
"turbo": "^2.4.4",
|
||||
"typescript": "^5",
|
||||
"typescript-eslint": "^8.26.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
4
benchmark/packages/cli/eslint.config.mjs
Normal file
4
benchmark/packages/cli/eslint.config.mjs
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
import { config } from "@benchmark/eslint-config/base"
|
||||
|
||||
/** @type {import("eslint").Linter.Config} */
|
||||
export default [...config]
|
||||
20
benchmark/packages/cli/package.json
Normal file
20
benchmark/packages/cli/package.json
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"name": "@benchmark/cli",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.6.5+sha512.cdf928fca20832cd59ec53826492b7dc25dc524d4370b6b4adbf65803d32efaa6c1c88147c0ae4e8d579a6c9eec715757b50d4fa35eea179d868eada4ed043af",
|
||||
"scripts": {
|
||||
"lint": "eslint src --ext ts --max-warnings=0",
|
||||
"check-types": "tsc --noEmit",
|
||||
"format": "prettier --write .",
|
||||
"build": "pnpm --filter @benchmark/client build",
|
||||
"dev": "dotenvx run -f ../../.env -- tsx src/index.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@vscode/test-electron": "^2.4.0",
|
||||
"gluegun": "^5.1.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@benchmark/eslint-config": "workspace:^"
|
||||
}
|
||||
}
|
||||
|
|
@ -8,7 +8,7 @@ import { runTests } from "@vscode/test-electron"
|
|||
// <...>/Roo-Code/benchmark/src
|
||||
|
||||
const extensionDevelopmentPath = path.resolve(__dirname, "../../")
|
||||
const extensionTestsPath = path.resolve(__dirname, "../out/runExercise")
|
||||
const extensionTestsPath = path.resolve(__dirname, "../out/run.js")
|
||||
const promptsPath = path.resolve(__dirname, "../prompts")
|
||||
const exercisesPath = path.resolve(__dirname, "../../../exercises")
|
||||
const languages = ["cpp", "go", "java", "javascript", "python", "rust"]
|
||||
|
|
@ -125,7 +125,7 @@ async function createRun({ model }: { model: string }): Promise<{ id: number; mo
|
|||
|
||||
async function main() {
|
||||
const cli = build()
|
||||
.brand("benchmark-runner")
|
||||
.brand("benchmark-cli")
|
||||
.src(__dirname)
|
||||
.help()
|
||||
.version()
|
||||
22
benchmark/packages/cli/tsconfig.json
Normal file
22
benchmark/packages/cli/tsconfig.json
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"display": "Default",
|
||||
"compilerOptions": {
|
||||
"composite": false,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"inlineSources": false,
|
||||
"isolatedModules": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Node",
|
||||
"target": "es2020"
|
||||
},
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
4
benchmark/packages/client/eslint.config.mjs
Normal file
4
benchmark/packages/client/eslint.config.mjs
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
import { config } from "@benchmark/eslint-config/base"
|
||||
|
||||
/** @type {import("eslint").Linter.Config} */
|
||||
export default [...config]
|
||||
18
benchmark/packages/client/package.json
Normal file
18
benchmark/packages/client/package.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"name": "@benchmark/client",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.6.5",
|
||||
"scripts": {
|
||||
"lint": "eslint src --ext ts --max-warnings=0",
|
||||
"check-types": "tsc --noEmit",
|
||||
"format": "prettier --write .",
|
||||
"compile": "pnpm clean && tsc",
|
||||
"build": "pnpm compile && cd ../../.. && npm run compile && npm run build:webview",
|
||||
"clean": "rimraf out"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@benchmark/eslint-config": "workspace:^",
|
||||
"@types/vscode": "^1.98.0"
|
||||
}
|
||||
}
|
||||
260
benchmark/packages/client/src/roo-code.d.ts
vendored
Normal file
260
benchmark/packages/client/src/roo-code.d.ts
vendored
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
import { EventEmitter } from "events"
|
||||
|
||||
export interface TokenUsage {
|
||||
totalTokensIn: number
|
||||
totalTokensOut: number
|
||||
totalCacheWrites?: number
|
||||
totalCacheReads?: number
|
||||
totalCost: number
|
||||
contextTokens: number
|
||||
}
|
||||
|
||||
export interface RooCodeEvents {
|
||||
message: [{ taskId: string; action: "created" | "updated"; message: ClineMessage }]
|
||||
taskStarted: [taskId: string]
|
||||
taskPaused: [taskId: string]
|
||||
taskUnpaused: [taskId: string]
|
||||
taskAskResponded: [taskId: string]
|
||||
taskAborted: [taskId: string]
|
||||
taskSpawned: [taskId: string, childTaskId: string]
|
||||
taskCompleted: [taskId: string, usage: TokenUsage]
|
||||
taskTokenUsageUpdated: [taskId: string, usage: TokenUsage]
|
||||
}
|
||||
|
||||
export interface RooCodeAPI extends EventEmitter<RooCodeEvents> {
|
||||
/**
|
||||
* Starts a new task with an optional initial message and images.
|
||||
* @param task Optional initial task message.
|
||||
* @param images Optional array of image data URIs (e.g., "data:image/webp;base64,...").
|
||||
* @returns The ID of the new task.
|
||||
*/
|
||||
startNewTask(task?: string, images?: string[]): Promise<string>
|
||||
|
||||
/**
|
||||
* Returns the current task stack.
|
||||
* @returns An array of task IDs.
|
||||
*/
|
||||
getCurrentTaskStack(): string[]
|
||||
|
||||
/**
|
||||
* Clears the current task.
|
||||
*/
|
||||
clearCurrentTask(lastMessage?: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Cancels the current task.
|
||||
*/
|
||||
cancelCurrentTask(): Promise<void>
|
||||
|
||||
/**
|
||||
* Sends a message to the current task.
|
||||
* @param message Optional message to send.
|
||||
* @param images Optional array of image data URIs (e.g., "data:image/webp;base64,...").
|
||||
*/
|
||||
sendMessage(message?: string, images?: string[]): Promise<void>
|
||||
|
||||
/**
|
||||
* Simulates pressing the primary button in the chat interface.
|
||||
*/
|
||||
pressPrimaryButton(): Promise<void>
|
||||
|
||||
/**
|
||||
* Simulates pressing the secondary button in the chat interface.
|
||||
*/
|
||||
pressSecondaryButton(): Promise<void>
|
||||
|
||||
/**
|
||||
* Sets the configuration for the current task.
|
||||
* @param values An object containing key-value pairs to set.
|
||||
*/
|
||||
setConfiguration(values: Partial<ConfigurationValues>): Promise<void>
|
||||
|
||||
/**
|
||||
* Returns true if the API is ready to use.
|
||||
*/
|
||||
isReady(): boolean
|
||||
|
||||
/**
|
||||
* Returns the messages for a given task.
|
||||
* @param taskId The ID of the task.
|
||||
* @returns An array of ClineMessage objects.
|
||||
*/
|
||||
getMessages(taskId: string): ClineMessage[]
|
||||
|
||||
/**
|
||||
* Returns the token usage for a given task.
|
||||
* @param taskId The ID of the task.
|
||||
* @returns A TokenUsage object.
|
||||
*/
|
||||
getTokenUsage(taskId: string): TokenUsage
|
||||
|
||||
/**
|
||||
* Logs a message to the output channel.
|
||||
* @param message The message to log.
|
||||
*/
|
||||
log(message: string): void
|
||||
}
|
||||
|
||||
export type ClineAsk =
|
||||
| "followup"
|
||||
| "command"
|
||||
| "command_output"
|
||||
| "completion_result"
|
||||
| "tool"
|
||||
| "api_req_failed"
|
||||
| "resume_task"
|
||||
| "resume_completed_task"
|
||||
| "mistake_limit_reached"
|
||||
| "browser_action_launch"
|
||||
| "use_mcp_server"
|
||||
| "finishTask"
|
||||
|
||||
export type ClineSay =
|
||||
| "task"
|
||||
| "error"
|
||||
| "api_req_started"
|
||||
| "api_req_finished"
|
||||
| "api_req_retried"
|
||||
| "api_req_retry_delayed"
|
||||
| "api_req_deleted"
|
||||
| "text"
|
||||
| "reasoning"
|
||||
| "completion_result"
|
||||
| "user_feedback"
|
||||
| "user_feedback_diff"
|
||||
| "command_output"
|
||||
| "tool"
|
||||
| "shell_integration_warning"
|
||||
| "browser_action"
|
||||
| "browser_action_result"
|
||||
| "command"
|
||||
| "mcp_server_request_started"
|
||||
| "mcp_server_response"
|
||||
| "new_task_started"
|
||||
| "new_task"
|
||||
| "checkpoint_saved"
|
||||
| "rooignore_error"
|
||||
|
||||
export interface ClineMessage {
|
||||
ts: number
|
||||
type: "ask" | "say"
|
||||
ask?: ClineAsk
|
||||
say?: ClineSay
|
||||
text?: string
|
||||
images?: string[]
|
||||
partial?: boolean
|
||||
reasoning?: string
|
||||
conversationHistoryIndex?: number
|
||||
checkpoint?: Record<string, unknown>
|
||||
progressStatus?: ToolProgressStatus
|
||||
}
|
||||
|
||||
export type SecretKey =
|
||||
| "apiKey"
|
||||
| "glamaApiKey"
|
||||
| "openRouterApiKey"
|
||||
| "awsAccessKey"
|
||||
| "awsSecretKey"
|
||||
| "awsSessionToken"
|
||||
| "openAiApiKey"
|
||||
| "geminiApiKey"
|
||||
| "openAiNativeApiKey"
|
||||
| "deepSeekApiKey"
|
||||
| "mistralApiKey"
|
||||
| "unboundApiKey"
|
||||
| "requestyApiKey"
|
||||
|
||||
export type GlobalStateKey =
|
||||
| "apiProvider"
|
||||
| "apiModelId"
|
||||
| "glamaModelId"
|
||||
| "glamaModelInfo"
|
||||
| "awsRegion"
|
||||
| "awsUseCrossRegionInference"
|
||||
| "awsProfile"
|
||||
| "awsUseProfile"
|
||||
| "awsCustomArn"
|
||||
| "vertexKeyFile"
|
||||
| "vertexJsonCredentials"
|
||||
| "vertexProjectId"
|
||||
| "vertexRegion"
|
||||
| "lastShownAnnouncementId"
|
||||
| "customInstructions"
|
||||
| "alwaysAllowReadOnly"
|
||||
| "alwaysAllowWrite"
|
||||
| "alwaysAllowExecute"
|
||||
| "alwaysAllowBrowser"
|
||||
| "alwaysAllowMcp"
|
||||
| "alwaysAllowModeSwitch"
|
||||
| "alwaysAllowSubtasks"
|
||||
| "taskHistory"
|
||||
| "openAiBaseUrl"
|
||||
| "openAiModelId"
|
||||
| "openAiCustomModelInfo"
|
||||
| "openAiUseAzure"
|
||||
| "ollamaModelId"
|
||||
| "ollamaBaseUrl"
|
||||
| "lmStudioModelId"
|
||||
| "lmStudioBaseUrl"
|
||||
| "anthropicBaseUrl"
|
||||
| "modelMaxThinkingTokens"
|
||||
| "azureApiVersion"
|
||||
| "openAiStreamingEnabled"
|
||||
| "openRouterModelId"
|
||||
| "openRouterModelInfo"
|
||||
| "openRouterBaseUrl"
|
||||
| "openRouterSpecificProvider"
|
||||
| "openRouterUseMiddleOutTransform"
|
||||
| "googleGeminiBaseUrl"
|
||||
| "allowedCommands"
|
||||
| "ttsEnabled"
|
||||
| "ttsSpeed"
|
||||
| "soundEnabled"
|
||||
| "soundVolume"
|
||||
| "diffEnabled"
|
||||
| "enableCheckpoints"
|
||||
| "checkpointStorage"
|
||||
| "browserViewportSize"
|
||||
| "screenshotQuality"
|
||||
| "remoteBrowserHost"
|
||||
| "fuzzyMatchThreshold"
|
||||
| "writeDelayMs"
|
||||
| "terminalOutputLineLimit"
|
||||
| "terminalShellIntegrationTimeout"
|
||||
| "mcpEnabled"
|
||||
| "enableMcpServerCreation"
|
||||
| "alwaysApproveResubmit"
|
||||
| "requestDelaySeconds"
|
||||
| "rateLimitSeconds"
|
||||
| "currentApiConfigName"
|
||||
| "listApiConfigMeta"
|
||||
| "vsCodeLmModelSelector"
|
||||
| "mode"
|
||||
| "modeApiConfigs"
|
||||
| "customModePrompts"
|
||||
| "customSupportPrompts"
|
||||
| "enhancementApiConfigId"
|
||||
| "experiments" // Map of experiment IDs to their enabled state
|
||||
| "autoApprovalEnabled"
|
||||
| "enableCustomModeCreation" // Enable the ability for Roo to create custom modes
|
||||
| "customModes" // Array of custom modes
|
||||
| "unboundModelId"
|
||||
| "requestyModelId"
|
||||
| "requestyModelInfo"
|
||||
| "unboundModelInfo"
|
||||
| "modelTemperature"
|
||||
| "modelMaxTokens"
|
||||
| "mistralCodestralUrl"
|
||||
| "maxOpenTabsContext"
|
||||
| "maxWorkspaceFiles"
|
||||
| "browserToolEnabled"
|
||||
| "lmStudioSpeculativeDecodingEnabled"
|
||||
| "lmStudioDraftModelId"
|
||||
| "telemetrySetting"
|
||||
| "showRooIgnoredFiles"
|
||||
| "remoteBrowserEnabled"
|
||||
| "language"
|
||||
|
||||
export type ConfigurationKey = GlobalStateKey | SecretKey
|
||||
|
||||
export type ConfigurationValues = Record<ConfigurationKey, unknown>
|
||||
|
|
@ -3,7 +3,7 @@ import * as path from "path"
|
|||
|
||||
import * as vscode from "vscode"
|
||||
|
||||
import { RooCodeAPI, TokenUsage } from "../../src/exports/roo-code"
|
||||
import { RooCodeAPI, TokenUsage } from "./roo-code"
|
||||
|
||||
import { waitUntilReady, waitUntilCompleted, sleep } from "./utils"
|
||||
|
||||
|
|
@ -83,7 +83,8 @@ export async function run() {
|
|||
|
||||
try {
|
||||
usage = await waitUntilCompleted({ api, taskId, timeout: 5 * 60 * 1_000 }) // 5m
|
||||
} catch (e) {
|
||||
} catch (e: unknown) {
|
||||
console.error(e)
|
||||
usage = api.getTokenUsage(taskId)
|
||||
}
|
||||
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import * as vscode from "vscode"
|
||||
|
||||
import { RooCodeAPI, TokenUsage } from "../../src/exports/roo-code"
|
||||
import { RooCodeAPI, TokenUsage } from "./roo-code"
|
||||
|
||||
type WaitForOptions = {
|
||||
timeout?: number
|
||||
|
|
@ -11,6 +11,5 @@
|
|||
"useUnknownInCatchVariables": false,
|
||||
"outDir": "out"
|
||||
},
|
||||
"include": ["src", "../src/exports/roo-code.d.ts"],
|
||||
"exclude": ["**/node_modules/**", "out"]
|
||||
"include": ["src", "../../../../src/exports/roo-code.d.ts"]
|
||||
}
|
||||
10
benchmark/packages/db/drizzle.config.ts
Normal file
10
benchmark/packages/db/drizzle.config.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { defineConfig } from "drizzle-kit"
|
||||
|
||||
export default defineConfig({
|
||||
out: "./drizzle",
|
||||
schema: "./src/schema.ts",
|
||||
dialect: "sqlite",
|
||||
dbCredentials: {
|
||||
url: process.env.BENCHMARKS_DB_PATH!,
|
||||
},
|
||||
})
|
||||
4
benchmark/packages/db/eslint.config.mjs
Normal file
4
benchmark/packages/db/eslint.config.mjs
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
import { config } from "@benchmark/eslint-config/base"
|
||||
|
||||
/** @type {import("eslint").Linter.Config} */
|
||||
export default [...config]
|
||||
26
benchmark/packages/db/package.json
Normal file
26
benchmark/packages/db/package.json
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
{
|
||||
"name": "@benchmark/db",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.6.5",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"lint": "eslint src --ext ts --max-warnings=0",
|
||||
"check-types": "tsc --noEmit",
|
||||
"format": "prettier --write .",
|
||||
"db:migrate": "dotenvx run -f ../../.env -- drizzle-kit push"
|
||||
},
|
||||
"dependencies": {
|
||||
"@libsql/client": "^0.14.0",
|
||||
"drizzle-orm": "^0.40.0",
|
||||
"drizzle-zod": "^0.7.0",
|
||||
"libsql": "^0.5.1",
|
||||
"zod": "^3.24.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@benchmark/eslint-config": "workspace:^",
|
||||
"drizzle-kit": "^0.30.5"
|
||||
}
|
||||
}
|
||||
5
benchmark/packages/db/src/db.ts
Normal file
5
benchmark/packages/db/src/db.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
import { drizzle } from "drizzle-orm/libsql"
|
||||
|
||||
export * from "./schema"
|
||||
|
||||
export const db = drizzle({ connection: { url: process.env.BENCHMARKS_DB_PATH! } })
|
||||
7
benchmark/packages/db/src/index.ts
Normal file
7
benchmark/packages/db/src/index.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
export { db } from "./db"
|
||||
|
||||
export { type Language, languages } from "./schema"
|
||||
export { type Run, insertRunSchema, runs } from "./schema"
|
||||
export { type Task, insertTaskSchema, tasks } from "./schema"
|
||||
|
||||
export { getRuns } from "./queries"
|
||||
21
benchmark/packages/db/src/queries.ts
Normal file
21
benchmark/packages/db/src/queries.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { desc, eq, sql } from "drizzle-orm"
|
||||
|
||||
import { db, runs, tasks } from "./db"
|
||||
|
||||
export const getRuns = () =>
|
||||
db
|
||||
.select({
|
||||
id: runs.id,
|
||||
model: runs.model,
|
||||
description: runs.description,
|
||||
createdAt: runs.createdAt,
|
||||
passed: sql<number>`sum(${tasks.passed})`,
|
||||
failed: sql<number>`sum(${tasks.passed} = 0)`,
|
||||
total: sql<number>`count(${tasks.id})`,
|
||||
rate: sql<number>`sum(${tasks.passed}) * 1.0 / count(${tasks.id})`,
|
||||
cost: sql<number>`sum(${tasks.cost})`,
|
||||
duration: sql<number>`sum(${tasks.duration})`,
|
||||
})
|
||||
.from(runs)
|
||||
.leftJoin(tasks, eq(runs.id, tasks.runId))
|
||||
.orderBy(desc(runs.id))
|
||||
60
benchmark/packages/db/src/schema.ts
Normal file
60
benchmark/packages/db/src/schema.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import { sqliteTable, text, real, integer } from "drizzle-orm/sqlite-core"
|
||||
import * as t from "drizzle-orm/sqlite-core"
|
||||
import { createInsertSchema } from "drizzle-zod"
|
||||
|
||||
/**
|
||||
* languages
|
||||
*/
|
||||
|
||||
export const languages = ["cpp", "go", "java", "javascript", "python", "rust"] as const
|
||||
|
||||
export type Language = (typeof languages)[number]
|
||||
|
||||
/**
|
||||
* runs
|
||||
*/
|
||||
|
||||
export const runs = sqliteTable("runs", {
|
||||
id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }),
|
||||
model: text().notNull(),
|
||||
description: text(),
|
||||
createdAt: integer({ mode: "timestamp" }).notNull(),
|
||||
})
|
||||
|
||||
export type Run = typeof runs.$inferSelect
|
||||
|
||||
export const insertRunSchema = createInsertSchema(runs).omit({
|
||||
id: true,
|
||||
createdAt: true,
|
||||
})
|
||||
|
||||
/**
|
||||
* tasks
|
||||
*/
|
||||
|
||||
export const tasks = sqliteTable(
|
||||
"tasks",
|
||||
{
|
||||
id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }),
|
||||
runId: integer({ mode: "number" }).notNull(),
|
||||
language: text({ enum: languages }).notNull(),
|
||||
exercise: text().notNull(),
|
||||
tokensIn: integer({ mode: "number" }).notNull(),
|
||||
tokensOut: integer({ mode: "number" }).notNull(),
|
||||
tokensContext: integer({ mode: "number" }).notNull(),
|
||||
cacheWrites: integer({ mode: "number" }).notNull(),
|
||||
cacheReads: integer({ mode: "number" }).notNull(),
|
||||
cost: real().notNull(),
|
||||
duration: integer({ mode: "number" }).notNull(),
|
||||
passed: integer({ mode: "boolean" }).notNull(),
|
||||
createdAt: integer({ mode: "timestamp" }).notNull(),
|
||||
},
|
||||
(table) => [t.uniqueIndex("language_exercise_idx").on(table.runId, table.language, table.exercise)],
|
||||
)
|
||||
|
||||
export type Task = typeof tasks.$inferSelect
|
||||
|
||||
export const insertTaskSchema = createInsertSchema(tasks).omit({
|
||||
id: true,
|
||||
createdAt: true,
|
||||
})
|
||||
23
benchmark/packages/db/tsconfig.json
Normal file
23
benchmark/packages/db/tsconfig.json
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"display": "Default",
|
||||
"compilerOptions": {
|
||||
"composite": false,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"inlineSources": false,
|
||||
"isolatedModules": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Node",
|
||||
"target": "es2020",
|
||||
"types": ["node"]
|
||||
},
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
|
|
@ -79,7 +79,7 @@ RUN /home/vscode/.local/bin/uv sync
|
|||
|
||||
# Build web-ui
|
||||
WORKDIR /home/vscode/exercises/web-ui
|
||||
RUN echo "DB_FILE_NAME=file:/tmp/benchmarks.db" > .env
|
||||
RUN echo "BENCHMARKS_DB_PATH=file:/tmp/benchmarks.db" > .env
|
||||
RUN pnpm install
|
||||
RUN npx drizzle-kit push
|
||||
|
||||
17
benchmark/packages/docker/package.json
Normal file
17
benchmark/packages/docker/package.json
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"name": "@benchmark/docker",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"docker:build": "docker build -f Dockerfile -t roo-code-benchmark ..",
|
||||
"docker:run": "touch /tmp/benchmarks.db && docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark",
|
||||
"docker:start": "npm run docker:build && npm run docker:run",
|
||||
"docker:shell": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) /bin/bash",
|
||||
"docker:cli": "docker exec -it -w /home/vscode/repo/benchmark $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) xvfb-run npm run cli --",
|
||||
"docker:stop": "docker stop $(docker ps --filter \"ancestor=roo-code-benchmark\" -q)",
|
||||
"docker:rm": "docker rm $(docker ps -a --filter \"ancestor=roo-code-benchmark\" -q)",
|
||||
"docker:clean": "npm run docker:stop && npm run docker:rm"
|
||||
},
|
||||
"devDependencies": {},
|
||||
"dependencies": {}
|
||||
}
|
||||
32
benchmark/packages/eslint-config/base.js
Normal file
32
benchmark/packages/eslint-config/base.js
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
import js from "@eslint/js"
|
||||
import eslintConfigPrettier from "eslint-config-prettier"
|
||||
import turboPlugin from "eslint-plugin-turbo"
|
||||
import tseslint from "typescript-eslint"
|
||||
import onlyWarn from "eslint-plugin-only-warn"
|
||||
|
||||
/**
|
||||
* A shared ESLint configuration for the repository.
|
||||
*
|
||||
* @type {import("eslint").Linter.Config[]}
|
||||
* */
|
||||
export const config = [
|
||||
js.configs.recommended,
|
||||
eslintConfigPrettier,
|
||||
...tseslint.configs.recommended,
|
||||
{
|
||||
plugins: {
|
||||
turbo: turboPlugin,
|
||||
},
|
||||
rules: {
|
||||
"turbo/no-undeclared-env-vars": "warn",
|
||||
},
|
||||
},
|
||||
{
|
||||
plugins: {
|
||||
onlyWarn,
|
||||
},
|
||||
},
|
||||
{
|
||||
ignores: ["dist/**"],
|
||||
},
|
||||
]
|
||||
49
benchmark/packages/eslint-config/next.js
Normal file
49
benchmark/packages/eslint-config/next.js
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import js from "@eslint/js"
|
||||
import eslintConfigPrettier from "eslint-config-prettier"
|
||||
import tseslint from "typescript-eslint"
|
||||
import pluginReactHooks from "eslint-plugin-react-hooks"
|
||||
import pluginReact from "eslint-plugin-react"
|
||||
import globals from "globals"
|
||||
import pluginNext from "@next/eslint-plugin-next"
|
||||
import { config as baseConfig } from "./base.js"
|
||||
|
||||
/**
|
||||
* A custom ESLint configuration for libraries that use Next.js.
|
||||
*
|
||||
* @type {import("eslint").Linter.Config[]}
|
||||
* */
|
||||
export const nextJsConfig = [
|
||||
...baseConfig,
|
||||
js.configs.recommended,
|
||||
eslintConfigPrettier,
|
||||
...tseslint.configs.recommended,
|
||||
{
|
||||
...pluginReact.configs.flat.recommended,
|
||||
languageOptions: {
|
||||
...pluginReact.configs.flat.recommended.languageOptions,
|
||||
globals: {
|
||||
...globals.serviceworker,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
plugins: {
|
||||
"@next/next": pluginNext,
|
||||
},
|
||||
rules: {
|
||||
...pluginNext.configs.recommended.rules,
|
||||
...pluginNext.configs["core-web-vitals"].rules,
|
||||
},
|
||||
},
|
||||
{
|
||||
plugins: {
|
||||
"react-hooks": pluginReactHooks,
|
||||
},
|
||||
settings: { react: { version: "detect" } },
|
||||
rules: {
|
||||
...pluginReactHooks.configs.recommended.rules,
|
||||
// React scope no longer necessary with new JSX transform.
|
||||
"react/react-in-jsx-scope": "off",
|
||||
},
|
||||
},
|
||||
]
|
||||
24
benchmark/packages/eslint-config/package.json
Normal file
24
benchmark/packages/eslint-config/package.json
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"name": "@benchmark/eslint-config",
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.6.5",
|
||||
"exports": {
|
||||
"./base": "./base.js",
|
||||
"./next-js": "./next.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.22.0",
|
||||
"@next/eslint-plugin-next": "^15.2.1",
|
||||
"eslint": "^9.22.0",
|
||||
"eslint-config-prettier": "^10.1.1",
|
||||
"eslint-plugin-only-warn": "^1.1.0",
|
||||
"eslint-plugin-react": "^7.37.4",
|
||||
"eslint-plugin-react-hooks": "^5.2.0",
|
||||
"eslint-plugin-turbo": "^2.4.4",
|
||||
"globals": "^16.0.0",
|
||||
"typescript": "^5",
|
||||
"typescript-eslint": "^8.26.0"
|
||||
}
|
||||
}
|
||||
0
benchmark/packages/eslint-config/shared/rules.js
Normal file
0
benchmark/packages/eslint-config/shared/rules.js
Normal file
4
benchmark/packages/ipc/eslint.config.mjs
Normal file
4
benchmark/packages/ipc/eslint.config.mjs
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
import { config } from "@benchmark/eslint-config/base"
|
||||
|
||||
/** @type {import("eslint").Linter.Config} */
|
||||
export default [...config]
|
||||
21
benchmark/packages/ipc/package.json
Normal file
21
benchmark/packages/ipc/package.json
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"name": "@benchmark/ipc",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.6.5",
|
||||
"scripts": {
|
||||
"lint": "eslint src --ext ts --max-warnings=0",
|
||||
"check-types": "tsc --noEmit",
|
||||
"format": "prettier --write .",
|
||||
"test:server": "tsx scripts/server.ts",
|
||||
"test:client": "tsx scripts/client.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"node-ipc": "^12.0.0",
|
||||
"zod": "^3.24.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@benchmark/eslint-config": "workspace:^",
|
||||
"@types/node-ipc": "^9.2.3"
|
||||
}
|
||||
}
|
||||
34
benchmark/packages/ipc/scripts/client.ts
Normal file
34
benchmark/packages/ipc/scripts/client.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import { IpcClient } from "../src/ipcClient"
|
||||
|
||||
async function main(socketPath: string) {
|
||||
try {
|
||||
const startTime = Date.now()
|
||||
const client = new IpcClient(socketPath)
|
||||
client.connect()
|
||||
|
||||
while (!client.isConnected) {
|
||||
if (Date.now() - startTime > 5000) {
|
||||
throw new Error("Failed to connect to server.")
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000))
|
||||
}
|
||||
|
||||
while (client.isConnected) {
|
||||
client.ping()
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000))
|
||||
}
|
||||
|
||||
process.exit(0)
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
if (!process.argv[2]) {
|
||||
console.error("Usage: npx tsx scripts/client.ts <socketPath>")
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
main(process.argv[2])
|
||||
19
benchmark/packages/ipc/scripts/server.ts
Normal file
19
benchmark/packages/ipc/scripts/server.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { IpcServer } from "../src/ipcServer"
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const server = new IpcServer()
|
||||
server.listen()
|
||||
|
||||
while (server.isListening) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000))
|
||||
}
|
||||
|
||||
process.exit(0)
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
88
benchmark/packages/ipc/src/ipcClient.ts
Normal file
88
benchmark/packages/ipc/src/ipcClient.ts
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
import ipc from "node-ipc"
|
||||
|
||||
import { ClientMessage, ClientMessageType, ServerMessageType, serverMessageSchema } from "./schemas"
|
||||
|
||||
export class IpcClient {
|
||||
private readonly _socketPath: string
|
||||
private _isConnected = false
|
||||
private _clientId?: string
|
||||
|
||||
constructor(socketPath: string) {
|
||||
this._socketPath = socketPath
|
||||
}
|
||||
|
||||
connect() {
|
||||
ipc.config.silent = true
|
||||
|
||||
ipc.connectTo("benchmarkServer", this.socketPath, () => {
|
||||
ipc.of.benchmarkServer.on("connect", (args) => this.onConnect(args))
|
||||
ipc.of.benchmarkServer.on("message", (data) => this.onMessage(data))
|
||||
ipc.of.benchmarkServer.on("disconnect", (args) => this.onDisconnect(args))
|
||||
})
|
||||
}
|
||||
|
||||
private onConnect(args: unknown) {
|
||||
console.log("[client#onConnect]", args)
|
||||
this._isConnected = true
|
||||
}
|
||||
|
||||
private onMessage(data: unknown) {
|
||||
if (typeof data !== "object") {
|
||||
console.log("[client#onMessage] invalid data", data)
|
||||
return
|
||||
}
|
||||
|
||||
const result = serverMessageSchema.safeParse(data)
|
||||
|
||||
if (!result.success) {
|
||||
console.log("[client#onMessage] invalid payload", result.error)
|
||||
return
|
||||
}
|
||||
|
||||
const payload = result.data
|
||||
|
||||
switch (payload.type) {
|
||||
case ServerMessageType.Hello:
|
||||
console.log(`[client#Hello] ${payload.data.clientId}`)
|
||||
this._clientId = payload.data.clientId
|
||||
break
|
||||
case ServerMessageType.Pong:
|
||||
console.log(`[client#Pong]`)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
private onDisconnect(args: unknown) {
|
||||
console.log("[client#onDisconnect]", args)
|
||||
this._isConnected = false
|
||||
}
|
||||
|
||||
public sendMessage(message: ClientMessage) {
|
||||
ipc.of.benchmarkServer.emit("message", message)
|
||||
}
|
||||
|
||||
public ping() {
|
||||
if (!this.isReady) {
|
||||
return false
|
||||
}
|
||||
|
||||
this.sendMessage({ type: ClientMessageType.Ping, data: { clientId: this._clientId! } })
|
||||
return true
|
||||
}
|
||||
|
||||
public get socketPath() {
|
||||
return this._socketPath
|
||||
}
|
||||
|
||||
public get clientId() {
|
||||
return this._clientId
|
||||
}
|
||||
|
||||
public get isConnected() {
|
||||
return this._isConnected
|
||||
}
|
||||
|
||||
public get isReady() {
|
||||
return this._isConnected && this._clientId !== undefined
|
||||
}
|
||||
}
|
||||
82
benchmark/packages/ipc/src/ipcServer.ts
Normal file
82
benchmark/packages/ipc/src/ipcServer.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import { Socket } from "node:net"
|
||||
import ipc from "node-ipc"
|
||||
import * as os from "node:os"
|
||||
import * as path from "node:path"
|
||||
import * as crypto from "node:crypto"
|
||||
|
||||
import { ClientMessageType, ServerMessage, ServerMessageType, clientMessageSchema } from "./schemas"
|
||||
|
||||
export class IpcServer {
|
||||
private _isListening = false
|
||||
private _socketId: string
|
||||
private _clients: Map<string, Socket>
|
||||
|
||||
constructor() {
|
||||
this._socketId = "benchmark"
|
||||
this._clients = new Map()
|
||||
}
|
||||
|
||||
public listen() {
|
||||
this._isListening = true
|
||||
|
||||
ipc.config.id = this._socketId
|
||||
ipc.config.silent = true
|
||||
|
||||
ipc.serve(this.socketPath, () => {
|
||||
ipc.server.on("connect", (socket) => this.onConnect(socket))
|
||||
ipc.server.on("message", (data, socket) => this.onMessage(data, socket))
|
||||
ipc.server.on("socket.disconnected", (socket, id) => this.onDisconnect(socket, id))
|
||||
})
|
||||
|
||||
ipc.server.start()
|
||||
}
|
||||
|
||||
private onConnect(socket: Socket) {
|
||||
const clientId = crypto.randomBytes(6).toString("hex")
|
||||
console.log(`[server#onConnect]`, clientId)
|
||||
this._clients.set(clientId, socket)
|
||||
this.sendMessage(socket, { type: ServerMessageType.Hello, data: { clientId } })
|
||||
}
|
||||
|
||||
private onMessage(data: unknown, socket: Socket) {
|
||||
if (typeof data !== "object") {
|
||||
console.log("[server#onMessage] invalid data", data)
|
||||
return
|
||||
}
|
||||
|
||||
const result = clientMessageSchema.safeParse(data)
|
||||
|
||||
if (!result.success) {
|
||||
console.log("[server#onMessage] invalid payload", result.error)
|
||||
return
|
||||
}
|
||||
|
||||
const payload = result.data
|
||||
|
||||
switch (payload.type) {
|
||||
case ClientMessageType.Message:
|
||||
console.log(`[server#Message] ${payload.data.message}`)
|
||||
break
|
||||
case ClientMessageType.Ping:
|
||||
console.log(`[server#Ping]`)
|
||||
this.sendMessage(socket, { type: ServerMessageType.Pong })
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
private onDisconnect(socket: Socket, destroyedSocketID: string) {
|
||||
console.log(`[server#socket.disconnected] ${destroyedSocketID}`)
|
||||
}
|
||||
|
||||
public sendMessage(socket: Socket, message: ServerMessage) {
|
||||
ipc.server.emit(socket, "message", message)
|
||||
}
|
||||
|
||||
public get socketPath() {
|
||||
return path.join(os.tmpdir(), `${this._socketId}.sock`)
|
||||
}
|
||||
|
||||
public get isListening() {
|
||||
return this._isListening
|
||||
}
|
||||
}
|
||||
51
benchmark/packages/ipc/src/schemas.ts
Normal file
51
benchmark/packages/ipc/src/schemas.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import { z } from "zod"
|
||||
|
||||
/**
|
||||
* Client
|
||||
*/
|
||||
|
||||
export enum ClientMessageType {
|
||||
Message = "message",
|
||||
Ping = "ping",
|
||||
}
|
||||
|
||||
export const clientMessageSchema = z.discriminatedUnion("type", [
|
||||
z.object({
|
||||
type: z.literal(ClientMessageType.Message),
|
||||
data: z.object({
|
||||
clientId: z.string(),
|
||||
message: z.string(),
|
||||
}),
|
||||
}),
|
||||
z.object({
|
||||
type: z.literal(ClientMessageType.Ping),
|
||||
data: z.object({
|
||||
clientId: z.string(),
|
||||
}),
|
||||
}),
|
||||
])
|
||||
|
||||
export type ClientMessage = z.infer<typeof clientMessageSchema>
|
||||
|
||||
/**
|
||||
* Server
|
||||
*/
|
||||
|
||||
export enum ServerMessageType {
|
||||
Hello = "hello",
|
||||
Pong = "pong",
|
||||
}
|
||||
|
||||
export const serverMessageSchema = z.discriminatedUnion("type", [
|
||||
z.object({
|
||||
type: z.literal(ServerMessageType.Hello),
|
||||
data: z.object({
|
||||
clientId: z.string(),
|
||||
}),
|
||||
}),
|
||||
z.object({
|
||||
type: z.literal(ServerMessageType.Pong),
|
||||
}),
|
||||
])
|
||||
|
||||
export type ServerMessage = z.infer<typeof serverMessageSchema>
|
||||
22
benchmark/packages/ipc/tsconfig.json
Normal file
22
benchmark/packages/ipc/tsconfig.json
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"display": "Default",
|
||||
"compilerOptions": {
|
||||
"composite": false,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"inlineSources": false,
|
||||
"isolatedModules": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"preserveWatchOutput": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Node",
|
||||
"target": "es2020"
|
||||
},
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
45
benchmark/packages/server/.gitignore
vendored
Normal file
45
benchmark/packages/server/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
/.pnp
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/versions
|
||||
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# next.js
|
||||
/.next/
|
||||
/out/
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
*.pem
|
||||
|
||||
# debug
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
!.env.sample
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
||||
|
||||
# sqlite
|
||||
local.db
|
||||
56
benchmark/packages/server/README.md
Normal file
56
benchmark/packages/server/README.md
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
# Roo Code Benchmark Web UI
|
||||
|
||||
## Getting Started
|
||||
|
||||
Install dependencies:
|
||||
|
||||
```sh
|
||||
pnpm install
|
||||
```
|
||||
|
||||
Create your SQLite database:
|
||||
|
||||
```sh
|
||||
cp .env.sample .env
|
||||
# Update path to SQLite database as needed in `.env`.
|
||||
npx drizzle-kit push
|
||||
```
|
||||
|
||||
Start the app:
|
||||
|
||||
```sh
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
## API
|
||||
|
||||
```sh
|
||||
curl -f -X POST http://localhost:3000/api/runs \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "Claude 3.7 Sonnet"}'
|
||||
|
||||
curl -f -X POST http://localhost:3000/api/tasks \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"runId": 1,
|
||||
"language": "javascript",
|
||||
"exercise": "binary",
|
||||
"tokensIn": 1000000,
|
||||
"tokensOut": 50000,
|
||||
"tokensContext": 50,
|
||||
"cacheWrites": 5,
|
||||
"cacheReads": 10,
|
||||
"cost": 0.543,
|
||||
"duration": 150000,
|
||||
"passed": true
|
||||
}'
|
||||
```
|
||||
|
||||
## Recipes
|
||||
|
||||
Zero-out and re-create database file:
|
||||
|
||||
```sh
|
||||
truncate -s 0 /tmp/benchmarks.db
|
||||
npx drizzle-kit push
|
||||
```
|
||||
21
benchmark/packages/server/components.json
Normal file
21
benchmark/packages/server/components.json
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"$schema": "https://ui.shadcn.com/schema.json",
|
||||
"style": "new-york",
|
||||
"rsc": true,
|
||||
"tsx": true,
|
||||
"tailwind": {
|
||||
"config": "",
|
||||
"css": "src/app/globals.css",
|
||||
"baseColor": "neutral",
|
||||
"cssVariables": true,
|
||||
"prefix": ""
|
||||
},
|
||||
"aliases": {
|
||||
"components": "@/components",
|
||||
"utils": "@/lib/utils",
|
||||
"ui": "@/components/ui",
|
||||
"lib": "@/lib",
|
||||
"hooks": "@/hooks"
|
||||
},
|
||||
"iconLibrary": "lucide"
|
||||
}
|
||||
4
benchmark/packages/server/eslint.config.mjs
Normal file
4
benchmark/packages/server/eslint.config.mjs
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
import { nextJsConfig } from "@benchmark/eslint-config/next-js"
|
||||
|
||||
/** @type {import("eslint").Linter.Config} */
|
||||
export default [...nextJsConfig]
|
||||
7
benchmark/packages/server/next.config.ts
Normal file
7
benchmark/packages/server/next.config.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import type { NextConfig } from "next"
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
/* config options here */
|
||||
}
|
||||
|
||||
export default nextConfig
|
||||
35
benchmark/packages/server/package.json
Normal file
35
benchmark/packages/server/package.json
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
{
|
||||
"name": "@benchmark/server",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.6.5",
|
||||
"scripts": {
|
||||
"dev": "dotenvx run -f ../../.env -- next dev --turbopack",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
"check-types": "tsc -b",
|
||||
"format": "prettier --write ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@benchmark/db": "workspace:^",
|
||||
"@radix-ui/react-slot": "^1.1.2",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"lucide-react": "^0.479.0",
|
||||
"next": "15.2.2",
|
||||
"next-themes": "^0.4.6",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"tailwind-merge": "^3.0.2",
|
||||
"tailwindcss-animate": "^1.0.7",
|
||||
"zod": "^3.24.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@benchmark/eslint-config": "workspace:^",
|
||||
"@tailwindcss/postcss": "^4",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"tailwindcss": "^4"
|
||||
}
|
||||
}
|
||||
5
benchmark/packages/server/postcss.config.mjs
Normal file
5
benchmark/packages/server/postcss.config.mjs
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
const config = {
|
||||
plugins: ["@tailwindcss/postcss"],
|
||||
}
|
||||
|
||||
export default config
|
||||
0
benchmark/packages/server/public/.gitkeep
Normal file
0
benchmark/packages/server/public/.gitkeep
Normal file
19
benchmark/packages/server/src/app/api/runs/route.ts
Normal file
19
benchmark/packages/server/src/app/api/runs/route.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { NextResponse } from "next/server"
|
||||
|
||||
import { db, runs, insertRunSchema } from "@benchmark/db"
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const run = await db
|
||||
.insert(runs)
|
||||
.values({
|
||||
...insertRunSchema.parse(await request.json()),
|
||||
createdAt: new Date(),
|
||||
})
|
||||
.returning()
|
||||
|
||||
return NextResponse.json({ run }, { status: 201 })
|
||||
} catch (error) {
|
||||
return NextResponse.json({ error: (error as Error).message }, { status: 500 })
|
||||
}
|
||||
}
|
||||
19
benchmark/packages/server/src/app/api/tasks/route.ts
Normal file
19
benchmark/packages/server/src/app/api/tasks/route.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import { NextResponse } from "next/server"
|
||||
|
||||
import { db, tasks, insertTaskSchema } from "@benchmark/db"
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const task = await db
|
||||
.insert(tasks)
|
||||
.values({
|
||||
...insertTaskSchema.parse(await request.json()),
|
||||
createdAt: new Date(),
|
||||
})
|
||||
.returning()
|
||||
|
||||
return NextResponse.json({ task }, { status: 201 })
|
||||
} catch (error) {
|
||||
return NextResponse.json({ error: (error as Error).message }, { status: 500 })
|
||||
}
|
||||
}
|
||||
BIN
benchmark/packages/server/src/app/favicon.ico
Normal file
BIN
benchmark/packages/server/src/app/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
123
benchmark/packages/server/src/app/globals.css
Normal file
123
benchmark/packages/server/src/app/globals.css
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
@import "tailwindcss";
|
||||
|
||||
@plugin "tailwindcss-animate";
|
||||
|
||||
@custom-variant dark (&:is(.dark *));
|
||||
|
||||
@theme inline {
|
||||
--color-background: var(--background);
|
||||
--color-foreground: var(--foreground);
|
||||
--font-sans: var(--font-geist-sans);
|
||||
--font-mono: var(--font-geist-mono);
|
||||
--color-sidebar-ring: var(--sidebar-ring);
|
||||
--color-sidebar-border: var(--sidebar-border);
|
||||
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
|
||||
--color-sidebar-accent: var(--sidebar-accent);
|
||||
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
|
||||
--color-sidebar-primary: var(--sidebar-primary);
|
||||
--color-sidebar-foreground: var(--sidebar-foreground);
|
||||
--color-sidebar: var(--sidebar);
|
||||
--color-chart-5: var(--chart-5);
|
||||
--color-chart-4: var(--chart-4);
|
||||
--color-chart-3: var(--chart-3);
|
||||
--color-chart-2: var(--chart-2);
|
||||
--color-chart-1: var(--chart-1);
|
||||
--color-ring: var(--ring);
|
||||
--color-input: var(--input);
|
||||
--color-border: var(--border);
|
||||
--color-destructive: var(--destructive);
|
||||
--color-accent-foreground: var(--accent-foreground);
|
||||
--color-accent: var(--accent);
|
||||
--color-muted-foreground: var(--muted-foreground);
|
||||
--color-muted: var(--muted);
|
||||
--color-secondary-foreground: var(--secondary-foreground);
|
||||
--color-secondary: var(--secondary);
|
||||
--color-primary-foreground: var(--primary-foreground);
|
||||
--color-primary: var(--primary);
|
||||
--color-popover-foreground: var(--popover-foreground);
|
||||
--color-popover: var(--popover);
|
||||
--color-card-foreground: var(--card-foreground);
|
||||
--color-card: var(--card);
|
||||
--radius-sm: calc(var(--radius) - 4px);
|
||||
--radius-md: calc(var(--radius) - 2px);
|
||||
--radius-lg: var(--radius);
|
||||
--radius-xl: calc(var(--radius) + 4px);
|
||||
}
|
||||
|
||||
:root {
|
||||
--radius: 0.625rem;
|
||||
--background: oklch(1 0 0);
|
||||
--foreground: oklch(0.145 0 0);
|
||||
--card: oklch(1 0 0);
|
||||
--card-foreground: oklch(0.145 0 0);
|
||||
--popover: oklch(1 0 0);
|
||||
--popover-foreground: oklch(0.145 0 0);
|
||||
--primary: oklch(0.205 0 0);
|
||||
--primary-foreground: oklch(0.985 0 0);
|
||||
--secondary: oklch(0.97 0 0);
|
||||
--secondary-foreground: oklch(0.205 0 0);
|
||||
--muted: oklch(0.97 0 0);
|
||||
--muted-foreground: oklch(0.556 0 0);
|
||||
--accent: oklch(0.97 0 0);
|
||||
--accent-foreground: oklch(0.205 0 0);
|
||||
--destructive: oklch(0.577 0.245 27.325);
|
||||
--border: oklch(0.922 0 0);
|
||||
--input: oklch(0.922 0 0);
|
||||
--ring: oklch(0.708 0 0);
|
||||
--chart-1: oklch(0.646 0.222 41.116);
|
||||
--chart-2: oklch(0.6 0.118 184.704);
|
||||
--chart-3: oklch(0.398 0.07 227.392);
|
||||
--chart-4: oklch(0.828 0.189 84.429);
|
||||
--chart-5: oklch(0.769 0.188 70.08);
|
||||
--sidebar: oklch(0.985 0 0);
|
||||
--sidebar-foreground: oklch(0.145 0 0);
|
||||
--sidebar-primary: oklch(0.205 0 0);
|
||||
--sidebar-primary-foreground: oklch(0.985 0 0);
|
||||
--sidebar-accent: oklch(0.97 0 0);
|
||||
--sidebar-accent-foreground: oklch(0.205 0 0);
|
||||
--sidebar-border: oklch(0.922 0 0);
|
||||
--sidebar-ring: oklch(0.708 0 0);
|
||||
}
|
||||
|
||||
.dark {
|
||||
--background: oklch(0.145 0 0);
|
||||
--foreground: oklch(0.985 0 0);
|
||||
--card: oklch(0.205 0 0);
|
||||
--card-foreground: oklch(0.985 0 0);
|
||||
--popover: oklch(0.205 0 0);
|
||||
--popover-foreground: oklch(0.985 0 0);
|
||||
--primary: oklch(0.922 0 0);
|
||||
--primary-foreground: oklch(0.205 0 0);
|
||||
--secondary: oklch(0.269 0 0);
|
||||
--secondary-foreground: oklch(0.985 0 0);
|
||||
--muted: oklch(0.269 0 0);
|
||||
--muted-foreground: oklch(0.708 0 0);
|
||||
--accent: oklch(0.269 0 0);
|
||||
--accent-foreground: oklch(0.985 0 0);
|
||||
--destructive: oklch(0.704 0.191 22.216);
|
||||
--border: oklch(1 0 0 / 10%);
|
||||
--input: oklch(1 0 0 / 15%);
|
||||
--ring: oklch(0.556 0 0);
|
||||
--chart-1: oklch(0.488 0.243 264.376);
|
||||
--chart-2: oklch(0.696 0.17 162.48);
|
||||
--chart-3: oklch(0.769 0.188 70.08);
|
||||
--chart-4: oklch(0.627 0.265 303.9);
|
||||
--chart-5: oklch(0.645 0.246 16.439);
|
||||
--sidebar: oklch(0.205 0 0);
|
||||
--sidebar-foreground: oklch(0.985 0 0);
|
||||
--sidebar-primary: oklch(0.488 0.243 264.376);
|
||||
--sidebar-primary-foreground: oklch(0.985 0 0);
|
||||
--sidebar-accent: oklch(0.269 0 0);
|
||||
--sidebar-accent-foreground: oklch(0.985 0 0);
|
||||
--sidebar-border: oklch(1 0 0 / 10%);
|
||||
--sidebar-ring: oklch(0.556 0 0);
|
||||
}
|
||||
|
||||
@layer base {
|
||||
* {
|
||||
@apply border-border outline-ring/50;
|
||||
}
|
||||
body {
|
||||
@apply bg-background text-foreground;
|
||||
}
|
||||
}
|
||||
37
benchmark/packages/server/src/app/layout.tsx
Normal file
37
benchmark/packages/server/src/app/layout.tsx
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import type { Metadata } from "next"
|
||||
import { Geist, Geist_Mono } from "next/font/google"
|
||||
|
||||
import { ThemeProvider } from "@/components/theme-provider"
|
||||
|
||||
import "./globals.css"
|
||||
|
||||
const geistSans = Geist({
|
||||
variable: "--font-geist-sans",
|
||||
subsets: ["latin"],
|
||||
})
|
||||
|
||||
const geistMono = Geist_Mono({
|
||||
variable: "--font-geist-mono",
|
||||
|
||||
subsets: ["latin"],
|
||||
})
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "Roo Code Benchmarks",
|
||||
}
|
||||
|
||||
export default function RootLayout({
|
||||
children,
|
||||
}: Readonly<{
|
||||
children: React.ReactNode
|
||||
}>) {
|
||||
return (
|
||||
<html lang="en">
|
||||
<body className={`${geistSans.variable} ${geistMono.variable} antialiased`}>
|
||||
<ThemeProvider attribute="class" defaultTheme="system" enableSystem disableTransitionOnChange>
|
||||
{children}
|
||||
</ThemeProvider>
|
||||
</body>
|
||||
</html>
|
||||
)
|
||||
}
|
||||
43
benchmark/packages/server/src/app/page.tsx
Normal file
43
benchmark/packages/server/src/app/page.tsx
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
import { getRuns } from "@benchmark/db"
|
||||
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui"
|
||||
import { formatCurrency, formatDuration } from "@/lib"
|
||||
|
||||
export const dynamic = "force-dynamic"
|
||||
|
||||
export default async function Home() {
|
||||
const runs = await getRuns()
|
||||
|
||||
return (
|
||||
<div className="mx-auto my-20 w-3xl">
|
||||
<Table className="border">
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>ID</TableHead>
|
||||
<TableHead>Model</TableHead>
|
||||
<TableHead>Timestamp</TableHead>
|
||||
<TableHead>Passed</TableHead>
|
||||
<TableHead>Failed</TableHead>
|
||||
<TableHead>% Correct</TableHead>
|
||||
<TableHead>Cost</TableHead>
|
||||
<TableHead>Duration</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{runs.map((run) => (
|
||||
<TableRow key={run.id}>
|
||||
<TableCell>{run.id}</TableCell>
|
||||
<TableCell>{run.model}</TableCell>
|
||||
<TableCell>{new Date(run.createdAt).toLocaleString()}</TableCell>
|
||||
<TableCell>{run.passed}</TableCell>
|
||||
<TableCell>{run.failed}</TableCell>
|
||||
<TableCell>{(run.rate * 100).toFixed(1)}%</TableCell>
|
||||
<TableCell>{formatCurrency(run.cost)}</TableCell>
|
||||
<TableCell>{formatDuration(run.duration)}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
13
benchmark/packages/server/src/components/theme-provider.tsx
Normal file
13
benchmark/packages/server/src/components/theme-provider.tsx
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
"use client"
|
||||
|
||||
import * as React from "react"
|
||||
import { type ThemeProviderProps } from "next-themes"
|
||||
import dynamic from "next/dynamic"
|
||||
|
||||
const NextThemesProvider = dynamic(() => import("next-themes").then((e) => e.ThemeProvider), {
|
||||
ssr: false,
|
||||
})
|
||||
|
||||
export function ThemeProvider({ children, ...props }: ThemeProviderProps) {
|
||||
return <NextThemesProvider {...props}>{children}</NextThemesProvider>
|
||||
}
|
||||
50
benchmark/packages/server/src/components/ui/button.tsx
Normal file
50
benchmark/packages/server/src/components/ui/button.tsx
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import * as React from "react"
|
||||
import { Slot } from "@radix-ui/react-slot"
|
||||
import { cva, type VariantProps } from "class-variance-authority"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const buttonVariants = cva(
|
||||
"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
|
||||
{
|
||||
variants: {
|
||||
variant: {
|
||||
default: "bg-primary text-primary-foreground shadow-xs hover:bg-primary/90",
|
||||
destructive:
|
||||
"bg-destructive text-white shadow-xs hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60",
|
||||
outline:
|
||||
"border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input dark:hover:bg-input/50",
|
||||
secondary: "bg-secondary text-secondary-foreground shadow-xs hover:bg-secondary/80",
|
||||
ghost: "hover:bg-accent hover:text-accent-foreground dark:hover:bg-accent/50",
|
||||
link: "text-primary underline-offset-4 hover:underline",
|
||||
},
|
||||
size: {
|
||||
default: "h-9 px-4 py-2 has-[>svg]:px-3",
|
||||
sm: "h-8 rounded-md gap-1.5 px-3 has-[>svg]:px-2.5",
|
||||
lg: "h-10 rounded-md px-6 has-[>svg]:px-4",
|
||||
icon: "size-9",
|
||||
},
|
||||
},
|
||||
defaultVariants: {
|
||||
variant: "default",
|
||||
size: "default",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
function Button({
|
||||
className,
|
||||
variant,
|
||||
size,
|
||||
asChild = false,
|
||||
...props
|
||||
}: React.ComponentProps<"button"> &
|
||||
VariantProps<typeof buttonVariants> & {
|
||||
asChild?: boolean
|
||||
}) {
|
||||
const Comp = asChild ? Slot : "button"
|
||||
|
||||
return <Comp data-slot="button" className={cn(buttonVariants({ variant, size, className }))} {...props} />
|
||||
}
|
||||
|
||||
export { Button, buttonVariants }
|
||||
2
benchmark/packages/server/src/components/ui/index.ts
Normal file
2
benchmark/packages/server/src/components/ui/index.ts
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
export * from "./button"
|
||||
export * from "./table"
|
||||
75
benchmark/packages/server/src/components/ui/table.tsx
Normal file
75
benchmark/packages/server/src/components/ui/table.tsx
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
"use client"
|
||||
|
||||
import * as React from "react"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
function Table({ className, ...props }: React.ComponentProps<"table">) {
|
||||
return (
|
||||
<div data-slot="table-container" className="relative w-full overflow-x-auto">
|
||||
<table data-slot="table" className={cn("w-full caption-bottom text-sm", className)} {...props} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function TableHeader({ className, ...props }: React.ComponentProps<"thead">) {
|
||||
return <thead data-slot="table-header" className={cn("[&_tr]:border-b", className)} {...props} />
|
||||
}
|
||||
|
||||
function TableBody({ className, ...props }: React.ComponentProps<"tbody">) {
|
||||
return <tbody data-slot="table-body" className={cn("[&_tr:last-child]:border-0", className)} {...props} />
|
||||
}
|
||||
|
||||
function TableFooter({ className, ...props }: React.ComponentProps<"tfoot">) {
|
||||
return (
|
||||
<tfoot
|
||||
data-slot="table-footer"
|
||||
className={cn("bg-muted/50 border-t font-medium [&>tr]:last:border-b-0", className)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function TableRow({ className, ...props }: React.ComponentProps<"tr">) {
|
||||
return (
|
||||
<tr
|
||||
data-slot="table-row"
|
||||
className={cn("hover:bg-muted/50 data-[state=selected]:bg-muted border-b transition-colors", className)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function TableHead({ className, ...props }: React.ComponentProps<"th">) {
|
||||
return (
|
||||
<th
|
||||
data-slot="table-head"
|
||||
className={cn(
|
||||
"text-muted-foreground h-10 px-2 text-left align-middle font-medium whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
|
||||
className,
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function TableCell({ className, ...props }: React.ComponentProps<"td">) {
|
||||
return (
|
||||
<td
|
||||
data-slot="table-cell"
|
||||
className={cn(
|
||||
"p-2 align-middle whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
|
||||
className,
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function TableCaption({ className, ...props }: React.ComponentProps<"caption">) {
|
||||
return (
|
||||
<caption data-slot="table-caption" className={cn("text-muted-foreground mt-4 text-sm", className)} {...props} />
|
||||
)
|
||||
}
|
||||
|
||||
export { Table, TableHeader, TableBody, TableFooter, TableHead, TableRow, TableCell, TableCaption }
|
||||
6
benchmark/packages/server/src/lib/formatCurrency.ts
Normal file
6
benchmark/packages/server/src/lib/formatCurrency.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
const formatter = new Intl.NumberFormat("en-US", {
|
||||
style: "currency",
|
||||
currency: "USD",
|
||||
})
|
||||
|
||||
export const formatCurrency = (amount: number) => formatter.format(amount)
|
||||
22
benchmark/packages/server/src/lib/formatDuration.ts
Normal file
22
benchmark/packages/server/src/lib/formatDuration.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
export const formatDuration = (durationMs: number) => {
|
||||
const seconds = Math.floor(durationMs / 1000)
|
||||
const hours = Math.floor(seconds / 3600)
|
||||
const minutes = Math.floor((seconds % 3600) / 60)
|
||||
const remainingSeconds = seconds % 60
|
||||
|
||||
const parts = []
|
||||
|
||||
if (hours > 0) {
|
||||
parts.push(`${hours}h`)
|
||||
}
|
||||
|
||||
if (minutes > 0) {
|
||||
parts.push(`${minutes}m`)
|
||||
}
|
||||
|
||||
if (remainingSeconds > 0 || parts.length === 0) {
|
||||
parts.push(`${remainingSeconds}s`)
|
||||
}
|
||||
|
||||
return parts.join(" ")
|
||||
}
|
||||
2
benchmark/packages/server/src/lib/index.ts
Normal file
2
benchmark/packages/server/src/lib/index.ts
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
export { formatCurrency } from "./formatCurrency"
|
||||
export { formatDuration } from "./formatDuration"
|
||||
6
benchmark/packages/server/src/lib/utils.ts
Normal file
6
benchmark/packages/server/src/lib/utils.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { clsx, type ClassValue } from "clsx"
|
||||
import { twMerge } from "tailwind-merge"
|
||||
|
||||
export function cn(...inputs: ClassValue[]) {
|
||||
return twMerge(clsx(inputs))
|
||||
}
|
||||
35
benchmark/packages/server/tsconfig.json
Normal file
35
benchmark/packages/server/tsconfig.json
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2017",
|
||||
"lib": ["dom", "dom.iterable", "esnext"],
|
||||
"allowJs": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"module": "esnext",
|
||||
"moduleResolution": "bundler",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"jsx": "preserve",
|
||||
"incremental": true,
|
||||
"plugins": [
|
||||
{
|
||||
"name": "next"
|
||||
}
|
||||
],
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": [
|
||||
"next-env.d.ts",
|
||||
"**/*.ts",
|
||||
"**/*.tsx",
|
||||
".next/types/**/*.ts",
|
||||
"../db/src/index.ts",
|
||||
"../db/src/queries.ts",
|
||||
"../db/src/schema.ts"
|
||||
],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
5952
benchmark/pnpm-lock.yaml
generated
Normal file
5952
benchmark/pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load diff
2
benchmark/pnpm-workspace.yaml
Normal file
2
benchmark/pnpm-workspace.yaml
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
packages:
|
||||
- 'packages/*'
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
Your job is to complete a coding exercise described by `.docs/instructions.md`.
|
||||
|
||||
A file with the implementation stubbed out has been created for you, along with a test file.
|
||||
|
||||
To successfully complete the exercise, you must pass all the tests in the test file.
|
||||
|
||||
To confirm that your solution is correct, you can compile your code and run the tests with:
|
||||
|
||||
```
|
||||
mkdir -p build && cd build
|
||||
cmake -G "Unix Makefiles" -DEXERCISM_RUN_ALL_TESTS=1 ..
|
||||
make
|
||||
```
|
||||
|
||||
Note that running `make` will compile the tests and generate compile time errors. Once the errors are fixed, running `make` will build and run the tests.
|
||||
|
||||
Do not alter the test file; it should be run as-is.
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
Your job is to complete a coding exercise described by `.docs/instructions.md`.
|
||||
|
||||
A file with the implementation stubbed out has been created for you, along with a test file.
|
||||
|
||||
To successfully complete the exercise, you must pass all the tests in the test file.
|
||||
|
||||
To confirm that your solution is correct, run the tests with `go test`. Do not alter the test file; it should be run as-is.
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
Your job is to complete a coding exercise described by `.docs/instructions.md`.
|
||||
|
||||
A file with the implementation stubbed out has been created for you, along with a test file.
|
||||
|
||||
To successfully complete the exercise, you must pass all the tests in the test file.
|
||||
|
||||
To confirm that your solution is correct, run the tests with `./gradlew test`. Do not alter the test file; it should be run as-is.
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
Your job is to complete a coding exercise described by `.docs/instructions.md`.
|
||||
|
||||
A file with the implementation stubbed out has been created for you, along with a test file.
|
||||
|
||||
To successfully complete the exercise, you must pass all the tests in the test file.
|
||||
|
||||
To confirm that your solution is correct, run the tests with `pnpm test`. Do not alter the test file; it should be run as-is.
|
||||
|
||||
Before running the tests make sure your environment is set up by running `pnpm install` to install the dependencies.
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
Your job is to complete a coding exercise described by `.docs/instructions.md`.
|
||||
|
||||
A file with the implementation stubbed out has been created for you, along with a test file.
|
||||
|
||||
To successfully complete the exercise, you must pass all the tests in the test file.
|
||||
|
||||
To confirm that your solution is correct, run the tests with `uv run python3 -m pytest -o markers=task [name]_test.py`. Do not alter the test file; it should be run as-is.
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
Your job is to complete a coding exercise described by `.docs/instructions.md`.
|
||||
|
||||
A file with the implementation stubbed out has been created for you, along with a test file.
|
||||
|
||||
To successfully complete the exercise, you must pass all the tests in the test file.
|
||||
|
||||
To confirm that your solution is correct, run the tests with `cargo test`. Do not alter the test file; it should be run as-is.
|
||||
28
benchmark/turbo.json
Normal file
28
benchmark/turbo.json
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
{
|
||||
"$schema": "https://turbo.build/schema.json",
|
||||
"globalDependencies": ["**/.env.*local"],
|
||||
"globalEnv": [
|
||||
"NODE_ENV",
|
||||
"NEXT_RUNTIME",
|
||||
"RUN_ID",
|
||||
"OPENROUTER_API_KEY",
|
||||
"OPENROUTER_MODEL_ID",
|
||||
"PROMPT_PATH",
|
||||
"WORKSPACE_PATH",
|
||||
"BENCHMARKS_DB_PATH"
|
||||
],
|
||||
"tasks": {
|
||||
"format": {},
|
||||
"lint": {},
|
||||
"check-types": {},
|
||||
"test": {},
|
||||
"dev": {
|
||||
"cache": false,
|
||||
"persistent": true
|
||||
},
|
||||
"build": {
|
||||
"dependsOn": ["^build"],
|
||||
"outputs": [".next/**", "!.next/cache/**"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -284,17 +284,14 @@
|
|||
"install-extension": "npm install",
|
||||
"install-webview": "cd webview-ui && npm install",
|
||||
"install-e2e": "cd e2e && npm install",
|
||||
"install-benchmark": "cd benchmark && npm install",
|
||||
"lint": "npm-run-all -p lint:*",
|
||||
"lint:extension": "eslint src --ext ts",
|
||||
"lint:webview": "cd webview-ui && npm run lint",
|
||||
"lint:e2e": "cd e2e && npm run lint",
|
||||
"lint:benchmark": "cd benchmark && npm run lint",
|
||||
"check-types": "npm-run-all -p check-types:*",
|
||||
"check-types:extension": "tsc --noEmit",
|
||||
"check-types:webview": "cd webview-ui && npm run check-types",
|
||||
"check-types:e2e": "cd e2e && npm run check-types",
|
||||
"check-types:benchmark": "cd benchmark && npm run check-types",
|
||||
"package": "npm-run-all -p build:webview build:esbuild check-types lint",
|
||||
"pretest": "npm run compile",
|
||||
"dev": "cd webview-ui && npm run dev",
|
||||
|
|
@ -317,7 +314,6 @@
|
|||
"clean:extension": "rimraf bin dist out",
|
||||
"clean:webview": "cd webview-ui && npm run clean",
|
||||
"clean:e2e": "cd e2e && npm run clean",
|
||||
"clean:benchmark": "cd benchmark && npm run clean",
|
||||
"update-contributors": "node scripts/update-contributors.js"
|
||||
},
|
||||
"dependencies": {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue