diff --git a/benchmark/apps/cli/package.json b/benchmark/apps/cli/package.json index 87656318bc..db48ab0edf 100644 --- a/benchmark/apps/cli/package.json +++ b/benchmark/apps/cli/package.json @@ -12,6 +12,7 @@ "@benchmark/db": "workspace:^", "@benchmark/types": "workspace:^", "@vscode/test-electron": "^2.4.0", + "execa": "^9.5.2", "gluegun": "^5.1.2", "p-map": "^7.0.3" }, diff --git a/benchmark/apps/cli/src/index.ts b/benchmark/apps/cli/src/index.ts index 192c7b7d45..8fafb6b98b 100644 --- a/benchmark/apps/cli/src/index.ts +++ b/benchmark/apps/cli/src/index.ts @@ -5,15 +5,25 @@ import * as os from "os" import pMap from "p-map" import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun" import { runTests } from "@vscode/test-electron" +import { execa, parseCommandString } from "execa" import { type Language, languages } from "@benchmark/types" -import { type Run, findRun, createRun, finishRun, createTask, Task, getTasks } from "@benchmark/db" +import { type Run, findRun, createRun, finishRun, createTask, Task, getTasks, updateTask } from "@benchmark/db" import { __dirname, extensionDevelopmentPath, extensionTestsPath, exercisesPath } from "./paths.js" import { getExercises } from "./exercises.js" export const isLanguage = (language: string): language is Language => languages.includes(language as Language) +const testCommands: Record = { + cpp: { commands: ["cmake -G 'Unix\\ Makefiles' -DEXERCISM_RUN_ALL_TESTS=1 ..", "make"], cwd: "build" }, // timeout 15s bash -c "cd '$dir' && mkdir -p build && cd build && cmake -G 'Unix Makefiles' -DEXERCISM_RUN_ALL_TESTS=1 .. >/dev/null 2>&1 && make >/dev/null 2>&1" + go: { commands: ["go test"] }, // timeout 15s bash -c "cd '$dir' && go test > /dev/null 2>&1" + java: { commands: ["./gradlew test"] }, // timeout --foreground 15s bash -c "cd '$dir' && ./gradlew test > /dev/null 2>&1" + javascript: { commands: ["pnpm install", "pnpm test"], timeout: 30_000 }, // timeout 30s bash -c "cd '$dir' && pnpm install >/dev/null 2>&1 && pnpm test >/dev/null 2>&1" + python: { commands: ["uv run python3 -m pytest -o markers=task *_test.py"] }, // timeout 15s bash -c "cd '$dir' && uv run python3 -m pytest -o markers=task *_test.py" + rust: { commands: ["cargo test"] }, // timeout 15s bash -c "cd '$dir' && cargo test > /dev/null 2>&1" +} + const run = async (toolbox: GluegunToolbox) => { const { config, prompt } = toolbox @@ -62,6 +72,37 @@ const run = async (toolbox: GluegunToolbox) => { for (const task of tasks) { await runExercise({ run, task }) + + const cmd = testCommands[task.language] + const exercisePath = path.resolve(exercisesPath, task.language, task.exercise) + const cwd = cmd.cwd ? path.resolve(exercisePath, cmd.cwd) : exercisePath + const commands = cmd.commands.map((cs) => parseCommandString(cs)) + + let passed = true + + for (const command of commands) { + const controller = new AbortController() + const cancelSignal = controller.signal + const timeout = setTimeout(() => controller.abort(), cmd.timeout ?? 15_000) + + try { + const result = await execa({ cwd, shell: true, reject: false, cancelSignal })`${command}` + console.log({ ...result, cwd, command }) + + clearTimeout(timeout) + + if (result.failed) { + passed = false + break + } + } catch (error) { + console.log(error) + passed = false + break + } + } + + await updateTask(task.id, { passed }) } const result = await finishRun(run.id) diff --git a/benchmark/packages/db/src/queries/runs.ts b/benchmark/packages/db/src/queries/runs.ts index faed9b67ff..8bef926a9f 100644 --- a/benchmark/packages/db/src/queries/runs.ts +++ b/benchmark/packages/db/src/queries/runs.ts @@ -1,4 +1,4 @@ -import { desc, eq, sum } from "drizzle-orm" +import { desc, eq, sql, sum } from "drizzle-orm" import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js" import type { InsertRun, UpdateRun } from "../schema.js" @@ -59,6 +59,8 @@ export const finishRun = async (runId: number) => { cacheReads: sum(schema.taskMetrics.cacheReads).mapWith(Number), cost: sum(schema.taskMetrics.cost).mapWith(Number), duration: sum(schema.taskMetrics.duration).mapWith(Number), + passed: sql`sum(${schema.tasks.passed} = 1)`, + failed: sql`sum(${schema.tasks.passed} = 0)`, }) .from(schema.taskMetrics) .innerJoin(schema.tasks, eq(schema.taskMetrics.id, schema.tasks.taskMetricsId)) @@ -69,8 +71,9 @@ export const finishRun = async (runId: number) => { throw new RecordNotFoundError() } - const taskMetrics = await createTaskMetrics(values) - await updateRun(runId, { taskMetricsId: taskMetrics.id }) + const { passed, failed, ...rest } = values + const taskMetrics = await createTaskMetrics(rest) + await updateRun(runId, { taskMetricsId: taskMetrics.id, passed, failed }) const run = await db.query.runs.findFirst({ where: eq(table.id, runId), with: { taskMetrics: true } }) diff --git a/benchmark/pnpm-lock.yaml b/benchmark/pnpm-lock.yaml index 9e0b033eb4..c418caa588 100644 --- a/benchmark/pnpm-lock.yaml +++ b/benchmark/pnpm-lock.yaml @@ -50,6 +50,9 @@ importers: '@vscode/test-electron': specifier: ^2.4.0 version: 2.4.1 + execa: + specifier: ^9.5.2 + version: 9.5.2 gluegun: specifier: ^5.1.2 version: 5.2.0 @@ -1742,6 +1745,13 @@ packages: '@rushstack/ts-command-line@4.23.6': resolution: {integrity: sha512-7WepygaF3YPEoToh4MAL/mmHkiIImQq3/uAkQX46kVoKTNOOlCtFGyNnze6OYuWw2o9rxsyrHVfIBKxq/am2RA==} + '@sec-ant/readable-stream@0.4.1': + resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} + + '@sindresorhus/merge-streams@4.0.0': + resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} + engines: {node: '>=18'} + '@standard-schema/utils@0.3.0': resolution: {integrity: sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==} @@ -2586,6 +2596,10 @@ packages: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} + execa@9.5.2: + resolution: {integrity: sha512-EHlpxMCpHWSAh1dgS6bVeoLAXGnJNdR93aabr4QCGbzOM73o5XmRfM/e5FUqsw3aagP8S8XEWUWFAxnRBnAF0Q==} + engines: {node: ^18.19.0 || >=20.5.0} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2624,6 +2638,10 @@ packages: resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} engines: {node: ^12.20 || >= 14.13} + figures@6.1.0: + resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} + engines: {node: '>=18'} + file-entry-cache@8.0.0: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} @@ -2720,6 +2738,10 @@ packages: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} engines: {node: '>=10'} + get-stream@9.0.1: + resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} + engines: {node: '>=18'} + get-symbol-description@1.1.0: resolution: {integrity: sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg==} engines: {node: '>= 0.4'} @@ -2812,6 +2834,10 @@ packages: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} + human-signals@8.0.0: + resolution: {integrity: sha512-/1/GPCpDUCCYwlERiYjxoczfP0zfvZMU/OWgQPMya9AbAE24vseigFdhAMObpc8Q4lc/kjutPfUddDYyAmejnA==} + engines: {node: '>=18.18.0'} + hyphenate-style-name@1.1.0: resolution: {integrity: sha512-WDC/ui2VVRrz3jOVi+XtjqkDjiVjTtFaAGiW37k6b+ohyQ5wYDOGkvCZa8+H0nx3gyvv0+BST9xuOgIyGQ00gw==} @@ -2929,6 +2955,10 @@ packages: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} + is-plain-obj@4.1.0: + resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} + engines: {node: '>=12'} + is-regex@1.2.1: resolution: {integrity: sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==} engines: {node: '>= 0.4'} @@ -2945,6 +2975,10 @@ packages: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} + is-stream@4.0.1: + resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==} + engines: {node: '>=18'} + is-string@1.1.1: resolution: {integrity: sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==} engines: {node: '>= 0.4'} @@ -2961,6 +2995,10 @@ packages: resolution: {integrity: sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==} engines: {node: '>=12'} + is-unicode-supported@2.1.0: + resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} + engines: {node: '>=18'} + is-weakmap@2.0.2: resolution: {integrity: sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w==} engines: {node: '>= 0.4'} @@ -3341,6 +3379,10 @@ packages: resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} engines: {node: '>=8'} + npm-run-path@6.0.0: + resolution: {integrity: sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==} + engines: {node: '>=18'} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -3422,6 +3464,10 @@ packages: resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} engines: {node: '>=8'} + parse-ms@4.0.0: + resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==} + engines: {node: '>=18'} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -3434,6 +3480,10 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + path-key@4.0.0: + resolution: {integrity: sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==} + engines: {node: '>=12'} + path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} @@ -3503,6 +3553,10 @@ packages: engines: {node: '>=14'} hasBin: true + pretty-ms@9.2.0: + resolution: {integrity: sha512-4yf0QO/sllf/1zbZWYnvWw3NxCQwLXKzIj0G849LSufP15BXKM0rbD2Z3wVnkMfjdn/CB0Dpp444gYAACdsplg==} + engines: {node: '>=18'} + process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} @@ -3880,6 +3934,10 @@ packages: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} engines: {node: '>=6'} + strip-final-newline@4.0.0: + resolution: {integrity: sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==} + engines: {node: '>=18'} + strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -4091,6 +4149,10 @@ packages: undici-types@6.19.8: resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} + unicorn-magic@0.3.0: + resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} + engines: {node: '>=18'} + universalify@2.0.1: resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} engines: {node: '>= 10.0.0'} @@ -4222,6 +4284,10 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + yoctocolors@2.1.1: + resolution: {integrity: sha512-GQHQqAopRhwU8Kt1DDM8NjibDXHC8eoh1erhGAJPEyveY9qqVeXvVikNKrDz69sHowPMorbPUrH/mx8c50eiBQ==} + engines: {node: '>=18'} + zod@3.24.2: resolution: {integrity: sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==} @@ -5395,6 +5461,10 @@ snapshots: - '@types/node' optional: true + '@sec-ant/readable-stream@0.4.1': {} + + '@sindresorhus/merge-streams@4.0.0': {} + '@standard-schema/utils@0.3.0': {} '@swc/counter@0.1.3': {} @@ -6382,6 +6452,21 @@ snapshots: signal-exit: 3.0.7 strip-final-newline: 2.0.0 + execa@9.5.2: + dependencies: + '@sindresorhus/merge-streams': 4.0.0 + cross-spawn: 7.0.6 + figures: 6.1.0 + get-stream: 9.0.1 + human-signals: 8.0.0 + is-plain-obj: 4.1.0 + is-stream: 4.0.1 + npm-run-path: 6.0.0 + pretty-ms: 9.2.0 + signal-exit: 4.1.0 + strip-final-newline: 4.0.0 + yoctocolors: 2.1.1 + fast-deep-equal@3.1.3: {} fast-glob@3.3.1: @@ -6421,6 +6506,10 @@ snapshots: node-domexception: 1.0.0 web-streams-polyfill: 3.3.3 + figures@6.1.0: + dependencies: + is-unicode-supported: 2.1.0 + file-entry-cache@8.0.0: dependencies: flat-cache: 4.0.1 @@ -6527,6 +6616,11 @@ snapshots: get-stream@6.0.1: {} + get-stream@9.0.1: + dependencies: + '@sec-ant/readable-stream': 0.4.1 + is-stream: 4.0.1 + get-symbol-description@1.1.0: dependencies: call-bound: 1.0.4 @@ -6653,6 +6747,8 @@ snapshots: human-signals@2.1.0: {} + human-signals@8.0.0: {} + hyphenate-style-name@1.1.0: {} ieee754@1.2.1: {} @@ -6765,6 +6861,8 @@ snapshots: is-number@7.0.0: {} + is-plain-obj@4.1.0: {} + is-regex@1.2.1: dependencies: call-bound: 1.0.4 @@ -6780,6 +6878,8 @@ snapshots: is-stream@2.0.1: {} + is-stream@4.0.1: {} + is-string@1.1.1: dependencies: call-bound: 1.0.4 @@ -6797,6 +6897,8 @@ snapshots: is-unicode-supported@1.3.0: {} + is-unicode-supported@2.1.0: {} + is-weakmap@2.0.2: {} is-weakref@1.1.1: @@ -7151,6 +7253,11 @@ snapshots: dependencies: path-key: 3.1.1 + npm-run-path@6.0.0: + dependencies: + path-key: 4.0.0 + unicorn-magic: 0.3.0 + object-assign@4.1.1: {} object-inspect@1.13.4: {} @@ -7259,12 +7366,16 @@ snapshots: json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 + parse-ms@4.0.0: {} + path-exists@4.0.0: {} path-is-absolute@1.0.1: {} path-key@3.1.1: {} + path-key@4.0.0: {} + path-parse@1.0.7: {} path-scurry@1.11.1: @@ -7310,6 +7421,10 @@ snapshots: prettier@3.5.3: {} + pretty-ms@9.2.0: + dependencies: + parse-ms: 4.0.0 + process-nextick-args@2.0.1: {} promise-limit@2.7.0: {} @@ -7789,6 +7904,8 @@ snapshots: strip-final-newline@2.0.0: {} + strip-final-newline@4.0.0: {} + strip-json-comments@3.1.1: {} strong-type@0.1.6: {} @@ -8001,6 +8118,8 @@ snapshots: undici-types@6.19.8: {} + unicorn-magic@0.3.0: {} + universalify@2.0.1: optional: true @@ -8132,4 +8251,6 @@ snapshots: yocto-queue@0.1.0: {} + yoctocolors@2.1.1: {} + zod@3.24.2: {}