From afea705e36955be318bff21f9fe2cfd26ddfcf99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Tue, 27 Oct 2026 07:32:26 +0000 Subject: [PATCH] test(mcp): retain paired performance measurements (#3730) Co-authored-by: Gergo Magyar --- .github/workflows/ci-tests.yml | 53 +++ gitnexus/bench/mcp-query/README.md | 65 +++ gitnexus/bench/mcp-query/measure.mjs | 401 ++++++++++++++++++ gitnexus/bench/mcp-query/statistics.mjs | 193 +++++++++ .../test/unit/mcp-query-benchmark.test.ts | 54 +++ 5 files changed, 766 insertions(+) create mode 100644 gitnexus/bench/mcp-query/README.md create mode 100644 gitnexus/bench/mcp-query/measure.mjs create mode 100644 gitnexus/bench/mcp-query/statistics.mjs create mode 100644 gitnexus/test/unit/mcp-query-benchmark.test.ts diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index c9e0a8d41..d246ab31c 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -7,6 +7,59 @@ permissions: contents: read jobs: + mcp-query-benchmark: + name: MCP query performance + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - name: Pin comparison revision + id: baseline + shell: bash + run: printf 'sha=%s\n' "$(git rev-parse HEAD^1)" >> "$GITHUB_OUTPUT" + - uses: ./.github/actions/setup-gitnexus + with: + build: 'true' + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ steps.baseline.outputs.sha }} + path: baseline + persist-credentials: false + - name: Build baseline runtime + working-directory: baseline/gitnexus + run: | + npm ci + npm run build + npx tsx scripts/ensure-fts.ts + - name: Install search extensions + working-directory: gitnexus + run: npx tsx scripts/ensure-fts.ts + - name: Measure paired MCP requests + working-directory: gitnexus + run: node bench/mcp-query/measure.mjs --baseline-root ../baseline/gitnexus --out "$RUNNER_TEMP/mcp-query" --check + - name: Publish performance summary + if: always() + run: | + if [[ -f "$RUNNER_TEMP/mcp-query/report.md" ]]; then + cat "$RUNNER_TEMP/mcp-query/report.md" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Retain MCP measurements + if: always() + uses: ./.github/actions/upload-test-artifact + with: + name: mcp-query-performance + path: | + ${{ runner.temp }}/mcp-query/report.json + ${{ runner.temp }}/mcp-query/report.md + ${{ runner.temp }}/mcp-query/raw.jsonl + ${{ runner.temp }}/mcp-query/baseline.log + ${{ runner.temp }}/mcp-query/candidate.log + ${{ runner.temp }}/mcp-query/index.log + retention-days: 30 + tool-accuracy: name: release / deterministic tool accuracy runs-on: ubuntu-latest diff --git a/gitnexus/bench/mcp-query/README.md b/gitnexus/bench/mcp-query/README.md new file mode 100644 index 000000000..81c5df126 --- /dev/null +++ b/gitnexus/bench/mcp-query/README.md @@ -0,0 +1,65 @@ +# MCP query performance + +The **MCP query performance** CI job compares the checked-out commit with its +first parent. For a PR merge commit this is the exact base revision; release CI +compares the new main commit with the preceding main revision. Both complete +runtimes are built from their own lockfiles and run on the same Linux runner. + +The job writes its table to the Actions summary and retains `report.json`, +`report.md`, `raw.jsonl`, and fixture-only server/index logs as +**mcp-query-performance** for 30 days. Reports +contain exact revisions, dependency fingerprints, fixture hash, all observations, +backend phase timings, response bytes/source characters, and server CPU/peak RSS. +No model API calls or keys are used. The existing release agent evaluation remains +the source for dollar cost and task outcomes. + +## Run locally + +Build both checkouts with `npm ci` and `npm run build` in `gitnexus/`, then install +the FTS extension using `npx tsx scripts/ensure-fts.ts`. From the candidate package: + +```sh +node bench/mcp-query/measure.mjs \ + --baseline-root /path/to/base/gitnexus \ + --out /tmp/mcp-query-measurements --check +``` + +The output directory must not exist. `--candidate-root` selects another built +candidate. Without `--check`, `--sessions 1 --rounds 1` is a smoke run and cannot +establish a performance verdict. For an identical-code control, point both roots +at the same build. Do not discard slow samples or rerun until a check turns green. + +The candidate indexes one fixed generated repository containing 32 short and 32 +oversized TypeScript functions. Both servers read that same index. Index creation +is outside the timing window. Schemas must be compatible; an incompatible base +fails explicitly. Corpus generation and dependency installation use no model. + +Eight fresh server pairs each run a warmup cycle and five measured rounds. +Request order alternates between baseline and candidate. Workloads cover small +and large sources, metadata-only queries, empty results, indexed-source context, +tool listing, and four concurrent large-source queries. Startup and startup plus +first query are recorded separately. Filesystem caches are not flushed. + +## Decisions and limits + +- Response identities, order and diagnostics must agree; timing, guidance and + source representation are excluded from query equality. Indexed context source + must agree. At least ten known matches must survive each nonempty query. +- Candidate query source cannot exceed 16,000 Unicode characters, cannot disappear + entirely, and large previews must announce truncation. Context must retain the + original indexed snippet (the indexer separately caps long function snippets). + These guards prevent less work from masquerading as speed. +- Missing samples, query errors, incomplete results and invalid resource readings + fail. A failure report and completed pairs remain available for diagnosis. +- Latency comparisons use geometric-mean paired ratios and 10,000 bootstrap draws + clustered by server pair. A regression fails timing only when the lower bounds + exceed both 5% and 5 ms. An upper bound below either threshold is within the + threshold; other results are explicitly **inconclusive**, not a no-slowdown pass. +- Intervals are pointwise, not simultaneous guarantees over all workloads. p95, + CPU and RSS are descriptive. CPU includes native worker threads; RSS is each + server's final high-water mark. Client resources and indexing are excluded. + +This bounded generated corpus guards MCP behavior and latency; it cannot establish +performance on all repository sizes, embedding search, platforms, or complete +agent sessions. Retain the separate agent-cost measurements when judging whether +an optimization actually makes agents cheaper. diff --git a/gitnexus/bench/mcp-query/measure.mjs b/gitnexus/bench/mcp-query/measure.mjs new file mode 100644 index 000000000..e490650e0 --- /dev/null +++ b/gitnexus/bench/mcp-query/measure.mjs @@ -0,0 +1,401 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; +import { performance } from 'node:perf_hooks'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { summarize, formatReport } from './statistics.mjs'; + +const { values } = parseArgs({ + options: { + 'baseline-root': { type: 'string' }, + 'candidate-root': { type: 'string' }, + out: { type: 'string' }, + sessions: { type: 'string', default: '8' }, + rounds: { type: 'string', default: '5' }, + check: { type: 'boolean', default: false }, + }, +}); +assert.equal(process.platform, 'linux', 'This benchmark measures Linux /proc server resources'); +assert( + values['baseline-root'] && values.out, + 'Required: --baseline-root --out ', +); +const sessions = Number(values.sessions), + rounds = Number(values.rounds); +assert( + [sessions, rounds].every((n) => Number.isSafeInteger(n) && n > 0 && n <= 100), + 'Invalid sample count', +); +assert( + !values.check || (sessions >= 8 && rounds >= 5), + '--check requires at least 8 sessions and 5 rounds', +); +const out = path.resolve(values.out); +fs.mkdirSync(out); // Exclusive creation also rejects existing directories and symlinks. +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const hash = (text) => createHash('sha256').update(text).digest('hex'); +const runtimes = Object.fromEntries( + ['baseline', 'candidate'].map((variant) => { + const directory = fs.realpathSync(path.resolve(values[`${variant}-root`] ?? root)); + const cli = path.join(directory, 'dist/cli/index.js'); + assert(fs.existsSync(cli), `Build ${directory} first`); + return [ + variant, + { + directory, + cli, + sha: execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: directory, + encoding: 'utf8', + }).trim(), + lockSha256: hash(fs.readFileSync(path.join(directory, 'package-lock.json'))), + }, + ]; + }), +); +const repo = path.join(out, 'repo'); +const index = path.join(out, 'index'); +const env = { + PATH: process.env.PATH, + HOME: process.env.HOME, + LANG: 'C.UTF-8', + TZ: 'UTC', + ...(process.env.LD_LIBRARY_PATH ? { LD_LIBRARY_PATH: process.env.LD_LIBRARY_PATH } : {}), + GITNEXUS_HOME: path.join(out, 'home'), + GITNEXUS_STORAGE_PATH: index, + GITNEXUS_SHARED_STORE: 'off', + GITNEXUS_MEMORY: 'off', + GITNEXUS_CONTENT_RETENTION: 'full', + GITNEXUS_MCP_DEFAULT_REPO: repo, + GITNEXUS_MCP_READ_ONLY: '1', + GITNEXUS_LBUG_EXTENSION_INSTALL: 'load-only', +}; +const queries = [ + { name: 'small_source', symbol: 'lookupSmallSource', source: true }, + { name: 'small_metadata', symbol: 'lookupSmallSource', source: false }, + { name: 'large_source', symbol: 'lookupOversizedSource', source: true }, + { name: 'large_metadata', symbol: 'lookupOversizedSource', source: false }, + { name: 'empty', symbol: 'zqxvnoBenchmarkMatch9172', source: true }, +].map((q) => ({ + ...q, + tool: 'query', + args: { search_query: q.symbol, include_content: q.source }, +})); +const cases = [ + ...queries, + { + name: 'context', + tool: 'context', + args: { uid: 'Function:large/0.ts:lookupOversizedSource', include_content: true }, + }, + { name: 'tools_list', tool: 'tools/list' }, +]; + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .filter((k) => !['timing', 'next', 'content', 'content_truncated'].includes(k)) + .map((k) => [k, canonical(value[k])]), + ); +} + +function inspect(response, entry, variant) { + assert(!response.isError, `MCP error in ${entry.name}`); + if (entry.tool === 'tools/list') { + for (const name of ['query', 'context']) + assert( + response.tools.some((t) => t.name === name), + `Missing ${name} tool`, + ); + return { bytes: Buffer.byteLength(JSON.stringify(response)) }; + } + const text = response.content + .filter((r) => r.type === 'text') + .map((r) => r.text) + .join('\n'); + const trailer = text.indexOf('\n\n---\n'); + const body = JSON.parse(trailer < 0 ? text : text.slice(0, trailer)); + assert( + !body.error && !body.partial, + `Incomplete ${variant}/${entry.name} response: ${JSON.stringify({ error: body.error, partial: body.partial, warning: body.warning })}`, + ); + const rows = entry.tool === 'query' ? [...body.process_symbols, ...body.definitions] : []; + if (entry.tool === 'query') { + if (entry.name === 'empty') assert.equal(rows.length, 0); + else + assert( + rows.filter((r) => r.name === entry.symbol).length >= 10, + `Missing expected ${entry.symbol} matches`, + ); + } + const sourceChars = rows.reduce((n, row) => n + [...(row.content ?? '')].length, 0); + if (entry.tool === 'query' && variant === 'candidate') { + assert(sourceChars <= 16000, 'Query source exceeds the reviewed 16,000-character allowance'); + if (entry.source && entry.name !== 'empty') assert(sourceChars > 0, 'Query dropped all source'); + if (!entry.source) assert.equal(sourceChars, 0); + if (entry.name === 'large_source') + assert( + rows.some((r) => r.content_truncated), + 'Missing incomplete-preview marker', + ); + } + if (entry.tool === 'context') { + assert(body.symbol?.content?.length > 4096, 'Context must preserve the indexed source snippet'); + } + return { + bytes: Buffer.byteLength(text), + rows: rows.length, + sourceChars, + timing: body.timing, + digest: hash( + JSON.stringify( + entry.tool === 'context' + ? { ...canonical(body), content: body.symbol.content } + : canonical(body), + ), + ), + }; +} + +function resources(pid) { + const status = fs.readFileSync(`/proc/${pid}/status`, 'utf8'); + const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8'); + const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + const peakRssKiB = Number(status.match(/^VmHWM:\s+(\d+)/m)?.[1]); + const ticks = Number(execFileSync('getconf', ['CLK_TCK'], { encoding: 'utf8' }).trim()); + const cpuMs = ((Number(fields[11]) + Number(fields[12])) * 1000) / ticks; + assert( + [peakRssKiB, cpuMs].every((n) => Number.isFinite(n) && n > 0), + 'Missing server resources', + ); + return { peakRssKiB, cpuMs }; +} + +async function open(variant, clients) { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [runtimes[variant].cli, 'mcp'], + cwd: repo, + env, + stderr: 'pipe', + }); + const stderr = fs.createWriteStream(path.join(out, `${variant}.log`), { flags: 'a' }); + transport.stderr.pipe(stderr, { end: false }); + const client = new Client({ name: 'mcp-query-benchmark', version: '1.0.0' }); + let pending = 0, + timedOut = false; + const connection = { + invoke: async (entry) => { + pending++; + const start = performance.now(); + try { + const response = await (entry.tool === 'tools/list' + ? client.listTools(undefined, { timeout: 600000 }) + : client.callTool({ name: entry.tool, arguments: { repo, ...entry.args } }, undefined, { + timeout: 600000, + })); + return { ms: performance.now() - start, response }; + } catch (error) { + if (error.code === -32001) timedOut = true; + throw error; + } finally { + pending--; + } + }, + resources: () => resources(transport.pid), + close: async () => { + // Never terminate a server with outstanding native work after an RPC timeout. + if (pending === 0 && !timedOut) await client.close(); + stderr.end(); + }, + }; + clients[variant] = connection; + const start = performance.now(); + await client.connect(transport); + connection.startup = { ms: performance.now() - start }; + const first = await connection.invoke(cases[0]); + connection.first_query = { + ...inspect(first.response, cases[0], variant), + ms: connection.startup.ms + first.ms, + }; + return connection; +} + +const records = [], + finalResources = []; +const report = { + version: 1, + harnessSha256: hash( + ['measure.mjs', 'statistics.mjs'] + .map((file) => fs.readFileSync(new URL(file, import.meta.url), 'utf8')) + .join('\n'), + ), + runtimes, + sessions, + rounds, + cases, + environment: { + node: process.version, + platform: process.platform, + arch: process.arch, + cpu: os.cpus()[0].model, + cpuCount: os.cpus().length, + totalMemory: os.totalmem(), + }, + thresholds: { relative: 0.05, absoluteMs: 5 }, + records, + finalResources, +}; +try { + fs.mkdirSync(repo); + const source = []; + for (const [kind, symbol, length] of [ + ['small', 'lookupSmallSource', 128], + ['large', 'lookupOversizedSource', 32768], + ]) { + fs.mkdirSync(path.join(repo, kind)); + for (let i = 0; i < 32; i++) { + const file = `${kind}/${i}.ts`; + const text = `export function ${symbol}() {\n return ${JSON.stringify('x'.repeat(length) + i)};\n}\n`; + fs.writeFileSync(path.join(repo, file), text); + source.push([file, hash(text)]); + } + } + fs.writeFileSync(path.join(repo, '.gitignore'), '.gitnexus/\n'); + const gitEnv = { + ...env, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_AUTHOR_NAME: 'benchmark', + GIT_AUTHOR_EMAIL: 'benchmark@example.com', + GIT_COMMITTER_NAME: 'benchmark', + GIT_COMMITTER_EMAIL: 'benchmark@example.com', + }; + for (const args of [ + ['init', '-b', 'main'], + ['add', '.'], + ['-c', 'commit.gpgsign=false', 'commit', '-m', 'benchmark fixture'], + ]) { + execFileSync('git', args, { cwd: repo, env: gitEnv, stdio: 'pipe' }); + } + report.fixtureSha256 = hash(JSON.stringify(source)); + const log = fs.openSync(path.join(out, 'index.log'), 'wx'); + try { + execFileSync(process.execPath, [runtimes.candidate.cli, 'analyze', repo, '--index-only'], { + cwd: repo, + env, + stdio: ['ignore', log, log], + }); + } finally { + fs.closeSync(log); + } + report.index = JSON.parse(fs.readFileSync(path.join(index, 'gitnexus.json'), 'utf8')); + assert( + report.index.stats.nodes >= 128 && report.index.stats.files === 64, + 'Incomplete fixture index', + ); + assert.equal(report.index.stats.embeddings ?? 0, 0, 'Benchmark must not generate embeddings'); + for (let session = 0; session < sessions; session++) { + const clients = {}; + try { + for (const variant of session % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate']) + await open(variant, clients); + for (const [i, entry] of cases.entries()) { + const warmup = {}; + for (const variant of (session + i) % 2 + ? ['candidate', 'baseline'] + : ['baseline', 'candidate']) { + warmup[variant] = inspect( + (await clients[variant].invoke(entry)).response, + entry, + variant, + ); + } + assert.equal( + warmup.baseline.digest, + warmup.candidate.digest, + `Warmup parity: ${entry.name}`, + ); + } + for (let round = 0; round < rounds; round++) { + const row = { session, round, baseline: {}, candidate: {} }; + if (round === 0) + for (const variant of ['baseline', 'candidate']) { + row[variant].startup = clients[variant].startup; + row[variant].first_query = clients[variant].first_query; + } + for (let i = 0; i < cases.length; i++) { + const entry = cases[(i + round) % cases.length]; + const order = + (session + round + i) % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate']; + for (const variant of order) { + const result = await clients[variant].invoke(entry); + row[variant][entry.name] = { + ms: result.ms, + ...inspect(result.response, entry, variant), + }; + } + assert.equal( + row.baseline[entry.name].digest, + row.candidate[entry.name].digest, + `Result parity: ${entry.name}`, + ); + } + for (const variant of (session + round) % 2 + ? ['candidate', 'baseline'] + : ['baseline', 'candidate']) { + const start = performance.now(); + const batch = await Promise.allSettled( + Array.from({ length: 4 }, () => clients[variant].invoke(cases[2])), + ); + const ms = performance.now() - start; + for (const item of batch) { + assert.equal(item.status, 'fulfilled', 'Concurrent request failed'); + assert.equal( + inspect(item.value.response, cases[2], variant).digest, + row[variant].large_source.digest, + 'Concurrent parity', + ); + } + row[variant].concurrent4 = { ms, requestsPerSecond: 4000 / ms }; + } + records.push(row); + fs.appendFileSync(path.join(out, 'raw.jsonl'), JSON.stringify(row) + '\n'); + } + finalResources.push({ + session, + baseline: clients.baseline.resources(), + candidate: clients.candidate.resources(), + }); + console.log(`Completed process pair ${session + 1}/${sessions}`); + } finally { + for (const client of Object.values(clients)) await client.close(); + } + } + report.metrics = summarize(records, sessions, rounds, [ + ...cases.map((c) => c.name), + 'concurrent4', + ]); + fs.writeFileSync(path.join(out, 'report.json'), JSON.stringify(report, null, 2) + '\n'); + fs.writeFileSync(path.join(out, 'report.md'), formatReport(report)); + if (values.check && report.metrics.some((r) => r.verdict === 'regression')) process.exitCode = 1; + console.log(`Measurements: ${path.join(out, 'report.md')}`); +} catch (error) { + report.error = error.message; + fs.writeFileSync(path.join(out, 'report.json'), JSON.stringify(report, null, 2) + '\n'); + fs.writeFileSync( + path.join(out, 'report.md'), + `## MCP query performance\n\nFAILED: ${error.message}\n\nNo complete performance verdict. ${records.length}/${sessions * rounds} paired rounds retained.\n`, + ); + console.error(error); + process.exitCode = 1; +} diff --git a/gitnexus/bench/mcp-query/statistics.mjs b/gitnexus/bench/mcp-query/statistics.mjs new file mode 100644 index 000000000..5abe46b83 --- /dev/null +++ b/gitnexus/bench/mcp-query/statistics.mjs @@ -0,0 +1,193 @@ +import assert from 'node:assert/strict'; + +export function quantile(values, fraction) { + const sorted = [...values].sort((a, b) => a - b); + const position = (sorted.length - 1) * fraction; + const lo = Math.floor(position); + return sorted[lo] + (sorted[Math.ceil(position)] - sorted[lo]) * (position - lo); +} + +/** Resample whole process pairs: requests in one process are not independent. */ +export function compare(samples, { relative = 0.05, absolute = 5 } = {}) { + assert(samples.length > 0, 'Missing measurements'); + const groups = new Map(); + for (const row of samples) { + assert(Number.isInteger(row.session) && row.session >= 0, 'Invalid session'); + assert( + [row.baseline, row.candidate].every((n) => Number.isFinite(n) && n > 0), + 'Invalid measurement', + ); + if (!groups.has(row.session)) groups.set(row.session, []); + groups.get(row.session).push(row); + } + const clusters = [...groups.values()]; + const estimate = (rows) => ({ + ratio: Math.exp( + rows.reduce((sum, r) => sum + Math.log(r.candidate / r.baseline), 0) / rows.length, + ), + delta: rows.reduce((sum, r) => sum + r.candidate - r.baseline, 0) / rows.length, + }); + let seed = 3728; + const random = () => (seed = (1664525 * seed + 1013904223) >>> 0) / 2 ** 32; + const resampled = Array.from({ length: 10000 }, () => + estimate(clusters.flatMap(() => clusters[Math.floor(random() * clusters.length)])), + ); + const ratioCi95 = [0.025, 0.975].map((p) => + quantile( + resampled.map((r) => r.ratio), + p, + ), + ); + const deltaCi95 = [0.025, 0.975].map((p) => + quantile( + resampled.map((r) => r.delta), + p, + ), + ); + return { + ...estimate(samples), + pairs: samples.length, + sessions: clusters.length, + baselineMedian: quantile( + samples.map((r) => r.baseline), + 0.5, + ), + candidateMedian: quantile( + samples.map((r) => r.candidate), + 0.5, + ), + baselineP95: quantile( + samples.map((r) => r.baseline), + 0.95, + ), + candidateP95: quantile( + samples.map((r) => r.candidate), + 0.95, + ), + ratioCi95, + deltaCi95, + verdict: + clusters.length < 8 + ? 'insufficient-sessions' + : ratioCi95[0] > 1 + relative && deltaCi95[0] > absolute + ? 'regression' + : ratioCi95[1] <= 1 + relative || deltaCi95[1] <= absolute + ? 'within-threshold' + : 'inconclusive', + }; +} + +export function summarize(records, sessions, rounds, names) { + assert( + [sessions, rounds].every((n) => Number.isInteger(n) && n > 0), + 'Invalid experiment size', + ); + assert(names.length > 0 && new Set(names).size === names.length, 'Invalid workload list'); + assert.equal(records.length, sessions * rounds, 'Incomplete measurement set'); + const identities = new Set(); + for (const record of records) { + assert( + Number.isInteger(record.session) && + Number.isInteger(record.round) && + record.session >= 0 && + record.session < sessions && + record.round >= 0 && + record.round < rounds, + 'Invalid sample identity', + ); + const key = `${record.session}/${record.round}`; + assert(!identities.has(key), 'Duplicate sample'); + identities.add(key); + for (const variant of ['baseline', 'candidate']) { + for (const name of names) assert(record[variant][name], `Missing ${variant}/${name}`); + } + } + const metrics = names.map((name) => ({ + name, + ...compare( + records.map((r) => ({ + session: r.session, + baseline: r.baseline[name].ms, + candidate: r.candidate[name].ms, + })), + ), + })); + for (const name of ['startup', 'first_query']) { + metrics.push({ + name, + ...compare( + records + .filter((r) => r.round === 0) + .map((r) => ({ + session: r.session, + baseline: r.baseline[name].ms, + candidate: r.candidate[name].ms, + })), + ), + }); + } + return metrics; +} + +export function formatReport(report) { + const percent = (r) => `${r >= 1 ? '+' : ''}${((r - 1) * 100).toFixed(1)}%`; + const table = report.metrics + .map( + (r) => + `| ${r.name} | ${r.baselineMedian.toFixed(1)} → ${r.candidateMedian.toFixed(1)} | ${r.baselineP95.toFixed(1)} → ${r.candidateP95.toFixed(1)} | ${percent(r.ratio)} (${r.ratioCi95.map(percent).join(' to ')}) | ${r.verdict} |`, + ) + .join('\n'); + const resources = [ + ['cpuMs', 'Server CPU seconds', 1000], + ['peakRssKiB', 'Server peak RSS MiB', 1024], + ] + .map( + ([key, label, divisor]) => + `| ${label} | ${['baseline', 'candidate'] + .map((variant) => + ( + quantile( + report.finalResources.map((r) => r[variant][key]), + 0.5, + ) / divisor + ).toFixed(2), + ) + .join(' → ')} |`, + ) + .join('\n'); + const payloads = ['small_source', 'large_source', 'context', 'tools_list'] + .map( + (name) => + `| ${name} | ${['baseline', 'candidate'] + .map((variant) => + quantile( + report.records.map((r) => r[variant][name].bytes), + 0.5, + ).toFixed(0), + ) + .join(' → ')} |`, + ) + .join('\n'); + return `## MCP query performance + +Base \`${report.runtimes.baseline.sha}\`; candidate \`${report.runtimes.candidate.sha}\`. + +${report.sessions} fresh process pairs × ${report.rounds} rounds. Real MCP stdio, one shared keyword-search index, alternating requests. No model calls. + +| Workload | Median ms, base → candidate | p95 ms, base → candidate | Paired change (95% interval) | Assessment | +| --- | ---: | ---: | --- | --- | +${table} + +| Resource | Median, base → candidate | +| --- | ---: | +${resources} + +| Response | Median UTF-8 bytes, base → candidate | +| --- | ---: | +${payloads} + +Intervals resample whole process pairs (10,000 draws). The regression threshold is >5% AND >5 ms; only a confirmed regression fails timing. Inconclusive is not proof of no slowdown. p95, CPU and RSS are descriptive. Startup uses fresh processes with warm filesystem caches. + +Correctness, source budget, incomplete samples, and request errors always fail. JSON contains every response size, source-character count, backend phase timing, final server CPU/peak RSS, environment and fixture fingerprint; raw.jsonl retains every completed pair. These generated fixtures do not measure agent dollar savings or all repository workloads. +`; +} diff --git a/gitnexus/test/unit/mcp-query-benchmark.test.ts b/gitnexus/test/unit/mcp-query-benchmark.test.ts new file mode 100644 index 000000000..a78e3eefe --- /dev/null +++ b/gitnexus/test/unit/mcp-query-benchmark.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; + +const { compare, summarize } = await import( + new URL('../../bench/mcp-query/statistics.mjs', import.meta.url).href +); +const samples = (ratio: number, baseline = 100) => + Array.from({ length: 8 }, (_, session) => ({ session, baseline, candidate: baseline * ratio })); + +describe('MCP performance regression decision', () => { + it('detects a consistent material slowdown', () => { + expect(compare(samples(1.2)).verdict).toBe('regression'); + }); + + it('accepts faster requests and negligible absolute overhead', () => { + expect(compare(samples(0.8)).verdict).toBe('within-threshold'); + expect(compare(samples(2, 1)).verdict).toBe('within-threshold'); + }); + + it('does not call process variation a proven improvement or regression', () => { + const rows = samples(1).map((row, i) => ({ ...row, candidate: i % 2 ? 150 : 70 })); + expect(compare(rows).verdict).toBe('inconclusive'); + }); + + it('does not gain confidence by repeating correlated requests in one process', () => { + const rows = samples(1).map((row, i) => ({ ...row, candidate: i % 2 ? 150 : 70 })); + const repeated = rows.flatMap((row) => Array.from({ length: 5 }, () => ({ ...row }))); + const original = compare(rows).ratioCi95; + compare(repeated).ratioCi95.forEach((value: number, i: number) => + expect(value).toBeCloseTo(original[i], 12), + ); + expect(compare(samples(2).map((row) => ({ ...row, session: 0 }))).verdict).toBe( + 'insufficient-sessions', + ); + }); + + it.each([0, -1, NaN, Infinity, undefined])('rejects invalid measurements (%s)', (value) => { + expect(() => compare([{ session: 0, baseline: 100, candidate: value }])).toThrow(); + }); + + it('rejects empty, incomplete, duplicate, and missing workload evidence', () => { + expect(() => compare([])).toThrow(); + expect(() => summarize([], 8, 5, ['query'])).toThrow('Incomplete'); + const record = { + session: 0, + round: 0, + baseline: { query: { ms: 100 } }, + candidate: { query: { ms: 110 } }, + }; + expect(() => summarize([record, record], 1, 2, ['query'])).toThrow('Duplicate'); + expect(() => summarize([record], 1, 1, ['context'])).toThrow('Missing'); + expect(() => summarize([record], 1, 1, [])).toThrow('workload'); + expect(() => summarize([{ ...record, round: 0.5 }], 1, 1, ['query'])).toThrow('identity'); + }); +});