test(mcp): retain paired performance measurements (#3730)

Co-authored-by: Gergo Magyar <gergomagyar0@gmail.com>

.github/workflows/ci-tests.yml

@@ -7,6 +7,59 @@ permissions:
   contents: read
 
 jobs:
+  mcp-query-benchmark:
+    name: MCP query performance
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+      - name: Pin comparison revision
+        id: baseline
+        shell: bash
+        run: printf 'sha=%s\n' "$(git rev-parse HEAD^1)" >> "$GITHUB_OUTPUT"
+      - uses: ./.github/actions/setup-gitnexus
+        with:
+          build: 'true'
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ steps.baseline.outputs.sha }}
+          path: baseline
+          persist-credentials: false
+      - name: Build baseline runtime
+        working-directory: baseline/gitnexus
+        run: |
+          npm ci
+          npm run build
+          npx tsx scripts/ensure-fts.ts
+      - name: Install search extensions
+        working-directory: gitnexus
+        run: npx tsx scripts/ensure-fts.ts
+      - name: Measure paired MCP requests
+        working-directory: gitnexus
+        run: node bench/mcp-query/measure.mjs --baseline-root ../baseline/gitnexus --out "$RUNNER_TEMP/mcp-query" --check
+      - name: Publish performance summary
+        if: always()
+        run: |
+          if [[ -f "$RUNNER_TEMP/mcp-query/report.md" ]]; then
+            cat "$RUNNER_TEMP/mcp-query/report.md" >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - name: Retain MCP measurements
+        if: always()
+        uses: ./.github/actions/upload-test-artifact
+        with:
+          name: mcp-query-performance
+          path: |
+            ${{ runner.temp }}/mcp-query/report.json
+            ${{ runner.temp }}/mcp-query/report.md
+            ${{ runner.temp }}/mcp-query/raw.jsonl
+            ${{ runner.temp }}/mcp-query/baseline.log
+            ${{ runner.temp }}/mcp-query/candidate.log
+            ${{ runner.temp }}/mcp-query/index.log
+          retention-days: 30
+
   tool-accuracy:
     name: release / deterministic tool accuracy
     runs-on: ubuntu-latest

gitnexus/bench/mcp-query/README.md

@@ -0,0 +1,65 @@
+# MCP query performance
+
+The **MCP query performance** CI job compares the checked-out commit with its
+first parent. For a PR merge commit this is the exact base revision; release CI
+compares the new main commit with the preceding main revision. Both complete
+runtimes are built from their own lockfiles and run on the same Linux runner.
+
+The job writes its table to the Actions summary and retains `report.json`,
+`report.md`, `raw.jsonl`, and fixture-only server/index logs as
+**mcp-query-performance** for 30 days. Reports
+contain exact revisions, dependency fingerprints, fixture hash, all observations,
+backend phase timings, response bytes/source characters, and server CPU/peak RSS.
+No model API calls or keys are used. The existing release agent evaluation remains
+the source for dollar cost and task outcomes.
+
+## Run locally
+
+Build both checkouts with `npm ci` and `npm run build` in `gitnexus/`, then install
+the FTS extension using `npx tsx scripts/ensure-fts.ts`. From the candidate package:
+
+```sh
+node bench/mcp-query/measure.mjs \
+  --baseline-root /path/to/base/gitnexus \
+  --out /tmp/mcp-query-measurements --check
+```
+
+The output directory must not exist. `--candidate-root` selects another built
+candidate. Without `--check`, `--sessions 1 --rounds 1` is a smoke run and cannot
+establish a performance verdict. For an identical-code control, point both roots
+at the same build. Do not discard slow samples or rerun until a check turns green.
+
+The candidate indexes one fixed generated repository containing 32 short and 32
+oversized TypeScript functions. Both servers read that same index. Index creation
+is outside the timing window. Schemas must be compatible; an incompatible base
+fails explicitly. Corpus generation and dependency installation use no model.
+
+Eight fresh server pairs each run a warmup cycle and five measured rounds.
+Request order alternates between baseline and candidate. Workloads cover small
+and large sources, metadata-only queries, empty results, indexed-source context,
+tool listing, and four concurrent large-source queries. Startup and startup plus
+first query are recorded separately. Filesystem caches are not flushed.
+
+## Decisions and limits
+
+- Response identities, order and diagnostics must agree; timing, guidance and
+  source representation are excluded from query equality. Indexed context source
+  must agree. At least ten known matches must survive each nonempty query.
+- Candidate query source cannot exceed 16,000 Unicode characters, cannot disappear
+  entirely, and large previews must announce truncation. Context must retain the
+  original indexed snippet (the indexer separately caps long function snippets).
+  These guards prevent less work from masquerading as speed.
+- Missing samples, query errors, incomplete results and invalid resource readings
+  fail. A failure report and completed pairs remain available for diagnosis.
+- Latency comparisons use geometric-mean paired ratios and 10,000 bootstrap draws
+  clustered by server pair. A regression fails timing only when the lower bounds
+  exceed both 5% and 5 ms. An upper bound below either threshold is within the
+  threshold; other results are explicitly **inconclusive**, not a no-slowdown pass.
+- Intervals are pointwise, not simultaneous guarantees over all workloads. p95,
+  CPU and RSS are descriptive. CPU includes native worker threads; RSS is each
+  server's final high-water mark. Client resources and indexing are excluded.
+
+This bounded generated corpus guards MCP behavior and latency; it cannot establish
+performance on all repository sizes, embedding search, platforms, or complete
+agent sessions. Retain the separate agent-cost measurements when judging whether
+an optimization actually makes agents cheaper.

gitnexus/bench/mcp-query/measure.mjs

@@ -0,0 +1,401 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { execFileSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
+import { performance } from 'node:perf_hooks';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
+import { summarize, formatReport } from './statistics.mjs';
+
+const { values } = parseArgs({
+  options: {
+    'baseline-root': { type: 'string' },
+    'candidate-root': { type: 'string' },
+    out: { type: 'string' },
+    sessions: { type: 'string', default: '8' },
+    rounds: { type: 'string', default: '5' },
+    check: { type: 'boolean', default: false },
+  },
+});
+assert.equal(process.platform, 'linux', 'This benchmark measures Linux /proc server resources');
+assert(
+  values['baseline-root'] && values.out,
+  'Required: --baseline-root <gitnexus directory> --out <new directory>',
+);
+const sessions = Number(values.sessions),
+  rounds = Number(values.rounds);
+assert(
+  [sessions, rounds].every((n) => Number.isSafeInteger(n) && n > 0 && n <= 100),
+  'Invalid sample count',
+);
+assert(
+  !values.check || (sessions >= 8 && rounds >= 5),
+  '--check requires at least 8 sessions and 5 rounds',
+);
+const out = path.resolve(values.out);
+fs.mkdirSync(out); // Exclusive creation also rejects existing directories and symlinks.
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
+const hash = (text) => createHash('sha256').update(text).digest('hex');
+const runtimes = Object.fromEntries(
+  ['baseline', 'candidate'].map((variant) => {
+    const directory = fs.realpathSync(path.resolve(values[`${variant}-root`] ?? root));
+    const cli = path.join(directory, 'dist/cli/index.js');
+    assert(fs.existsSync(cli), `Build ${directory} first`);
+    return [
+      variant,
+      {
+        directory,
+        cli,
+        sha: execFileSync('git', ['rev-parse', 'HEAD'], {
+          cwd: directory,
+          encoding: 'utf8',
+        }).trim(),
+        lockSha256: hash(fs.readFileSync(path.join(directory, 'package-lock.json'))),
+      },
+    ];
+  }),
+);
+const repo = path.join(out, 'repo');
+const index = path.join(out, 'index');
+const env = {
+  PATH: process.env.PATH,
+  HOME: process.env.HOME,
+  LANG: 'C.UTF-8',
+  TZ: 'UTC',
+  ...(process.env.LD_LIBRARY_PATH ? { LD_LIBRARY_PATH: process.env.LD_LIBRARY_PATH } : {}),
+  GITNEXUS_HOME: path.join(out, 'home'),
+  GITNEXUS_STORAGE_PATH: index,
+  GITNEXUS_SHARED_STORE: 'off',
+  GITNEXUS_MEMORY: 'off',
+  GITNEXUS_CONTENT_RETENTION: 'full',
+  GITNEXUS_MCP_DEFAULT_REPO: repo,
+  GITNEXUS_MCP_READ_ONLY: '1',
+  GITNEXUS_LBUG_EXTENSION_INSTALL: 'load-only',
+};
+const queries = [
+  { name: 'small_source', symbol: 'lookupSmallSource', source: true },
+  { name: 'small_metadata', symbol: 'lookupSmallSource', source: false },
+  { name: 'large_source', symbol: 'lookupOversizedSource', source: true },
+  { name: 'large_metadata', symbol: 'lookupOversizedSource', source: false },
+  { name: 'empty', symbol: 'zqxvnoBenchmarkMatch9172', source: true },
+].map((q) => ({
+  ...q,
+  tool: 'query',
+  args: { search_query: q.symbol, include_content: q.source },
+}));
+const cases = [
+  ...queries,
+  {
+    name: 'context',
+    tool: 'context',
+    args: { uid: 'Function:large/0.ts:lookupOversizedSource', include_content: true },
+  },
+  { name: 'tools_list', tool: 'tools/list' },
+];
+
+function canonical(value) {
+  if (Array.isArray(value)) return value.map(canonical);
+  if (!value || typeof value !== 'object') return value;
+  return Object.fromEntries(
+    Object.keys(value)
+      .sort()
+      .filter((k) => !['timing', 'next', 'content', 'content_truncated'].includes(k))
+      .map((k) => [k, canonical(value[k])]),
+  );
+}
+
+function inspect(response, entry, variant) {
+  assert(!response.isError, `MCP error in ${entry.name}`);
+  if (entry.tool === 'tools/list') {
+    for (const name of ['query', 'context'])
+      assert(
+        response.tools.some((t) => t.name === name),
+        `Missing ${name} tool`,
+      );
+    return { bytes: Buffer.byteLength(JSON.stringify(response)) };
+  }
+  const text = response.content
+    .filter((r) => r.type === 'text')
+    .map((r) => r.text)
+    .join('\n');
+  const trailer = text.indexOf('\n\n---\n');
+  const body = JSON.parse(trailer < 0 ? text : text.slice(0, trailer));
+  assert(
+    !body.error && !body.partial,
+    `Incomplete ${variant}/${entry.name} response: ${JSON.stringify({ error: body.error, partial: body.partial, warning: body.warning })}`,
+  );
+  const rows = entry.tool === 'query' ? [...body.process_symbols, ...body.definitions] : [];
+  if (entry.tool === 'query') {
+    if (entry.name === 'empty') assert.equal(rows.length, 0);
+    else
+      assert(
+        rows.filter((r) => r.name === entry.symbol).length >= 10,
+        `Missing expected ${entry.symbol} matches`,
+      );
+  }
+  const sourceChars = rows.reduce((n, row) => n + [...(row.content ?? '')].length, 0);
+  if (entry.tool === 'query' && variant === 'candidate') {
+    assert(sourceChars <= 16000, 'Query source exceeds the reviewed 16,000-character allowance');
+    if (entry.source && entry.name !== 'empty') assert(sourceChars > 0, 'Query dropped all source');
+    if (!entry.source) assert.equal(sourceChars, 0);
+    if (entry.name === 'large_source')
+      assert(
+        rows.some((r) => r.content_truncated),
+        'Missing incomplete-preview marker',
+      );
+  }
+  if (entry.tool === 'context') {
+    assert(body.symbol?.content?.length > 4096, 'Context must preserve the indexed source snippet');
+  }
+  return {
+    bytes: Buffer.byteLength(text),
+    rows: rows.length,
+    sourceChars,
+    timing: body.timing,
+    digest: hash(
+      JSON.stringify(
+        entry.tool === 'context'
+          ? { ...canonical(body), content: body.symbol.content }
+          : canonical(body),
+      ),
+    ),
+  };
+}
+
+function resources(pid) {
+  const status = fs.readFileSync(`/proc/${pid}/status`, 'utf8');
+  const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8');
+  const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+  const peakRssKiB = Number(status.match(/^VmHWM:\s+(\d+)/m)?.[1]);
+  const ticks = Number(execFileSync('getconf', ['CLK_TCK'], { encoding: 'utf8' }).trim());
+  const cpuMs = ((Number(fields[11]) + Number(fields[12])) * 1000) / ticks;
+  assert(
+    [peakRssKiB, cpuMs].every((n) => Number.isFinite(n) && n > 0),
+    'Missing server resources',
+  );
+  return { peakRssKiB, cpuMs };
+}
+
+async function open(variant, clients) {
+  const transport = new StdioClientTransport({
+    command: process.execPath,
+    args: [runtimes[variant].cli, 'mcp'],
+    cwd: repo,
+    env,
+    stderr: 'pipe',
+  });
+  const stderr = fs.createWriteStream(path.join(out, `${variant}.log`), { flags: 'a' });
+  transport.stderr.pipe(stderr, { end: false });
+  const client = new Client({ name: 'mcp-query-benchmark', version: '1.0.0' });
+  let pending = 0,
+    timedOut = false;
+  const connection = {
+    invoke: async (entry) => {
+      pending++;
+      const start = performance.now();
+      try {
+        const response = await (entry.tool === 'tools/list'
+          ? client.listTools(undefined, { timeout: 600000 })
+          : client.callTool({ name: entry.tool, arguments: { repo, ...entry.args } }, undefined, {
+              timeout: 600000,
+            }));
+        return { ms: performance.now() - start, response };
+      } catch (error) {
+        if (error.code === -32001) timedOut = true;
+        throw error;
+      } finally {
+        pending--;
+      }
+    },
+    resources: () => resources(transport.pid),
+    close: async () => {
+      // Never terminate a server with outstanding native work after an RPC timeout.
+      if (pending === 0 && !timedOut) await client.close();
+      stderr.end();
+    },
+  };
+  clients[variant] = connection;
+  const start = performance.now();
+  await client.connect(transport);
+  connection.startup = { ms: performance.now() - start };
+  const first = await connection.invoke(cases[0]);
+  connection.first_query = {
+    ...inspect(first.response, cases[0], variant),
+    ms: connection.startup.ms + first.ms,
+  };
+  return connection;
+}
+
+const records = [],
+  finalResources = [];
+const report = {
+  version: 1,
+  harnessSha256: hash(
+    ['measure.mjs', 'statistics.mjs']
+      .map((file) => fs.readFileSync(new URL(file, import.meta.url), 'utf8'))
+      .join('\n'),
+  ),
+  runtimes,
+  sessions,
+  rounds,
+  cases,
+  environment: {
+    node: process.version,
+    platform: process.platform,
+    arch: process.arch,
+    cpu: os.cpus()[0].model,
+    cpuCount: os.cpus().length,
+    totalMemory: os.totalmem(),
+  },
+  thresholds: { relative: 0.05, absoluteMs: 5 },
+  records,
+  finalResources,
+};
+try {
+  fs.mkdirSync(repo);
+  const source = [];
+  for (const [kind, symbol, length] of [
+    ['small', 'lookupSmallSource', 128],
+    ['large', 'lookupOversizedSource', 32768],
+  ]) {
+    fs.mkdirSync(path.join(repo, kind));
+    for (let i = 0; i < 32; i++) {
+      const file = `${kind}/${i}.ts`;
+      const text = `export function ${symbol}() {\n  return ${JSON.stringify('x'.repeat(length) + i)};\n}\n`;
+      fs.writeFileSync(path.join(repo, file), text);
+      source.push([file, hash(text)]);
+    }
+  }
+  fs.writeFileSync(path.join(repo, '.gitignore'), '.gitnexus/\n');
+  const gitEnv = {
+    ...env,
+    GIT_CONFIG_NOSYSTEM: '1',
+    GIT_CONFIG_GLOBAL: '/dev/null',
+    GIT_AUTHOR_NAME: 'benchmark',
+    GIT_AUTHOR_EMAIL: 'benchmark@example.com',
+    GIT_COMMITTER_NAME: 'benchmark',
+    GIT_COMMITTER_EMAIL: 'benchmark@example.com',
+  };
+  for (const args of [
+    ['init', '-b', 'main'],
+    ['add', '.'],
+    ['-c', 'commit.gpgsign=false', 'commit', '-m', 'benchmark fixture'],
+  ]) {
+    execFileSync('git', args, { cwd: repo, env: gitEnv, stdio: 'pipe' });
+  }
+  report.fixtureSha256 = hash(JSON.stringify(source));
+  const log = fs.openSync(path.join(out, 'index.log'), 'wx');
+  try {
+    execFileSync(process.execPath, [runtimes.candidate.cli, 'analyze', repo, '--index-only'], {
+      cwd: repo,
+      env,
+      stdio: ['ignore', log, log],
+    });
+  } finally {
+    fs.closeSync(log);
+  }
+  report.index = JSON.parse(fs.readFileSync(path.join(index, 'gitnexus.json'), 'utf8'));
+  assert(
+    report.index.stats.nodes >= 128 && report.index.stats.files === 64,
+    'Incomplete fixture index',
+  );
+  assert.equal(report.index.stats.embeddings ?? 0, 0, 'Benchmark must not generate embeddings');
+  for (let session = 0; session < sessions; session++) {
+    const clients = {};
+    try {
+      for (const variant of session % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate'])
+        await open(variant, clients);
+      for (const [i, entry] of cases.entries()) {
+        const warmup = {};
+        for (const variant of (session + i) % 2
+          ? ['candidate', 'baseline']
+          : ['baseline', 'candidate']) {
+          warmup[variant] = inspect(
+            (await clients[variant].invoke(entry)).response,
+            entry,
+            variant,
+          );
+        }
+        assert.equal(
+          warmup.baseline.digest,
+          warmup.candidate.digest,
+          `Warmup parity: ${entry.name}`,
+        );
+      }
+      for (let round = 0; round < rounds; round++) {
+        const row = { session, round, baseline: {}, candidate: {} };
+        if (round === 0)
+          for (const variant of ['baseline', 'candidate']) {
+            row[variant].startup = clients[variant].startup;
+            row[variant].first_query = clients[variant].first_query;
+          }
+        for (let i = 0; i < cases.length; i++) {
+          const entry = cases[(i + round) % cases.length];
+          const order =
+            (session + round + i) % 2 ? ['candidate', 'baseline'] : ['baseline', 'candidate'];
+          for (const variant of order) {
+            const result = await clients[variant].invoke(entry);
+            row[variant][entry.name] = {
+              ms: result.ms,
+              ...inspect(result.response, entry, variant),
+            };
+          }
+          assert.equal(
+            row.baseline[entry.name].digest,
+            row.candidate[entry.name].digest,
+            `Result parity: ${entry.name}`,
+          );
+        }
+        for (const variant of (session + round) % 2
+          ? ['candidate', 'baseline']
+          : ['baseline', 'candidate']) {
+          const start = performance.now();
+          const batch = await Promise.allSettled(
+            Array.from({ length: 4 }, () => clients[variant].invoke(cases[2])),
+          );
+          const ms = performance.now() - start;
+          for (const item of batch) {
+            assert.equal(item.status, 'fulfilled', 'Concurrent request failed');
+            assert.equal(
+              inspect(item.value.response, cases[2], variant).digest,
+              row[variant].large_source.digest,
+              'Concurrent parity',
+            );
+          }
+          row[variant].concurrent4 = { ms, requestsPerSecond: 4000 / ms };
+        }
+        records.push(row);
+        fs.appendFileSync(path.join(out, 'raw.jsonl'), JSON.stringify(row) + '\n');
+      }
+      finalResources.push({
+        session,
+        baseline: clients.baseline.resources(),
+        candidate: clients.candidate.resources(),
+      });
+      console.log(`Completed process pair ${session + 1}/${sessions}`);
+    } finally {
+      for (const client of Object.values(clients)) await client.close();
+    }
+  }
+  report.metrics = summarize(records, sessions, rounds, [
+    ...cases.map((c) => c.name),
+    'concurrent4',
+  ]);
+  fs.writeFileSync(path.join(out, 'report.json'), JSON.stringify(report, null, 2) + '\n');
+  fs.writeFileSync(path.join(out, 'report.md'), formatReport(report));
+  if (values.check && report.metrics.some((r) => r.verdict === 'regression')) process.exitCode = 1;
+  console.log(`Measurements: ${path.join(out, 'report.md')}`);
+} catch (error) {
+  report.error = error.message;
+  fs.writeFileSync(path.join(out, 'report.json'), JSON.stringify(report, null, 2) + '\n');
+  fs.writeFileSync(
+    path.join(out, 'report.md'),
+    `## MCP query performance\n\nFAILED: ${error.message}\n\nNo complete performance verdict. ${records.length}/${sessions * rounds} paired rounds retained.\n`,
+  );
+  console.error(error);
+  process.exitCode = 1;
+}

gitnexus/bench/mcp-query/statistics.mjs

@@ -0,0 +1,193 @@
+import assert from 'node:assert/strict';
+
+export function quantile(values, fraction) {
+  const sorted = [...values].sort((a, b) => a - b);
+  const position = (sorted.length - 1) * fraction;
+  const lo = Math.floor(position);
+  return sorted[lo] + (sorted[Math.ceil(position)] - sorted[lo]) * (position - lo);
+}
+
+/** Resample whole process pairs: requests in one process are not independent. */
+export function compare(samples, { relative = 0.05, absolute = 5 } = {}) {
+  assert(samples.length > 0, 'Missing measurements');
+  const groups = new Map();
+  for (const row of samples) {
+    assert(Number.isInteger(row.session) && row.session >= 0, 'Invalid session');
+    assert(
+      [row.baseline, row.candidate].every((n) => Number.isFinite(n) && n > 0),
+      'Invalid measurement',
+    );
+    if (!groups.has(row.session)) groups.set(row.session, []);
+    groups.get(row.session).push(row);
+  }
+  const clusters = [...groups.values()];
+  const estimate = (rows) => ({
+    ratio: Math.exp(
+      rows.reduce((sum, r) => sum + Math.log(r.candidate / r.baseline), 0) / rows.length,
+    ),
+    delta: rows.reduce((sum, r) => sum + r.candidate - r.baseline, 0) / rows.length,
+  });
+  let seed = 3728;
+  const random = () => (seed = (1664525 * seed + 1013904223) >>> 0) / 2 ** 32;
+  const resampled = Array.from({ length: 10000 }, () =>
+    estimate(clusters.flatMap(() => clusters[Math.floor(random() * clusters.length)])),
+  );
+  const ratioCi95 = [0.025, 0.975].map((p) =>
+    quantile(
+      resampled.map((r) => r.ratio),
+      p,
+    ),
+  );
+  const deltaCi95 = [0.025, 0.975].map((p) =>
+    quantile(
+      resampled.map((r) => r.delta),
+      p,
+    ),
+  );
+  return {
+    ...estimate(samples),
+    pairs: samples.length,
+    sessions: clusters.length,
+    baselineMedian: quantile(
+      samples.map((r) => r.baseline),
+      0.5,
+    ),
+    candidateMedian: quantile(
+      samples.map((r) => r.candidate),
+      0.5,
+    ),
+    baselineP95: quantile(
+      samples.map((r) => r.baseline),
+      0.95,
+    ),
+    candidateP95: quantile(
+      samples.map((r) => r.candidate),
+      0.95,
+    ),
+    ratioCi95,
+    deltaCi95,
+    verdict:
+      clusters.length < 8
+        ? 'insufficient-sessions'
+        : ratioCi95[0] > 1 + relative && deltaCi95[0] > absolute
+          ? 'regression'
+          : ratioCi95[1] <= 1 + relative || deltaCi95[1] <= absolute
+            ? 'within-threshold'
+            : 'inconclusive',
+  };
+}
+
+export function summarize(records, sessions, rounds, names) {
+  assert(
+    [sessions, rounds].every((n) => Number.isInteger(n) && n > 0),
+    'Invalid experiment size',
+  );
+  assert(names.length > 0 && new Set(names).size === names.length, 'Invalid workload list');
+  assert.equal(records.length, sessions * rounds, 'Incomplete measurement set');
+  const identities = new Set();
+  for (const record of records) {
+    assert(
+      Number.isInteger(record.session) &&
+        Number.isInteger(record.round) &&
+        record.session >= 0 &&
+        record.session < sessions &&
+        record.round >= 0 &&
+        record.round < rounds,
+      'Invalid sample identity',
+    );
+    const key = `${record.session}/${record.round}`;
+    assert(!identities.has(key), 'Duplicate sample');
+    identities.add(key);
+    for (const variant of ['baseline', 'candidate']) {
+      for (const name of names) assert(record[variant][name], `Missing ${variant}/${name}`);
+    }
+  }
+  const metrics = names.map((name) => ({
+    name,
+    ...compare(
+      records.map((r) => ({
+        session: r.session,
+        baseline: r.baseline[name].ms,
+        candidate: r.candidate[name].ms,
+      })),
+    ),
+  }));
+  for (const name of ['startup', 'first_query']) {
+    metrics.push({
+      name,
+      ...compare(
+        records
+          .filter((r) => r.round === 0)
+          .map((r) => ({
+            session: r.session,
+            baseline: r.baseline[name].ms,
+            candidate: r.candidate[name].ms,
+          })),
+      ),
+    });
+  }
+  return metrics;
+}
+
+export function formatReport(report) {
+  const percent = (r) => `${r >= 1 ? '+' : ''}${((r - 1) * 100).toFixed(1)}%`;
+  const table = report.metrics
+    .map(
+      (r) =>
+        `| ${r.name} | ${r.baselineMedian.toFixed(1)} → ${r.candidateMedian.toFixed(1)} | ${r.baselineP95.toFixed(1)} → ${r.candidateP95.toFixed(1)} | ${percent(r.ratio)} (${r.ratioCi95.map(percent).join(' to ')}) | ${r.verdict} |`,
+    )
+    .join('\n');
+  const resources = [
+    ['cpuMs', 'Server CPU seconds', 1000],
+    ['peakRssKiB', 'Server peak RSS MiB', 1024],
+  ]
+    .map(
+      ([key, label, divisor]) =>
+        `| ${label} | ${['baseline', 'candidate']
+          .map((variant) =>
+            (
+              quantile(
+                report.finalResources.map((r) => r[variant][key]),
+                0.5,
+              ) / divisor
+            ).toFixed(2),
+          )
+          .join(' → ')} |`,
+    )
+    .join('\n');
+  const payloads = ['small_source', 'large_source', 'context', 'tools_list']
+    .map(
+      (name) =>
+        `| ${name} | ${['baseline', 'candidate']
+          .map((variant) =>
+            quantile(
+              report.records.map((r) => r[variant][name].bytes),
+              0.5,
+            ).toFixed(0),
+          )
+          .join(' → ')} |`,
+    )
+    .join('\n');
+  return `## MCP query performance
+
+Base \`${report.runtimes.baseline.sha}\`; candidate \`${report.runtimes.candidate.sha}\`.
+
+${report.sessions} fresh process pairs × ${report.rounds} rounds. Real MCP stdio, one shared keyword-search index, alternating requests. No model calls.
+
+| Workload | Median ms, base → candidate | p95 ms, base → candidate | Paired change (95% interval) | Assessment |
+| --- | ---: | ---: | --- | --- |
+${table}
+
+| Resource | Median, base → candidate |
+| --- | ---: |
+${resources}
+
+| Response | Median UTF-8 bytes, base → candidate |
+| --- | ---: |
+${payloads}
+
+Intervals resample whole process pairs (10,000 draws). The regression threshold is >5% AND >5 ms; only a confirmed regression fails timing. Inconclusive is not proof of no slowdown. p95, CPU and RSS are descriptive. Startup uses fresh processes with warm filesystem caches.
+
+Correctness, source budget, incomplete samples, and request errors always fail. JSON contains every response size, source-character count, backend phase timing, final server CPU/peak RSS, environment and fixture fingerprint; raw.jsonl retains every completed pair. These generated fixtures do not measure agent dollar savings or all repository workloads.
+`;
+}

gitnexus/test/unit/mcp-query-benchmark.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from 'vitest';
+
+const { compare, summarize } = await import(
+  new URL('../../bench/mcp-query/statistics.mjs', import.meta.url).href
+);
+const samples = (ratio: number, baseline = 100) =>
+  Array.from({ length: 8 }, (_, session) => ({ session, baseline, candidate: baseline * ratio }));
+
+describe('MCP performance regression decision', () => {
+  it('detects a consistent material slowdown', () => {
+    expect(compare(samples(1.2)).verdict).toBe('regression');
+  });
+
+  it('accepts faster requests and negligible absolute overhead', () => {
+    expect(compare(samples(0.8)).verdict).toBe('within-threshold');
+    expect(compare(samples(2, 1)).verdict).toBe('within-threshold');
+  });
+
+  it('does not call process variation a proven improvement or regression', () => {
+    const rows = samples(1).map((row, i) => ({ ...row, candidate: i % 2 ? 150 : 70 }));
+    expect(compare(rows).verdict).toBe('inconclusive');
+  });
+
+  it('does not gain confidence by repeating correlated requests in one process', () => {
+    const rows = samples(1).map((row, i) => ({ ...row, candidate: i % 2 ? 150 : 70 }));
+    const repeated = rows.flatMap((row) => Array.from({ length: 5 }, () => ({ ...row })));
+    const original = compare(rows).ratioCi95;
+    compare(repeated).ratioCi95.forEach((value: number, i: number) =>
+      expect(value).toBeCloseTo(original[i], 12),
+    );
+    expect(compare(samples(2).map((row) => ({ ...row, session: 0 }))).verdict).toBe(
+      'insufficient-sessions',
+    );
+  });
+
+  it.each([0, -1, NaN, Infinity, undefined])('rejects invalid measurements (%s)', (value) => {
+    expect(() => compare([{ session: 0, baseline: 100, candidate: value }])).toThrow();
+  });
+
+  it('rejects empty, incomplete, duplicate, and missing workload evidence', () => {
+    expect(() => compare([])).toThrow();
+    expect(() => summarize([], 8, 5, ['query'])).toThrow('Incomplete');
+    const record = {
+      session: 0,
+      round: 0,
+      baseline: { query: { ms: 100 } },
+      candidate: { query: { ms: 110 } },
+    };
+    expect(() => summarize([record, record], 1, 2, ['query'])).toThrow('Duplicate');
+    expect(() => summarize([record], 1, 1, ['context'])).toThrow('Missing');
+    expect(() => summarize([record], 1, 1, [])).toThrow('workload');
+    expect(() => summarize([{ ...record, round: 0.5 }], 1, 1, ['query'])).toThrow('identity');
+  });
+});