openpetswithchatandmcp/website/scripts/run-ep07-validation-sweep.mjs

#!/usr/bin/env node

import { spawnSync } from 'node:child_process';
import { mkdirSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const DEFAULT_ITERATIONS = 3;
const DEFAULT_REPORT_BASENAME = 'ep07-validation-sweep-last';

function parseIterations(argv) {
  for (const arg of argv) {
    if (arg.startsWith('--iterations=')) {
      const raw = arg.slice('--iterations='.length).trim();
      const parsed = Number.parseInt(raw, 10);
      if (Number.isFinite(parsed) && parsed > 0) return parsed;
    }
  }
  return DEFAULT_ITERATIONS;
}

function parseWriteReport(argv) {
  return !argv.includes('--no-report');
}

function parseReportBasename(argv) {
  for (const arg of argv) {
    if (arg.startsWith('--report-basename=')) {
      const raw = arg.slice('--report-basename='.length).trim();
      if (raw) return raw;
    }
  }
  return DEFAULT_REPORT_BASENAME;
}

function formatMs(ms) {
  return `${(ms / 1000).toFixed(2)}s`;
}

function buildCommandString(command, args) {
  return [command, ...args].join(' ');
}

function runCommand(label, command, args) {
  const startedAt = Date.now();
  const startedAtIso = new Date(startedAt).toISOString();
  console.log(`\n[EP-07 sweep] ${label}`);
  const child = spawnSync(command, args, {
    stdio: 'inherit',
    shell: true,
  });
  const endedAt = Date.now();
  const endedAtIso = new Date(endedAt).toISOString();
  const durationMs = Date.now() - startedAt;
  const exitCode = child.status ?? 1;
  const ok = exitCode === 0;
  if (ok) {
    console.log(`[EP-07 sweep] ${label} passed in ${formatMs(durationMs)}.`);
  } else {
    console.error(`[EP-07 sweep] ${label} failed after ${formatMs(durationMs)}.`);
  }
  return {
    label,
    command,
    args,
    commandLine: buildCommandString(command, args),
    ok,
    exitCode,
    startedAtIso,
    endedAtIso,
    durationMs,
  };
}

function renderMarkdownReport(report) {
  const lines = [];
  lines.push('# EP-07 Validation Sweep Report');
  lines.push('');
  lines.push(`- Generated at: ${report.generatedAtIso}`);
  lines.push(`- Overall status: ${report.overallStatus}`);
  lines.push(`- Iterations requested: ${report.iterationsRequested}`);
  lines.push(`- Iterations completed: ${report.iterationsCompleted}`);
  lines.push(`- Total duration: ${formatMs(report.totalDurationMs)}`);
  lines.push(`- Iteration duration (avg/min/max): ${formatMs(report.iterationDuration.avgMs)} / ${formatMs(report.iterationDuration.minMs)} / ${formatMs(report.iterationDuration.maxMs)}`);
  lines.push('');
  lines.push('## Iterations');
  lines.push('');
  for (const iteration of report.iterations) {
    lines.push(`### Iteration ${iteration.iterationIndex}`);
    lines.push('');
    lines.push(`- Status: ${iteration.status}`);
    lines.push(`- Duration: ${formatMs(iteration.durationMs)}`);
    lines.push('');
    lines.push('| Step | Command | Status | Duration |');
    lines.push('| --- | --- | --- | --- |');
    for (const step of iteration.steps) {
      lines.push(`| ${step.label} | \`${step.commandLine}\` | ${step.ok ? 'passed' : `failed (${step.exitCode})`} | ${formatMs(step.durationMs)} |`);
    }
    lines.push('');
  }
  if (report.failure) {
    lines.push('## Failure');
    lines.push('');
    lines.push(`- Iteration: ${report.failure.iterationIndex}`);
    lines.push(`- Step: ${report.failure.stepLabel}`);
    lines.push(`- Exit code: ${report.failure.exitCode}`);
    lines.push('');
  }
  return `${lines.join('\n')}\n`;
}

function writeEvidenceReport(report, basename) {
  const scriptDir = path.dirname(fileURLToPath(import.meta.url));
  const repoRoot = path.resolve(scriptDir, '..', '..');
  const evidenceDir = path.join(repoRoot, 'docs', 'evidence', 'cross-corpus');
  mkdirSync(evidenceDir, { recursive: true });

  const jsonPath = path.join(evidenceDir, `${basename}.json`);
  const mdPath = path.join(evidenceDir, `${basename}.md`);
  writeFileSync(jsonPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
  writeFileSync(mdPath, renderMarkdownReport(report), 'utf8');

  console.log(`[EP-07 sweep] evidence report written: ${jsonPath}`);
  console.log(`[EP-07 sweep] evidence report written: ${mdPath}`);
}

function main() {
  const argv = process.argv.slice(2);
  const iterations = parseIterations(argv);
  const shouldWriteReport = parseWriteReport(argv);
  const reportBasename = parseReportBasename(argv);
  console.log(`[EP-07 sweep] starting validation sweep with ${iterations} iteration(s).`);

  const report = {
    generatedAtIso: new Date().toISOString(),
    overallStatus: 'passed',
    iterationsRequested: iterations,
    iterationsCompleted: 0,
    totalDurationMs: 0,
    iterationDuration: {
      avgMs: 0,
      minMs: 0,
      maxMs: 0,
    },
    iterations: [],
    failure: null,
  };
  const startedAt = Date.now();
  let failureExitCode = 0;

  for (let i = 1; i <= iterations; i += 1) {
    console.log(`\n[EP-07 sweep] iteration ${i}/${iterations}`);
    const iterationStartedAt = Date.now();
    const iterationRecord = {
      iterationIndex: i,
      status: 'passed',
      durationMs: 0,
      steps: [],
    };

    const serverStep = runCommand(
      'server verify-suite/stability deterministic suite',
      'npm',
      [
        '--prefix',
        '../ScriptoriumAI-Server',
        'test',
        '--',
        '__tests__/corpus.engine-gate-verify-suite.test.cjs',
        '__tests__/corpus.engine-gate-verify-suite.cli.test.cjs',
        '__tests__/corpus.engine-gate-verify-suite.cli.e2e.test.cjs',
        '__tests__/corpus.engine-gate-verify-suite-stability.test.cjs',
        '__tests__/corpus.engine-gate-verify-suite-stability.cli.e2e.test.cjs',
        '__tests__/corpus.engine-gate-verify-suite.e2e.test.cjs',
      ],
    );
    iterationRecord.steps.push(serverStep);
    if (!serverStep.ok) {
      iterationRecord.status = 'failed';
      report.overallStatus = 'failed';
      report.failure = {
        iterationIndex: i,
        stepLabel: serverStep.label,
        exitCode: serverStep.exitCode,
      };
      failureExitCode = serverStep.exitCode;
      iterationRecord.durationMs = Date.now() - iterationStartedAt;
      report.iterations.push(iterationRecord);
      break;
    }

    const uiContractStepSpecs = [
      {
        label: 'ui runboard verify-suite contracts',
        args: [
          'vitest',
          'run',
          'src/__tests__/Runboard.states.test.tsx',
          '--pool=forks',
          '--no-file-parallelism',
        ],
      },
      {
        label: 'ui command-palette verify-suite contracts',
        args: [
          'vitest',
          'run',
          'src/__tests__/command-palette.test.tsx',
          '--pool=forks',
          '--no-file-parallelism',
        ],
      },
      {
        label: 'ui typed-client verify-suite contracts',
        args: [
          'vitest',
          'run',
          'src/__tests__/scriptorium-client.test.ts',
          '--pool=forks',
          '--no-file-parallelism',
        ],
      },
    ];
    let uiFailureStep = null;
    for (const uiSpec of uiContractStepSpecs) {
      const uiStep = runCommand(uiSpec.label, 'npx', uiSpec.args);
      iterationRecord.steps.push(uiStep);
      if (!uiStep.ok) {
        uiFailureStep = uiStep;
        break;
      }
    }
    if (uiFailureStep) {
      iterationRecord.status = 'failed';
      report.overallStatus = 'failed';
      report.failure = {
        iterationIndex: i,
        stepLabel: uiFailureStep.label,
        exitCode: uiFailureStep.exitCode,
      };
      failureExitCode = uiFailureStep.exitCode;
      iterationRecord.durationMs = Date.now() - iterationStartedAt;
      report.iterations.push(iterationRecord);
      break;
    }

    const typeCheckStep = runCommand('ui type-check', 'npm', ['run', 'type-check']);
    iterationRecord.steps.push(typeCheckStep);
    if (!typeCheckStep.ok) {
      iterationRecord.status = 'failed';
      report.overallStatus = 'failed';
      report.failure = {
        iterationIndex: i,
        stepLabel: typeCheckStep.label,
        exitCode: typeCheckStep.exitCode,
      };
      failureExitCode = typeCheckStep.exitCode;
      iterationRecord.durationMs = Date.now() - iterationStartedAt;
      report.iterations.push(iterationRecord);
      break;
    }

    const iterationDurationMs = Date.now() - iterationStartedAt;
    iterationRecord.durationMs = iterationDurationMs;
    report.iterations.push(iterationRecord);
    report.iterationsCompleted += 1;
    console.log(`[EP-07 sweep] iteration ${i}/${iterations} passed in ${formatMs(iterationDurationMs)}.`);
  }

  const totalMs = Date.now() - startedAt;
  report.totalDurationMs = totalMs;
  if (report.iterations.length > 0) {
    const iterationDurations = report.iterations.map((iteration) => iteration.durationMs);
    const avgMs = iterationDurations.reduce((sum, value) => sum + value, 0) / iterationDurations.length;
    report.iterationDuration = {
      avgMs,
      minMs: Math.min(...iterationDurations),
      maxMs: Math.max(...iterationDurations),
    };
  }

  if (report.overallStatus === 'failed') {
    console.error('\n[EP-07 sweep] failed.');
    console.error(`[EP-07 sweep] iterations completed: ${report.iterationsCompleted}/${report.iterationsRequested}`);
  } else {
    console.log('\n[EP-07 sweep] success.');
    console.log(`[EP-07 sweep] iterations: ${iterations}`);
    console.log(`[EP-07 sweep] total: ${formatMs(totalMs)}`);
    console.log(
      `[EP-07 sweep] avg: ${formatMs(report.iterationDuration.avgMs)} | min: ${formatMs(report.iterationDuration.minMs)} | max: ${formatMs(report.iterationDuration.maxMs)}`,
    );
  }

  if (shouldWriteReport) {
    writeEvidenceReport(report, reportBasename);
  }

  if (failureExitCode !== 0) {
    process.exit(failureExitCode);
  }
}

main();