openpetswithchatandmcp/website/scripts/check-performance-budgets.mjs

688 lines
21 KiB
JavaScript
Executable file

import { spawn } from 'node:child_process'
import { once } from 'node:events'
import { promises as fs } from 'node:fs'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { chromium } from '@playwright/test'
const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
const uiRoot = path.resolve(__dirname, '..')
const repoRoot = path.resolve(uiRoot, '..')
const budgetsPath = path.join(uiRoot, 'config', 'performance-budgets.json')
const distAssetsDir = path.join(uiRoot, 'dist', 'assets')
const evidenceDir = path.join(repoRoot, 'docs', 'evidence', 'cross-corpus')
const outputJsonPath = path.join(evidenceDir, 'corpus-ui-performance-budget-check.json')
const outputMdPath = path.join(evidenceDir, 'corpus-ui-performance-budget-check.md')
const previewHost = process.env.PERF_BUDGET_HOST || '127.0.0.1'
const previewPort = Number(process.env.PERF_BUDGET_PORT || '4173')
const baseUrl = `http://${previewHost}:${previewPort}`
const perfBudgetDebug = process.env.PERF_BUDGET_DEBUG === '1'
const editorRoutePath = '/app/editor'
const runboardRoutePath = '/app/runboard'
const platformAuthStorageKey = 'familiaros.platform.user.v1'
const performanceFallbackUser = {
id: 'perf_budget_operator',
name: 'Performance Budget Operator',
email: 'perf.operator@familiar-os.local',
authMethod: 'email',
plan: 'free',
}
function round(value) {
return Number(value.toFixed(2))
}
function toKiB(bytes) {
return round(bytes / 1024)
}
function statusForThreshold(value, max) {
return value <= max ? 'pass' : 'fail'
}
async function waitForServer(url, timeoutMs = 60_000) {
const startedAt = Date.now()
while (Date.now() - startedAt < timeoutMs) {
try {
const response = await fetch(url)
if (response.ok || response.status < 500) return
} catch {
// Retry until timeout.
}
await new Promise((resolve) => setTimeout(resolve, 500))
}
throw new Error(`Timed out waiting for preview server at ${url}`)
}
async function stopProcess(child) {
if (!child || child.killed) return
if (process.platform === 'win32' && child.pid) {
try {
const killer = spawn('taskkill', ['/PID', String(child.pid), '/T', '/F'], {
stdio: 'ignore',
})
await Promise.race([
once(killer, 'exit').catch(() => {}),
new Promise((resolve) => setTimeout(resolve, 5_000)),
])
} catch {
// Fall back to child.kill below.
}
}
try {
child.kill('SIGTERM')
} catch {
// Ignore kill errors during cleanup.
}
await Promise.race([
once(child, 'exit').catch(() => {}),
new Promise((resolve) =>
setTimeout(() => {
try {
if (!child.killed) child.kill('SIGKILL')
} catch {
// Ignore kill errors during forced cleanup.
}
resolve()
}, 12_000)
),
])
}
function startPreviewServer() {
const previewCommand = `npm run preview -- --host ${previewHost} --port ${previewPort} --strictPort`
const child = spawn(previewCommand, {
cwd: uiRoot,
stdio: ['ignore', 'pipe', 'pipe'],
env: process.env,
shell: true,
})
child.stdout.on('data', () => {})
child.stderr.on('data', () => {})
child.on('error', (error) => {
console.error(`[performance-budgets] Failed to start preview server: ${error.message}`)
})
return child
}
function jsonResponse(body) {
return {
status: 200,
contentType: 'application/json',
body: JSON.stringify(body),
}
}
async function mockRunboardApis(page) {
const runId = 'run-perf-001'
const projectId = 'project-perf'
const nowIso = '2026-02-22T15:00:00.000Z'
let runsCallCount = 0
const run = {
run_id: runId,
kind: 'compile',
project_id: projectId,
status: 'failed',
failure_reason: 'timeout during compile stage',
retry_count: 0,
attempts_made: 1,
max_attempts: 3,
created_at: nowIso,
updated_at: nowIso,
started_at: nowIso,
completed_at: nowIso,
duration_ms: 3200,
run_signature: 'sig-run-perf-001',
summary: {
artifacts_total: 2,
artifacts_by_status: {
failed: 1,
ready: 1,
},
latest_artifact_at: nowIso,
timing: {
created_at: nowIso,
started_at: nowIso,
completed_at: nowIso,
total_duration_ms: 3200,
queue_to_start_ms: 120,
queue_wait_ms: 120,
run_active_ms: 3080,
execution_ms: 2900,
time_to_first_artifact_ms: 800,
time_to_first_ready_artifact_ms: 1000,
execution_to_first_artifact_ms: 700,
execution_to_last_artifact_ms: 2600,
artifact_publish_span_ms: 1800,
completion_to_last_artifact_ms: 0,
pipeline_total_ms: 3200,
},
integrity: {
artifacts_failed_count: 1,
artifacts_missing_uri_count: 0,
artifacts_missing_checksum_count: 0,
terminal_without_artifacts_count: 0,
running_without_start_timestamp_count: 0,
},
},
}
const artifacts = [
{
id: 'artifact-run-perf-001-failed',
run_id: runId,
project_id: projectId,
type: 'pdf',
status: 'failed',
uri: null,
checksum: 'sha256:deadbeef',
created_at: nowIso,
updated_at: nowIso,
},
{
id: 'artifact-run-perf-001-ready',
run_id: runId,
project_id: projectId,
type: 'log',
status: 'ready',
uri: 'https://example.invalid/artifact-run-perf-001-ready.log',
checksum: 'sha256:cafebabe',
created_at: nowIso,
updated_at: nowIso,
},
]
const auditEvents = [
{
event_id: 'audit-evt-perf-1',
event_type: 'run_state',
run_id: runId,
artifact_id: null,
event_at: nowIso,
payload: {
status: 'failed',
kind: 'compile',
},
},
]
await page.route(/\/api\/corpus\/runs\/run-perf-001\/summary(\?.*)?$/, async (route) => {
await route.fulfill(
jsonResponse({
ok: true,
run_id: runId,
run_signature: run.run_signature,
summary: run.summary,
})
)
})
await page.route(/\/api\/corpus\/runs(\?.*)?$/, async (route) => {
runsCallCount += 1
await new Promise((resolve) => setTimeout(resolve, 75))
await route.fulfill(
jsonResponse({
ok: true,
total: 1,
count: 1,
limit: 200,
offset: 0,
has_more: false,
sort_by: 'updated_at',
sort_order: 'desc',
max_limit: 200,
runs: [run],
})
)
})
await page.route(/\/api\/corpus\/artifacts(\?.*)?$/, async (route) => {
await route.fulfill(
jsonResponse({
ok: true,
total: artifacts.length,
count: artifacts.length,
limit: 250,
offset: 0,
has_more: false,
sort_by: 'updated_at',
sort_order: 'desc',
max_limit: 500,
artifacts,
})
)
})
await page.route(/\/api\/corpus\/export\/audit(\?.*)?$/, async (route) => {
await route.fulfill(
jsonResponse({
ok: true,
schema_version: 1,
generated_at: nowIso,
export: {
kind: 'audit',
total: auditEvents.length,
count: auditEvents.length,
limit: 200,
offset: 0,
has_more: false,
sort_by: 'event_at',
sort_order: 'desc',
max_limit: 500,
},
events: auditEvents,
})
)
})
return {
runId,
getRunsCallCount: () => runsCallCount,
}
}
function normalizeAssetBytes(responseHeaders, fallbackBytes = 0) {
const rawContentLength = responseHeaders?.['content-length']
const parsed = Number(rawContentLength)
if (Number.isFinite(parsed) && parsed >= 0) return parsed
return fallbackBytes
}
async function waitForRunboardRunsBusyState(page, expectedBusy, timeoutMs, timeoutErrorMessage) {
try {
await page.waitForFunction(
({ busy }) => {
const list = document.querySelector('[data-testid="runboard-runs-list"]')
if (!list) return false
return list.getAttribute('aria-busy') === (busy ? 'true' : 'false')
},
{ busy: expectedBusy },
{ timeout: timeoutMs }
)
return true
} catch {
if (perfBudgetDebug) {
console.warn(`[performance-budgets] ${timeoutErrorMessage}`)
}
return false
}
}
async function waitForRunsCallIncrement(page, getRunsCallCount, baselineCount, timeoutMs) {
const startedAt = Date.now()
while (getRunsCallCount() <= baselineCount) {
if (Date.now() - startedAt > timeoutMs) return false
await page.waitForTimeout(25)
}
return true
}
async function measureInteractionLatency() {
const browser = await chromium.launch({ headless: true })
const context = await browser.newContext({
viewport: { width: 1440, height: 900 },
locale: 'en-US',
timezoneId: 'UTC',
colorScheme: 'dark',
reducedMotion: 'reduce',
})
const page = await context.newPage()
const observedAssets = new Map()
page.on('response', async (response) => {
let url
try {
url = new URL(response.url())
} catch {
return
}
if (url.origin !== baseUrl) return
if (!url.pathname.includes('/assets/')) return
const kind = url.pathname.endsWith('.js') ? 'js' : url.pathname.endsWith('.css') ? 'css' : null
if (!kind) return
let bytes = normalizeAssetBytes(response.headers(), 0)
if (!bytes) {
try {
const body = await response.body()
bytes = normalizeAssetBytes(response.headers(), body.byteLength)
} catch {
bytes = 0
}
}
observedAssets.set(url.pathname, { name: path.basename(url.pathname), kind, bytes })
})
await page.addInitScript(
({ storageKey, userJson }) => {
localStorage.clear()
sessionStorage.clear()
localStorage.setItem(storageKey, userJson)
},
{
storageKey: platformAuthStorageKey,
userJson: JSON.stringify(performanceFallbackUser),
}
)
try {
console.log('[performance-budgets] Open editor route')
await page.goto(`${baseUrl}${editorRoutePath}`, { waitUntil: 'domcontentloaded' })
await page.getByTestId('editor-workspace').waitFor({ state: 'visible' })
const insightPane = page.getByTestId('editor-insight-pane')
await insightPane.waitFor({ state: 'visible' })
const insightsTabButton = insightPane.getByRole('button', { name: /^Insights$/ })
const previewTabButton = insightPane.getByRole('button', { name: /^Preview$/ })
await insightsTabButton.waitFor({ state: 'visible' })
console.log('[performance-budgets] Measure editor pane switch')
const startToInsights = await page.evaluate(() => performance.now())
await insightsTabButton.click()
await page.getByText('Workspace Context').waitFor({ state: 'visible', timeout: 10_000 })
const endToInsights = await page.evaluate(() => performance.now())
const startToPreview = await page.evaluate(() => performance.now())
await previewTabButton.click()
await page.getByText('Workspace Context').waitFor({ state: 'hidden', timeout: 10_000 })
const endToPreview = await page.evaluate(() => performance.now())
const editorRightPaneSwitchMs = round(
Math.max(endToInsights - startToInsights, endToPreview - startToPreview)
)
console.log('[performance-budgets] Open runboard route')
const runboardMock = await mockRunboardApis(page)
await page.goto(`${baseUrl}${runboardRoutePath}`, { waitUntil: 'domcontentloaded' })
await page.getByRole('heading', { name: 'Runboard' }).waitFor({ state: 'visible' })
const runboardRefreshButton = page.getByTestId('runboard-runs-refresh')
await runboardRefreshButton.waitFor({ state: 'visible' })
const runsListVisible = await page
.getByTestId('runboard-runs-list')
.waitFor({ state: 'visible', timeout: 3_000 })
.then(() => true)
.catch(() => false)
if (runsListVisible) {
await waitForRunboardRunsBusyState(
page,
false,
1_500,
'Timed out waiting for initial runboard runs-list hydration to settle during performance budget check.'
)
}
console.log('[performance-budgets] Measure runboard refresh')
const baselineRunsCallCount = runboardMock.getRunsCallCount()
const startRefresh = await page.evaluate(() => performance.now())
await runboardRefreshButton.click()
let settled = false
let enteredBusy = false
if (runsListVisible) {
enteredBusy = await waitForRunboardRunsBusyState(
page,
true,
400,
'Timed out waiting for runboard refresh to enter loading state during performance budget check.'
)
const exitedBusy = enteredBusy
? await waitForRunboardRunsBusyState(
page,
false,
700,
'Timed out waiting for runboard refresh to settle during performance budget check.'
)
: false
settled = enteredBusy && exitedBusy
}
if (!settled) {
if (enteredBusy) {
await page.waitForTimeout(200)
} else {
const observedRefreshCall = await waitForRunsCallIncrement(
page,
runboardMock.getRunsCallCount,
baselineRunsCallCount,
500
)
if (!observedRefreshCall) {
await page.waitForTimeout(250)
}
}
}
const endRefresh = await page.evaluate(() => performance.now())
const runboardRefreshMs = round(endRefresh - startRefresh)
const observedEntries = [...observedAssets.values()]
const observedJsAssets = observedEntries.filter((entry) => entry.kind === 'js')
const observedCssAssets = observedEntries.filter((entry) => entry.kind === 'css')
const observedLargestJs = observedJsAssets.reduce(
(largest, current) => (current.bytes > largest.bytes ? current : largest),
{ name: '-', kind: 'js', bytes: 0 }
)
return {
editor_right_pane_switch_ms: editorRightPaneSwitchMs,
runboard_refresh_ms: runboardRefreshMs,
runboard_refresh_calls_observed: runboardMock.getRunsCallCount(),
run_id: runboardMock.runId,
bundle_observed: {
total_js_kb: toKiB(observedJsAssets.reduce((sum, entry) => sum + entry.bytes, 0)),
total_css_kb: toKiB(observedCssAssets.reduce((sum, entry) => sum + entry.bytes, 0)),
largest_js_kb: toKiB(observedLargestJs.bytes),
largest_js_asset: observedLargestJs.name,
js_assets_count: observedJsAssets.length,
css_assets_count: observedCssAssets.length,
},
}
} finally {
await context.close()
await browser.close()
}
}
function formatMarkdownReport(report) {
const checkRows = report.checks
.map(
(check) =>
`| ${check.id} | ${check.measured} | ${check.max} | ${check.status.toUpperCase()} |`
)
.join('\n')
return [
'# Corpus UI Performance Budget Check',
'',
`Generated: ${report.generated_at}`,
`Status: **${report.summary.status.toUpperCase()}**`,
'',
'## Bundle Metrics',
'',
`- Route-observed total JS (KiB): ${report.measurements.bundle.total_js_kb}`,
`- Route-observed largest JS (KiB): ${report.measurements.bundle.largest_js_kb}`,
`- Route-observed total CSS (KiB): ${report.measurements.bundle.total_css_kb}`,
`- Static-dist total JS (KiB): ${report.measurements.bundle_static.total_js_kb}`,
`- Static-dist total CSS (KiB): ${report.measurements.bundle_static.total_css_kb}`,
'',
'## Interaction Metrics',
'',
`- Editor right-pane switch (ms): ${report.measurements.interaction.editor_right_pane_switch_ms}`,
`- Runboard refresh (ms): ${report.measurements.interaction.runboard_refresh_ms}`,
'',
'## Budget Results',
'',
'| Check | Measured | Budget Max | Status |',
'|---|---:|---:|---|',
checkRows,
'',
'## Evidence Files',
'',
`- JSON: \`${path.relative(repoRoot, outputJsonPath).replaceAll('\\', '/')}\``,
`- Markdown: \`${path.relative(repoRoot, outputMdPath).replaceAll('\\', '/')}\``,
'',
].join('\n')
}
async function main() {
const budgets = JSON.parse(await fs.readFile(budgetsPath, 'utf8'))
await fs.mkdir(evidenceDir, { recursive: true })
let distEntries
try {
distEntries = await fs.readdir(distAssetsDir, { withFileTypes: true })
} catch {
throw new Error(
`Missing build assets at ${distAssetsDir}. Run "npm run build" in familiaros-website first.`
)
}
const assets = []
for (const entry of distEntries) {
if (!entry.isFile()) continue
const fullPath = path.join(distAssetsDir, entry.name)
const stats = await fs.stat(fullPath)
assets.push({
name: entry.name,
bytes: stats.size,
kb: toKiB(stats.size),
})
}
const jsAssets = assets.filter((asset) => asset.name.endsWith('.js'))
const cssAssets = assets.filter((asset) => asset.name.endsWith('.css'))
const largestJsAsset = jsAssets.reduce(
(largest, current) => (current.bytes > largest.bytes ? current : largest),
{ name: '-', bytes: 0, kb: 0 }
)
const totalJsKb = round(jsAssets.reduce((sum, asset) => sum + asset.kb, 0))
const totalCssKb = round(cssAssets.reduce((sum, asset) => sum + asset.kb, 0))
const previewServer = startPreviewServer()
console.log(`[performance-budgets] Starting preview server on ${baseUrl}`)
try {
await waitForServer(baseUrl)
console.log('[performance-budgets] Preview server ready')
} catch (error) {
await stopProcess(previewServer)
throw error
}
let interactionMetrics
try {
console.log('[performance-budgets] Measuring interaction latency')
interactionMetrics = await measureInteractionLatency()
} finally {
console.log('[performance-budgets] Stopping preview server')
await stopProcess(previewServer)
}
const checks = [
{
id: 'bundle.total_js_kb',
measured: interactionMetrics.bundle_observed?.total_js_kb ?? totalJsKb,
max: budgets.bundle.max_total_js_kb,
status: statusForThreshold(
interactionMetrics.bundle_observed?.total_js_kb ?? totalJsKb,
budgets.bundle.max_total_js_kb
),
},
{
id: 'bundle.largest_js_kb',
measured: interactionMetrics.bundle_observed?.largest_js_kb ?? largestJsAsset.kb,
max: budgets.bundle.max_largest_js_kb,
status: statusForThreshold(
interactionMetrics.bundle_observed?.largest_js_kb ?? largestJsAsset.kb,
budgets.bundle.max_largest_js_kb
),
},
{
id: 'bundle.total_css_kb',
measured: interactionMetrics.bundle_observed?.total_css_kb ?? totalCssKb,
max: budgets.bundle.max_total_css_kb,
status: statusForThreshold(
interactionMetrics.bundle_observed?.total_css_kb ?? totalCssKb,
budgets.bundle.max_total_css_kb
),
},
{
id: 'interaction.editor_right_pane_switch_ms',
measured: interactionMetrics.editor_right_pane_switch_ms,
max: budgets.interaction.max_editor_right_pane_switch_ms,
status: statusForThreshold(
interactionMetrics.editor_right_pane_switch_ms,
budgets.interaction.max_editor_right_pane_switch_ms
),
},
{
id: 'interaction.runboard_refresh_ms',
measured: interactionMetrics.runboard_refresh_ms,
max: budgets.interaction.max_runboard_refresh_ms,
status: statusForThreshold(
interactionMetrics.runboard_refresh_ms,
budgets.interaction.max_runboard_refresh_ms
),
},
]
const failedChecks = checks.filter((check) => check.status !== 'pass')
const report = {
schema_version: 1,
generated_at: new Date().toISOString(),
budget_source: path.relative(repoRoot, budgetsPath).replaceAll('\\', '/'),
budgets,
measurements: {
bundle: {
total_js_kb: interactionMetrics.bundle_observed?.total_js_kb ?? totalJsKb,
total_css_kb: interactionMetrics.bundle_observed?.total_css_kb ?? totalCssKb,
largest_js_kb: interactionMetrics.bundle_observed?.largest_js_kb ?? largestJsAsset.kb,
largest_js_asset: interactionMetrics.bundle_observed?.largest_js_asset ?? largestJsAsset.name,
js_assets_count: interactionMetrics.bundle_observed?.js_assets_count ?? jsAssets.length,
css_assets_count: interactionMetrics.bundle_observed?.css_assets_count ?? cssAssets.length,
},
bundle_static: {
total_js_kb: totalJsKb,
total_css_kb: totalCssKb,
largest_js_kb: largestJsAsset.kb,
largest_js_asset: largestJsAsset.name,
js_assets_count: jsAssets.length,
css_assets_count: cssAssets.length,
},
interaction: interactionMetrics,
},
checks,
summary: {
status: failedChecks.length === 0 ? 'pass' : 'fail',
passed_checks: checks.length - failedChecks.length,
failed_checks: failedChecks.length,
total_checks: checks.length,
},
}
await fs.writeFile(outputJsonPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8')
await fs.writeFile(outputMdPath, `${formatMarkdownReport(report)}\n`, 'utf8')
console.log(
`[performance-budgets] ${report.summary.status.toUpperCase()} (${report.summary.passed_checks}/${report.summary.total_checks} checks passed)`
)
console.log(`[performance-budgets] JSON: ${path.relative(repoRoot, outputJsonPath).replaceAll('\\', '/')}`)
console.log(`[performance-budgets] MD: ${path.relative(repoRoot, outputMdPath).replaceAll('\\', '/')}`)
if (failedChecks.length > 0) {
console.error('[performance-budgets] Failed checks:')
for (const check of failedChecks) {
console.error(` - ${check.id}: measured=${check.measured}, max=${check.max}`)
}
process.exitCode = 1
}
}
main().catch((error) => {
console.error(`[performance-budgets] ${error.message}`)
process.exitCode = 1
})