diff --git a/docs/audits/NATIVE-UI-CONFORMANCE-1387.md b/docs/audits/NATIVE-UI-CONFORMANCE-1387.md index 036ba0bd..0f4d4744 100644 --- a/docs/audits/NATIVE-UI-CONFORMANCE-1387.md +++ b/docs/audits/NATIVE-UI-CONFORMANCE-1387.md @@ -12,6 +12,10 @@ The report hashes the entire bundle, including server/web resources and framewor `pnpm desktop:ui:verify ` rehashes the package and screenshots. It rejects incomplete/duplicate/failed entries, unbound builds, dirty candidates, wrong commits/versions/packages, stale reports, missing native environment, cropped or modified PNGs, transparent overlays, and shell geometry failures before and after interactions. It is a native-evidence verifier, not yet the release workflow's publication gate. +Primary routes also record heading, Back, header, and content rectangles. Checks enforce the shared typography, icon-only Back sizing/placement, route-title semantics, and cross-route alignment. Overlay parts record their actual computed padding against the shared rem-based inset contract, including intentionally unpadded compound/task containers. The runner's final status includes structural verification; an unexpected viewport or invalid recorded layout cannot remain a passing run merely because its clicks succeeded. + +After the normal matrix, the disposable renderer receives six deliberately injected faults: blank shell space, a clipped modal, a shifted heading, a board-only rail control on Activity, wrong popout padding, and a visible New Task button with its handler removed. Each fault must be detected by the corresponding geometry or behavioral assertion, retains a native screenshot, and is removed by reloading before the next fault. These expected-failure probes are separate from the ordinary matrix and cannot waive an ordinary failed state. Missing fault probes fail verification. + ## Development checkpoint The first complete development matrix reproduced Session menu Escape dismissal failure in both themes and both native window sizes. This is tracked separately in #1451; the runner must not waive it. Preview capture timing also exposed a harness defect: DOM content and geometry could pass while the native screenshot caught a transparent opening modal. Opacity measurement and paint synchronization address that defect, and the normal/minimum light-theme preview captures were visually rechecked. @@ -22,8 +26,7 @@ These diagnostic runs use an unsigned 6.1.6 package built from `afb447156fd77d24 ## Remaining acceptance work -- Resolve observed product failures, including #1451, and complete a passing full native run. -- Strengthen primary-heading/back alignment and overlay inset assertions, with seeded-failure coverage for those contracts and dead/context-invalid controls. +- Integrate the Session menu fix from #1453 and rerun the newly strengthened matrix. Its preceding 144-state clean candidate passed, but predates the new geometry fields and seeded renderer requirements. - Bind refreshed maintained screenshots/GIFs to their capture candidate and reject stale media. Do not relabel the existing browser captures as native evidence. - Integrate mandatory evidence verification into release validation and signed packaging before any upload. The current workflow uses `electron-builder --publish always`; adding a check after that command would be too late. - Update release and issue templates to distinguish browser, packaged candidate, installed app, signing, documentation, and publication boundaries. diff --git a/scripts/native-ui/contract.mjs b/scripts/native-ui/contract.mjs index 55fab8ef..2694748e 100644 --- a/scripts/native-ui/contract.mjs +++ b/scripts/native-ui/contract.mjs @@ -2,8 +2,17 @@ import { createHash } from 'node:crypto'; import { createReadStream } from 'node:fs'; import { lstat, readdir, readlink, realpath } from 'node:fs/promises'; import path from 'node:path'; +import { overlayInsetFailures, pageHeaderFailures } from './layout-contract.mjs'; export const schema = 'packaged-macos-ui-conformance/v1'; +export const seededCases = [ + ['shell-blank', 'shell does not fill viewport'], + ['clipped-modal', 'clipped overlay'], + ['heading-offset', 'inconsistent primary heading/Back geometry'], + ['context-invalid-header', 'context-invalid board rail control'], + ['overlay-padding', 'inconsistent overlay header padding'], + ['dead-header', 'dead header control'], +]; export const routes = [ ['activity', '/activity', 'Activity'], ['backlog', '/backlog', 'Backlog'], @@ -75,6 +84,48 @@ export function requiresOverlay(id) { ); } +// Expected production parts are keyed by the exercised surface, not whatever CSS +// selectors happen to survive a regression in that surface. +export function requiredOverlayParts(id) { + const state = id.split('/')[1]; + if (['task-chat', 'task-drawer', 'task-expanded'].includes(state)) + return ['task-header', 'task-body']; + if (state?.startsWith('settings-')) return ['header', 'body', 'scroll']; + if (state === 'search') return ['header', 'body']; + if (requiresOverlay(id) || ['clipped-modal', 'overlay-padding'].includes(state)) + return ['header', 'body', 'scroll', 'footer']; + return []; +} + +export function seededObservedFailures(seed) { + if (!seed?.geometry) return ['missing seeded geometry']; + const errors = geometryFailures(seed.geometry); + if (['seed/heading-offset', 'seed/context-invalid-header'].includes(seed.id)) + errors.push( + ...pageHeaderFailures( + seed.geometry.primaryHeader, + seed.geometry.rem, + 'Activity', + seed.geometry + ) + ); + if (seed.id === 'seed/dead-header') { + const action = seed.behavior; + if ( + action?.control === 'New Task' && + action.visibleBefore === true && + action.enabledBefore === true && + action.dialogVisibleBefore === false && + action.clickCompleted === true && + action.dialogVisibleAfter === false && + Number.isFinite(action.waitedMs) && + action.waitedMs >= 1000 + ) + errors.push('dead header control'); + } + return errors; +} + export async function fileDigest(file) { const hash = createHash('sha256'); for await (const chunk of createReadStream(file)) hash.update(chunk); @@ -108,6 +159,7 @@ export async function packageDigest(directory) { export function geometryFailures(g) { if (!g) return ['missing geometry']; const failures = []; + if (!Number.isFinite(g.rem) || g.rem <= 0) failures.push('missing root font size'); const validRect = (r) => [r.x, r.y, r.width, r.height, r.right, r.bottom].every(Number.isFinite) && r.width > 0 && @@ -137,6 +189,7 @@ export function geometryFailures(g) { ) failures.push('shell does not fill viewport'); for (const overlay of g.overlays ?? []) { + failures.push(...overlayInsetFailures(overlay, g.rem)); if (!validRect(overlay) || !Number.isFinite(overlay.overflow)) failures.push('unmeasured overlay'); if (!Number.isFinite(overlay.opacity) || overlay.opacity < 0.999 || overlay.opacity > 1) @@ -191,6 +244,22 @@ export function evidenceFailures(report, expected, now = Date.now()) { if (!Number.isFinite(completed) || completed > now || now - completed > 24 * 60 * 60 * 1000) errors.push('missing or stale native evidence'); const entries = report.entries ?? []; + const seeds = report.seededFailures ?? []; + if ( + seeds.length !== seededCases.length || + new Set(seeds.map((seed) => seed.id)).size !== seededCases.length + ) + errors.push('missing or duplicate seeded renderer checks'); + for (const [id, reason] of seededCases) { + const seed = seeds.find((entry) => entry.id === `seed/${id}`); + if ( + seed?.status !== 'detected' || + !seed.observedFailures?.includes(reason) || + !seededObservedFailures(seed).includes(reason) || + !seed.screenshot?.path + ) + errors.push(`seed/${id}: injected renderer failure not detected`); + } if ( entries.length !== requiredEntries.length || new Set(entries.map((e) => e.id)).size !== requiredEntries.length @@ -231,6 +300,21 @@ export function evidenceFailures(report, expected, now = Date.now()) { errors.push(`${id}: missing native environment`); if (requiresOverlay(id) && !entry.geometry?.overlays?.length) errors.push(`${id}: missing required overlay`); + for (const overlay of entry.geometry?.overlays ?? []) { + for (const kind of requiredOverlayParts(id)) + if (!overlay.parts?.some((part) => part.insetKind === kind)) + errors.push(`${id}: missing required overlay ${kind}`); + } + const route = routes.find(([name]) => id.endsWith(`/route-${name}`)); + if (route) + errors.push( + ...pageHeaderFailures( + entry.geometry?.primaryHeader, + entry.geometry?.rem, + route[2], + entry.geometry + ).map((error) => `${id}: ${error}`) + ); errors.push(...geometryFailures(entry.geometry).map((error) => `${id}: ${error}`)); errors.push( ...geometryFailures(entry.completionGeometry).map( @@ -238,5 +322,34 @@ export function evidenceFailures(report, expected, now = Date.now()) { ) ); } + for (const mode of modes) { + const headers = routes.map( + ([name]) => + entries.find((entry) => entry.id === `${mode.id}/route-${name}`)?.geometry?.primaryHeader + ); + const baseline = headers[0]; + if (!baseline) continue; + for (const header of headers.slice(1)) { + if (!header) continue; + const values = [ + [header.header?.y, baseline.header?.y], + [header.title?.y, baseline.title?.y], + ]; + if (mode.width === 1700) + values.push( + [header.header?.height, baseline.header?.height], + [header.content?.y, baseline.content?.y] + ); + if ( + values.some( + ([actual, expected]) => + !Number.isFinite(actual) || + !Number.isFinite(expected) || + Math.abs(actual - expected) > 2 + ) + ) + errors.push(`${mode.id}: inconsistent route header baseline`); + } + } return errors; } diff --git a/scripts/native-ui/contract.test.mjs b/scripts/native-ui/contract.test.mjs index 90e13749..879d052d 100644 --- a/scripts/native-ui/contract.test.mjs +++ b/scripts/native-ui/contract.test.mjs @@ -1,3 +1,4 @@ +/* global structuredClone */ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { @@ -6,8 +7,11 @@ import { modes, requiredEntries, requiresOverlay, + requiredOverlayParts, packageDigest, schema, + routes, + seededCases, } from './contract.mjs'; import { crc32, deflateSync } from 'node:zlib'; import { screenshotSize } from './png.mjs'; @@ -33,7 +37,7 @@ test('bundle identity includes web resources and rejects external symlinks', asy await assert.rejects(packageDigest(bundle), /escapes bundle/); }); function report() { - return { + const value = { schema, status: 'passed', boundary: 'packaged-macos', @@ -49,6 +53,12 @@ function report() { osVersion: '26.0', }, completedAt: new Date(now).toISOString(), + seededFailures: seededCases.map(([id, reason]) => ({ + id: `seed/${id}`, + status: 'detected', + observedFailures: [reason], + screenshot: { path: `seed--${id}.png`, sha256: digest }, + })), entries: requiredEntries.map((id) => { const mode = modes.find((mode) => id.startsWith(`${mode.id}/`)); return { @@ -62,6 +72,7 @@ function report() { scaleFactor: 2, }, completionGeometry: { + rem: 16, width: mode.width, height: mode.height, scrollWidth: mode.width, @@ -70,6 +81,19 @@ function report() { overlays: [], }, geometry: { + rem: 16, + primaryHeader: { + header: { x: 0, y: 0, width: 1000, height: 82, right: 1000, bottom: 82 }, + title: { x: 52, y: 0, width: 200, height: 32, right: 252, bottom: 32 }, + back: { x: 0, y: 2, width: 40, height: 40, right: 40, bottom: 42 }, + content: { x: 0, y: 82, width: 1000, height: 500, right: 1000, bottom: 582 }, + titleText: routes.find(([name]) => id.endsWith(`/route-${name}`))?.[2], + titleCount: 1, + backText: '', + boardRailControls: 0, + fontSize: 24, + lineHeight: 32, + }, width: mode.width, height: mode.height, scrollWidth: mode.width, @@ -86,16 +110,26 @@ function report() { bottom: mode.height - 1, overflow: 0, opacity: 1, - parts: ['header', 'body'].map((name) => ({ + parts: requiredOverlayParts(id).map((kind) => ({ x: 2, y: 2, width: 100, height: 30, right: 102, bottom: 32, - name, + name: kind.includes('header') + ? 'header' + : kind === 'footer' + ? 'footer' + : 'body', + insetKind: kind, visible: true, - padding: [16, 16, 16, 16], + padding: + kind === 'header' + ? [10, 16, 10, 16] + : kind === 'task-body' + ? [0, 0, 0, 0] + : [16, 16, 16, 16], })), }, ] @@ -104,6 +138,39 @@ function report() { }; }), }; + for (const seed of value.seededFailures) { + const state = ['seed/clipped-modal', 'seed/overlay-padding'].includes(seed.id) + ? 'create-task' + : 'route-activity'; + seed.geometry = structuredClone( + value.entries.find((entry) => entry.id === `light-normal/${state}`).geometry + ); + if (seed.id === 'seed/shell-blank') seed.geometry.shell.height -= 100; + if (seed.id === 'seed/clipped-modal') { + const overlay = seed.geometry.overlays[0]; + overlay.x += 1700; + overlay.right += 1700; + } + if (seed.id === 'seed/heading-offset') { + seed.geometry.primaryHeader.title.y += 16; + seed.geometry.primaryHeader.title.bottom += 16; + } + if (seed.id === 'seed/context-invalid-header') + seed.geometry.primaryHeader.boardRailControls = 1; + if (seed.id === 'seed/overlay-padding') + seed.geometry.overlays[0].parts[0].padding = [0, 0, 0, 0]; + if (seed.id === 'seed/dead-header') + seed.behavior = { + control: 'New Task', + visibleBefore: true, + enabledBefore: true, + dialogVisibleBefore: false, + clickCompleted: true, + waitedMs: 1000, + dialogVisibleAfter: false, + }; + } + return value; } const failures = (value) => evidenceFailures(value, { commit, packageDigest: digest, version: '6.1.6' }, now); @@ -114,6 +181,8 @@ test('partial, duplicate and failed matrices cannot pass', () => { (r) => r.entries.pop(), (r) => (r.entries[1] = r.entries[0]), (r) => (r.entries[0].status = 'failed'), + (r) => r.seededFailures.pop(), + (r) => (r.seededFailures[0].observedFailures = []), ]) { const r = report(); mutate(r); @@ -155,6 +224,7 @@ test('seeded clipped overlay and unreachable footer fail', () => { }, ]; assert.deepEqual(geometryFailures(g), [ + 'inconsistent overlay footer padding', 'unmeasured overlay', 'transparent or unmeasured overlay', 'missing overlay header/body', @@ -162,6 +232,116 @@ test('seeded clipped overlay and unreachable footer fail', () => { 'unreachable or unmeasured overlay footer', ]); }); +test('heading, Back, context-invalid controls and inconsistent route baselines fail closed', () => { + for (const mutate of [ + (h) => { + h.title.y += 8; + }, + (h) => { + h.back.width = 20; + }, + (h) => { + h.fontSize = 12; + }, + (h) => { + h.backText = 'Back'; + }, + (h) => { + h.boardRailControls = 1; + }, + (h) => { + h.content.y += 20; + }, + (h) => { + for (const key of ['header', 'title', 'back', 'content']) { + h[key].x -= 2000; + h[key].right -= 2000; + } + }, + (h) => { + h.back.right = 999; + }, + (h) => { + h.title.width = 1200; + h.title.right = h.title.x + 1200; + }, + ]) { + const r = report(); + mutate( + r.entries.find((entry) => entry.id === 'light-normal/route-workflows').geometry.primaryHeader + ); + assert(failures(r).length); + } +}); +test('insets use the shared rem contract, including zero-padding compound bodies', () => { + const r = report(); + const g = r.entries.find((entry) => entry.id === 'light-normal/create-task').geometry; + const overlay = g.overlays[0]; + overlay.parts[1].padding = [0, 0, 0, 0]; + assert(geometryFailures(g).includes('inconsistent overlay body padding')); + overlay.compound = true; + assert.deepEqual(geometryFailures(g), []); + overlay.parts[0].padding[1] = 4; + assert(geometryFailures(g).includes('inconsistent overlay header padding')); +}); +test('claimed seed detection requires retained defective measurements and behavioral evidence', () => { + for (const seedIndex of seededCases.keys()) { + const r = report(); + const seed = r.seededFailures[seedIndex]; + delete seed.geometry; + assert(failures(r).some((error) => error.startsWith(seed.id))); + } + for (const seedIndex of seededCases.keys()) { + const r = report(); + const seed = r.seededFailures[seedIndex]; + seed.geometry = structuredClone( + r.entries.find((entry) => entry.id === 'light-normal/route-activity').geometry + ); + delete seed.behavior; + assert(failures(r).some((error) => error.startsWith(seed.id))); + } + for (const mutate of [ + (b) => { + b.clickCompleted = false; + }, + (b) => { + b.visibleBefore = false; + }, + (b) => { + b.enabledBefore = false; + }, + (b) => { + b.dialogVisibleBefore = true; + }, + (b) => { + b.dialogVisibleAfter = true; + }, + (b) => { + b.waitedMs = 10; + }, + ]) { + const r = report(); + mutate(r.seededFailures.find((seed) => seed.id === 'seed/dead-header').behavior); + assert(failures(r).some((error) => error.startsWith('seed/dead-header'))); + } +}); +test('required footer and compound scroll measurements cannot disappear', () => { + for (const [state, kind] of [ + ['confirmation', 'footer'], + ['create-task', 'scroll'], + ['settings-general', 'scroll'], + ]) { + const r = report(); + const overlay = r.entries.find((entry) => entry.id === `light-normal/${state}`).geometry + .overlays[0]; + overlay.parts = overlay.parts.filter((part) => part.insetKind !== kind); + assert(failures(r).some((error) => error.includes(`missing required overlay ${kind}`))); + } + const g = report().entries.find((entry) => entry.id === 'light-normal/create-task').geometry; + g.overlays[0].compound = true; + g.overlays[0].parts = g.overlays[0].parts.filter((part) => part.insetKind !== 'scroll'); + assert(geometryFailures(g).includes('missing compound overlay scroll')); +}); test('missing overlay measurements and invalid native bounds fail closed', () => { for (const mutate of [ (r) => (r.entries.find((e) => requiresOverlay(e.id)).geometry.overlays = []), diff --git a/scripts/native-ui/layout-contract.mjs b/scripts/native-ui/layout-contract.mjs new file mode 100644 index 00000000..889fac11 --- /dev/null +++ b/scripts/native-ui/layout-contract.mjs @@ -0,0 +1,94 @@ +/* global document, getComputedStyle */ + +// Serializable renderer probe. This reads production CSS; it never repairs layout. +export function measurePageHeader() { + const page = document.querySelector('[data-page-shell="primary"]'); + if (!page) return null; + const rect = (el) => { + if (!el) return null; + const { x, y, width, height, right, bottom } = el.getBoundingClientRect(); + return { x, y, width, height, right, bottom }; + }; + const title = page.querySelector('h1'); + const back = page.querySelector('button[aria-label="Back"]'); + return { + header: rect(page.querySelector('.primary-page-header')), + title: rect(title), + back: rect(back), + content: rect(page.querySelector('.primary-page-content')), + titleText: title?.textContent?.trim(), + titleCount: document.querySelectorAll('h1').length, + backText: back?.textContent?.trim(), + fontSize: title ? parseFloat(getComputedStyle(title).fontSize) : null, + lineHeight: title ? parseFloat(getComputedStyle(title).lineHeight) : null, + boardRailControls: document.querySelectorAll( + '[aria-label="Expand right sidebar"], [aria-label="Collapse right sidebar"]' + ).length, + }; +} + +const near = (actual, expected) => Number.isFinite(actual) && Math.abs(actual - expected) <= 1; +const measured = (r) => + r && + [r.x, r.y, r.width, r.height, r.right, r.bottom].every(Number.isFinite) && + r.width > 0 && + r.height > 0 && + near(r.right, r.x + r.width) && + near(r.bottom, r.y + r.height); +const contains = (outer, inner) => + inner.x >= outer.x - 1 && + inner.y >= outer.y - 1 && + inner.right <= outer.right + 1 && + inner.bottom <= outer.bottom + 1; + +export function pageHeaderFailures(header, rem, expectedTitle, viewport) { + if (!header || !['header', 'title', 'back', 'content'].every((key) => measured(header[key]))) + return ['missing primary header geometry']; + const errors = []; + if ( + !viewport || + ![viewport.width, viewport.height].every((n) => Number.isFinite(n) && n > 0) || + !contains({ x: 0, y: 0, right: viewport.width, bottom: viewport.height }, header.header) || + !contains(header.header, header.title) || + !contains(header.header, header.back) + ) + errors.push('clipped primary heading/Back geometry'); + if (header.titleCount !== 1 || header.titleText !== expectedTitle || header.backText !== '') + errors.push('invalid primary heading or Back control'); + if (header.boardRailControls !== 0) errors.push('context-invalid board rail control'); + if ( + !near(header.back.width, 2.5 * rem) || + !near(header.back.height, 2.5 * rem) || + !near(header.back.y - header.header.y, 0.125 * rem) || + !near(header.title.x - header.back.right, 0.75 * rem) || + !near(header.title.y, header.header.y) || + !near(header.fontSize, 1.5 * rem) || + !near(header.lineHeight, 2 * rem) + ) + errors.push('inconsistent primary heading/Back geometry'); + return errors; +} + +export function overlayInsetFailures(overlay, rem) { + const errors = []; + if (overlay.compound && !overlay.parts?.some((part) => part.insetKind === 'scroll')) + errors.push('missing compound overlay scroll'); + for (const part of overlay.parts ?? []) { + const units = { + header: [0.625, 1, 0.625, 1], + body: overlay.compound ? [0, 0, 0, 0] : [1, 1, 1, 1], + footer: [1, 1, 1, 1], + scroll: [1, 1, 1, 1], + 'task-header': [1, 1, 1, 1], + 'task-body': [0, 0, 0, 0], + }[part.insetKind]; + if ( + !units || + !Array.isArray(part.padding) || + part.padding.length !== 4 || + !units.every((value, index) => near(part.padding[index], value * rem)) + ) + errors.push(`inconsistent overlay ${part.name} padding`); + } + return errors; +} diff --git a/scripts/native-ui/run.mjs b/scripts/native-ui/run.mjs index d0e6bcec..55dd6c90 100644 --- a/scripts/native-ui/run.mjs +++ b/scripts/native-ui/run.mjs @@ -10,6 +10,7 @@ import path from 'node:path'; import { _electron, expect } from '@playwright/test'; import { fileDigest, + evidenceFailures, geometryFailures, modes, packageDigest, @@ -17,8 +18,11 @@ import { schema, settingsSections, states, + seededCases, } from './contract.mjs'; import { installPreviewFixture } from './preview-fixture.mjs'; +import { measurePageHeader, pageHeaderFailures } from './layout-contract.mjs'; +import { performance } from 'node:perf_hooks'; const root = path.resolve(import.meta.dirname, '../..'); const [packageArgument, outputArgument] = process.argv.slice(2); @@ -168,7 +172,7 @@ async function configure(mode) { await expect(page.getByRole('button', { name: 'Settings', exact: true })).toBeVisible(); } async function metrics() { - return page.evaluate(() => { + const geometry = await page.evaluate(() => { const shell = document.querySelector('.desktop-app-shell'); if (!shell) throw new Error('Missing production desktop shell'); const rect = (el) => { @@ -187,6 +191,7 @@ async function metrics() { return { width: innerWidth, height: innerHeight, + rem: parseFloat(getComputedStyle(document.documentElement).fontSize), scrollWidth: document.documentElement.scrollWidth, scrollHeight: document.documentElement.scrollHeight, shell: rect(shell), @@ -196,16 +201,28 @@ async function metrics() { ), ].map((el) => ({ ...rect(el), + compound: !!el.closest('[data-overlay-compound]'), opacity: paintedOpacity(el), overflow: el.scrollWidth - el.clientWidth, parts: [ ...el.querySelectorAll( - '.vk-overlay-header, .vk-overlay-body, .vk-overlay-footer, .vk-task-workspace-header, .mantine-Drawer-body' + '.vk-overlay-header, .vk-overlay-body, .vk-overlay-footer, .vk-overlay-scroll, .vk-task-workspace-header, .mantine-Drawer-body' ), ].map((part) => { const style = getComputedStyle(part); const b = rect(part); return { + insetKind: part.matches('.vk-task-workspace-header') + ? 'task-header' + : part.matches('.mantine-Drawer-body') + ? 'task-body' + : part.matches('.vk-overlay-scroll') + ? 'scroll' + : part.matches('.vk-overlay-header') + ? 'header' + : part.matches('.vk-overlay-footer') + ? 'footer' + : 'body', name: part.matches('.vk-overlay-header, .vk-task-workspace-header') ? 'header' : part.matches('.vk-overlay-footer') @@ -230,6 +247,8 @@ async function metrics() { })), }; }); + geometry.primaryHeader = await page.evaluate(measurePageHeader); + return geometry; } async function capture(entry) { // Native capturePage does not wait for CSS transitions as Playwright screenshots do. @@ -259,6 +278,18 @@ async function capture(entry) { entry.nativeWindow = native; entry.screenshot = { path: name, sha256: await fileDigest(path.join(output, name)) }; assert.deepEqual(geometryFailures(entry.geometry), [], entry.id); + const route = routes.find(([name]) => entry.id.endsWith(`/route-${name}`)); + if (route) + assert.deepEqual( + pageHeaderFailures( + entry.geometry.primaryHeader, + entry.geometry.rem, + route[2], + entry.geometry + ), + [], + entry.id + ); } const button = (name) => page.getByRole('button', { name, exact: true }); async function dismiss(dialog, opener) { @@ -581,6 +612,111 @@ async function exercise(state, mode, shot) { throw new Error(`Required scenario has not been implemented: ${state}`); } } +async function checkSeededRendererFailures() { + report.seededFailures = []; + for (const [id, expectedFailure] of seededCases) { + const entry = { id: `seed/${id}`, status: 'running' }; + report.seededFailures.push(entry); + try { + await configure(modes[0]); + if (id === 'heading-offset' || id === 'context-invalid-header') { + await page + .getByRole('complementary', { name: 'Desktop navigation' }) + .getByRole('button', { name: 'Activity', exact: true }) + .click(); + await expect(page.getByRole('heading', { name: 'Activity', level: 1 })).toBeFocused(); + } + if (id === 'clipped-modal' || id === 'overlay-padding') { + await button('New Task').click(); + await expect(page.getByRole('dialog', { name: 'Create New Task' })).toBeVisible(); + } + await page.evaluate((fault) => { + if (fault === 'shell-blank') + document + .querySelector('.desktop-app-shell') + .style.setProperty('height', `${innerHeight - 100}px`, 'important'); + else if (fault === 'heading-offset') + document.querySelector('[data-page-shell="primary"] h1').style.transform = + 'translateY(16px)'; + else if (fault === 'context-invalid-header') { + const button = document.createElement('button'); + button.setAttribute('aria-label', 'Expand right sidebar'); + button.textContent = 'Invalid board control'; + document.querySelector('[aria-label="Main navigation"]').append(button); + } else if (fault === 'clipped-modal') + document + .querySelector('[data-overlay-active="true"] .vk-overlay-content') + .style.setProperty('transform', `translateX(${innerWidth}px)`, 'important'); + else if (fault === 'overlay-padding') + document + .querySelector('[data-overlay-active="true"] .vk-overlay-header') + .style.setProperty('padding', '0px', 'important'); + else if (fault === 'dead-header') { + const button = [ + ...document.querySelectorAll('[aria-label="Main navigation"] button'), + ].find((element) => element.textContent.trim() === 'New Task'); + button.replaceWith(button.cloneNode(true)); // Remove React's target handler, retain visible UI. + } + }, id); + entry.observedFailures = []; + if (id === 'dead-header') { + entry.behavior = { + control: 'New Task', + visibleBefore: await button('New Task').isVisible(), + enabledBefore: await button('New Task').isEnabled(), + dialogVisibleBefore: await page + .getByRole('dialog', { name: 'Create New Task' }) + .isVisible(), + }; + await button('New Task').click(); + entry.behavior.clickCompleted = true; + const started = performance.now(); + try { + await expect(page.getByRole('dialog', { name: 'Create New Task' })).toBeVisible({ + timeout: 1000, + }); + } catch (error) { + if (!error.message.includes('toBeVisible')) throw error; + entry.observedFailures.push('dead header control'); + } + entry.behavior.waitedMs = performance.now() - started; + entry.behavior.dialogVisibleAfter = await page + .getByRole('dialog', { name: 'Create New Task' }) + .isVisible(); + } + try { + await capture(entry); + } catch (error) { + // Geometry faults must retain their native capture, then be identified by the same validator. + if (!entry.screenshot) throw error; + entry.captureAssertion = error.message; + } + entry.observedFailures.push(...geometryFailures(entry.geometry)); + if (id === 'heading-offset' || id === 'context-invalid-header') + entry.observedFailures.push( + ...pageHeaderFailures( + entry.geometry.primaryHeader, + entry.geometry.rem, + 'Activity', + entry.geometry + ) + ); + assert( + entry.observedFailures.includes(expectedFailure), + `Injected fault was not detected: ${id}` + ); + entry.status = 'detected'; + } catch (error) { + entry.status = 'failed'; + entry.error = error.message; + } finally { + // Faults only affect this disposable renderer; reload removes all injected DOM/CSS. + await page.goto(origin); + } + await persist(); + console.log(`${entry.status}: ${entry.id}`); + } +} try { await launch(); for (const mode of modes) { @@ -645,6 +781,7 @@ try { await persist(); } } + await checkSeededRendererFailures(); report.status = report.entries.every((entry) => entry.status === 'passed') ? 'passed' : 'failed'; } catch (error) { report.status = 'failed'; @@ -652,6 +789,12 @@ try { } finally { if (app) await app.close(); report.completedAt = new Date().toISOString(); + report.validationErrors = evidenceFailures(report, { + commit, + version, + packageDigest: report.packageDigest, + }); + if (report.validationErrors.length) report.status = 'failed'; await persist(); } console.log(`Native evidence: ${path.join(output, 'evidence.json')} (${report.status})`); diff --git a/scripts/native-ui/verify.mjs b/scripts/native-ui/verify.mjs index f156a277..c48f5478 100644 --- a/scripts/native-ui/verify.mjs +++ b/scripts/native-ui/verify.mjs @@ -14,7 +14,7 @@ export async function verifyNativeEvidence({ evidencePath, appPath, commit, vers packageDigest: await packageDigest(appPath), }); const directory = await realpath(path.dirname(evidencePath)); - for (const entry of report.entries ?? []) { + for (const entry of [...(report.entries ?? []), ...(report.seededFailures ?? [])]) { const name = entry.screenshot?.path; if ( typeof name !== 'string' ||