diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bb5e967..c883f0a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,7 @@ jobs: run: >- node --test scripts/check-actions-pinned.test.mjs + scripts/check-browser-milestone.test.mjs scripts/check-delivery-cadence.test.mjs scripts/check-security-gates.test.mjs scripts/check-tracked-ignore.test.mjs diff --git a/.github/workflows/scheduled-qa.yml b/.github/workflows/scheduled-qa.yml index 8d2e36ce..a40d7345 100644 --- a/.github/workflows/scheduled-qa.yml +++ b/.github/workflows/scheduled-qa.yml @@ -38,18 +38,23 @@ env: jobs: playwright: - name: Playwright E2E + name: Playwright E2E (${{ matrix.shard }}/2) if: >- github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:full') runs-on: ubuntu-latest timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - name: Record milestone selection run: | { echo "### Playwright E2E milestone" echo + echo "- Shard: ${{ matrix.shard }}/2 (isolated runner, one worker)" echo "- Trigger: \`$GITHUB_EVENT_NAME\`" echo "- Reason: explicit \`ci:full\`, scheduled, or manual milestone" } >> "$GITHUB_STEP_SUMMARY" @@ -81,13 +86,13 @@ jobs: env: PLAYWRIGHT_HTML_REPORT: '1' PLAYWRIGHT_HTML_OPEN: never - run: pnpm test:e2e + run: pnpm test:e2e --shard=${{ matrix.shard }}/2 - name: Upload Playwright artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: playwright-artifacts + name: playwright-artifacts-${{ matrix.shard }} path: | playwright-report/ test-results/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3ed10e06..7becaa70 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -326,6 +326,13 @@ Follow the existing conventions in `.eslintrc.*`, `.prettierrc`, and `tsconfig.j Playwright does not retry failures. Screenshots and traces from the first failure are retained in `test-results/` and uploaded by Scheduled QA. + Scheduled QA splits the complete browser inventory across two isolated runners, + each with its own server/data directory and one worker. Both shards must pass; + a failing shard does not cancel the other. Existing test timeouts and the + 25-minute job limit are unchanged. Download `playwright-artifacts-1` and + `playwright-artifacts-2` for their separate reports and failure evidence. + A cancelled or incomplete shard is not a passing milestone. + - **Load smoke tests** use [k6](https://k6.io/): ```bash diff --git a/scripts/check-browser-milestone.test.mjs b/scripts/check-browser-milestone.test.mjs new file mode 100644 index 00000000..c9e9b77a --- /dev/null +++ b/scripts/check-browser-milestone.test.mjs @@ -0,0 +1,24 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; + +const workflow = readFileSync( + new URL('../.github/workflows/scheduled-qa.yml', import.meta.url), + 'utf8' +); +const browser = workflow.slice(workflow.indexOf(' playwright:'), workflow.indexOf('\n k6:')); +const config = readFileSync(new URL('../playwright.config.ts', import.meta.url), 'utf8'); + +test('browser milestone uses two independent bounded shards without dropping failure evidence', () => { + assert.match(browser, /timeout-minutes: 25/); + assert.match(browser, /fail-fast: false/); + assert.match(browser, /shard: \[1, 2\]/); + assert.match(browser, /pnpm test:e2e --shard=\$\{\{ matrix\.shard \}\}\/2/); + assert.match(browser, /VERITAS_DATA_DIR=\$RUNNER_TEMP\/veritas-playwright-data/); + assert.match(browser, /name: playwright-artifacts-\$\{\{ matrix\.shard \}\}/); + assert.match(browser, /if: always\(\)/); + assert.doesNotMatch(browser, /continue-on-error: true|--retries|--workers|--grep|--max-failures/); + assert.match(config, /fullyParallel: false/); + assert.match(config, /retries: 0/); + assert.match(config, /workers: 1/); +});